diff --git a/cmd/unleash.go b/cmd/unleash.go index 1db91dc7..59858f85 100644 --- a/cmd/unleash.go +++ b/cmd/unleash.go @@ -19,9 +19,9 @@ package cmd import ( "github.com/k3rn31/gremlins/coverage" "github.com/k3rn31/gremlins/log" - "github.com/k3rn31/gremlins/mutant" "github.com/k3rn31/gremlins/mutator" "github.com/k3rn31/gremlins/mutator/workdir" + "github.com/k3rn31/gremlins/report" "github.com/spf13/cobra" "io/ioutil" "os" @@ -77,25 +77,7 @@ func newUnleashCmd() *unleashCmd { mutator.WithBuildTags(buildTags)) results := mut.Run() - // Temporary reporting - var k int - var l int - var nc int - for _, m := range results { - if m.Status() == mutant.Killed { - k++ - } - if m.Status() == mutant.Lived { - l++ - } - if m.Status() == mutant.NotCovered { - nc++ - } - } - log.Infoln("-----") - log.Infof("Killed: %d, Lived: %d, Not covered: %d\n", k, l, nc) - log.Infof("Real coverage: %.2f%%\n", float64(k+l)/float64(k+l+nc)*100) - log.Infof("Test efficacy: %.2f%%\n", float64(k)/float64(k+l)*100) + report.Do(results) }, } diff --git a/codecov.yaml b/codecov.yaml index ffa5cbba..98673ac8 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -2,9 +2,8 @@ coverage: status: project: default: - target: 85% + target: auto threshold: 5% patch: default: - target: 85% - threshold: 2% + informational: true diff --git a/go.mod b/go.mod index 1eb7e9ef..ffcb8ca0 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( ) require ( + github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect diff --git a/go.sum b/go.sum index 91ce2c28..286cba7e 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b h1:wDUNC2eKiL35DbLvsDhiblTUXHxcOPwQSCzi7xpQUN4= +github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= diff --git a/log/log.go b/log/log.go index 89775152..6301baf1 100644 --- a/log/log.go +++ b/log/log.go @@ -19,16 +19,11 @@ package log import ( "fmt" "github.com/fatih/color" - "github.com/k3rn31/gremlins/mutant" "io" "sync" ) -var ( - fgRed = color.New(color.FgRed).SprintFunc() - fgGreen = color.New(color.FgGreen).SprintFunc() - fgHiBlack = color.New(color.FgHiBlack).SprintFunc() -) +var fgRed = color.New(color.FgRed).SprintFunc() type log struct { writer io.Writer @@ -95,33 +90,6 @@ func Errorln(a any) { instance.writeln(msg) } -// Mutant logs a mutant.Mutant. -// It reports the mutant.Status, the mutant.Type and its position. -func Mutant(m mutant.Mutant) { - if instance == nil { - return - } - status := m.Status().String() - switch m.Status() { - case mutant.Killed, mutant.Runnable: - status = fgGreen(m.Status()) - case mutant.Lived: - status = fgRed(m.Status()) - case mutant.NotCovered: - status = fgHiBlack(m.Status()) - } - instance.writef("%s%s %s at %s\n", padding(m.Status()), status, m.Type(), m.Position()) -} - -func padding(s mutant.Status) string { - var pad string - padLen := 12 - len(s.String()) - for i := 0; i < padLen; i++ { - pad += " " - } - return pad -} - func (l *log) writef(f string, args ...any) { _, _ = fmt.Fprintf(instance.writer, f, args...) } diff --git a/log/log_test.go b/log/log_test.go index 2fde90a9..b540c9fc 100644 --- a/log/log_test.go +++ b/log/log_test.go @@ -18,10 +18,7 @@ package log_test import ( "bytes" - "github.com/google/go-cmp/cmp" "github.com/k3rn31/gremlins/log" - "github.com/k3rn31/gremlins/mutant" - "go/token" "testing" ) @@ -106,76 +103,3 @@ func TestLogError(t *testing.T) { } }) } - -func TestMutantLog(t *testing.T) { - out := &bytes.Buffer{} - defer out.Reset() - log.Init(out) - defer log.Reset() - - m := stubMutant{mutant.Lived} - log.Mutant(m) - m = stubMutant{mutant.Killed} - log.Mutant(m) - m = stubMutant{mutant.NotCovered} - log.Mutant(m) - m = stubMutant{mutant.Runnable} - log.Mutant(m) - - got := out.String() - - want := "" + - " LIVED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + - " KILLED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + - " NOT COVERED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + - " RUNNABLE CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" - - if !cmp.Equal(got, want) { - t.Errorf(cmp.Diff(got, want)) - } -} - -type stubMutant struct { - status mutant.Status -} - -func (s stubMutant) Type() mutant.Type { - return mutant.ConditionalsBoundary -} - -func (s stubMutant) SetType(_ mutant.Type) { - panic("implement me") -} - -func (s stubMutant) Status() mutant.Status { - return s.status -} - -func (s stubMutant) SetStatus(_ mutant.Status) { - panic("implement me") -} - -func (s stubMutant) Position() token.Position { - return token.Position{ - Filename: "aFolder/aFile.go", - Offset: 0, - Line: 12, - Column: 3, - } -} - -func (s stubMutant) Pos() token.Pos { - return 123 -} - -func (s stubMutant) SetWorkdir(_ string) { - panic("implement me") -} - -func (s stubMutant) Apply() error { - panic("implement me") -} - -func (s stubMutant) Rollback() error { - panic("implement me") -} diff --git a/mutator/internal/mappings.go b/mutator/internal/mappings.go index 55794bb9..a1f09b0e 100644 --- a/mutator/internal/mappings.go +++ b/mutator/internal/mappings.go @@ -21,6 +21,8 @@ import ( "go/token" ) +// TokenMutantType is the mapping from each token.Token and all the +// mutant.Type that can be applied to it. var TokenMutantType = map[token.Token][]mutant.Type{ token.SUB: {mutant.InvertNegatives, mutant.ArithmeticBase}, token.ADD: {mutant.ArithmeticBase}, diff --git a/mutator/internal/tokenmutant.go b/mutator/internal/tokenmutant.go index 5addfd91..daaff763 100644 --- a/mutator/internal/tokenmutant.go +++ b/mutator/internal/tokenmutant.go @@ -45,18 +45,22 @@ func NewTokenMutant(set *token.FileSet, file *ast.File, node *NodeToken) *TokenM } } +// Type returns the mutant.Type of the mutant.Mutant. func (m *TokenMutant) Type() mutant.Type { return m.mutantType } +// SetType sets the mutant.Type of the mutant.Mutant. func (m *TokenMutant) SetType(mt mutant.Type) { m.mutantType = mt } +// Status returns the mutant.Status of the mutant.Mutant. func (m *TokenMutant) Status() mutant.Status { return m.status } +// SetStatus sets the mutant.Status of the mutant.Mutant. func (m *TokenMutant) SetStatus(s mutant.Status) { m.status = s } diff --git a/mutator/mutator.go b/mutator/mutator.go index 9ba123e9..2f0778d5 100644 --- a/mutator/mutator.go +++ b/mutator/mutator.go @@ -22,6 +22,7 @@ import ( "github.com/k3rn31/gremlins/mutant" "github.com/k3rn31/gremlins/mutator/internal" "github.com/k3rn31/gremlins/mutator/workdir" + "github.com/k3rn31/gremlins/report" "go/ast" "go/parser" "go/token" @@ -31,6 +32,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" ) // Mutator is the "engine" that performs the mutation testing. @@ -128,7 +130,8 @@ func WithApplyAndRollback(a func(m mutant.Mutant) error, r func(m mutant.Mutant) // KILLED or LIVED depending on the result. If the tests pass, it means the // TokenMutant survived, so it will be LIVED, if the tests fail, the TokenMutant will // be KILLED. -func (mu Mutator) Run() []mutant.Mutant { +func (mu Mutator) Run() report.Results { + start := time.Now() log.Infoln("Looking for mutants...") mu.mutantStream = make(chan mutant.Mutant) go func() { @@ -141,8 +144,11 @@ func (mu Mutator) Run() []mutant.Mutant { }) close(mu.mutantStream) }() + res := mu.executeTests() + end := time.Now() + res.Elapsed = end.Sub(start) - return mu.executeTests() + return res } func (mu Mutator) runOnFile(fileName string, src io.Reader) { @@ -182,7 +188,7 @@ func (mu Mutator) mutationStatus(pos token.Position) mutant.Status { return status } -func (mu Mutator) executeTests() []mutant.Mutant { +func (mu Mutator) executeTests() report.Results { if mu.dryRun { log.Infoln("Running in 'dry-run' mode.") } else { @@ -195,12 +201,12 @@ func (mu Mutator) executeTests() []mutant.Mutant { defer cl() _ = os.Chdir(wd) - var results []mutant.Mutant + var mutants []mutant.Mutant for m := range mu.mutantStream { m.SetWorkdir(wd) if m.Status() == mutant.NotCovered || mu.dryRun { - results = append(results, m) - log.Mutant(m) + mutants = append(mutants, m) + report.Mutant(m) continue } if err := mu.apply(m); err != nil { @@ -221,8 +227,12 @@ func (mu Mutator) executeTests() []mutant.Mutant { log.Errorf("failed to restore mutation at %s - %s\n\t%v", m.Position(), m.Status(), err) // What should we do now? } - log.Mutant(m) - results = append(results, m) + report.Mutant(m) + mutants = append(mutants, m) + } + + results := report.Results{ + Mutants: mutants, } return results } diff --git a/mutator/mutator_test.go b/mutator/mutator_test.go index 644e9c34..b25e0db7 100644 --- a/mutator/mutator_test.go +++ b/mutator/mutator_test.go @@ -233,7 +233,8 @@ func TestMutations(t *testing.T) { filename: {Data: src}, } mut := mutator.New(mapFS, tc.covProfile, dealerStub{}, mutator.WithDryRun(true)) - got := mut.Run() + res := mut.Run() + got := res.Mutants if tc.token == token.ILLEGAL { if len(got) != 0 { @@ -265,7 +266,8 @@ func TestSkipTestAndNonGoFiles(t *testing.T) { "folder1/file": {Data: file}, } mut := mutator.New(sys, nil, dealerStub{}, mutator.WithDryRun(true)) - got := mut.Run() + res := mut.Run() + got := res.Mutants if len(got) != 0 { t.Errorf("should not receive results") @@ -362,7 +364,8 @@ func TestMutatorTestExecution(t *testing.T) { func(m mutant.Mutant) error { return nil })) - got := mut.Run() + res := mut.Run() + got := res.Mutants if len(got) < 1 { t.Fatal("no mutants received") @@ -370,6 +373,9 @@ func TestMutatorTestExecution(t *testing.T) { if got[0].Status() != tc.wantMutStatus { t.Errorf("expected mutation to be %v, but got: %v", tc.wantMutStatus, got[0].Status()) } + if res.Elapsed <= 0 { + t.Errorf("expected elapsed time to be greater than zero, got %s", res.Elapsed) + } }) } } diff --git a/report/report.go b/report/report.go new file mode 100644 index 00000000..2d9c1a7e --- /dev/null +++ b/report/report.go @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package report + +import ( + "github.com/fatih/color" + "github.com/hako/durafmt" + "github.com/k3rn31/gremlins/log" + "github.com/k3rn31/gremlins/mutant" + "time" +) + +var ( + fgRed = color.New(color.FgRed).SprintFunc() + fgGreen = color.New(color.FgGreen).SprintFunc() + fgHiYellow = color.New(color.FgYellow).SprintFunc() +) + +// Results contains the list of mutant.Mutant to be reported +// and the time it took to discover and test them. +type Results struct { + Mutants []mutant.Mutant + Elapsed time.Duration +} + +// Do generates the report of the Results received. +// This function uses the log package in gremlins to write to the +// chosen io.Writer, so it is necessary to call log.Init before +// the report generation. +func Do(results Results) { + if len(results.Mutants) == 0 { + log.Infoln("\nNo results to report.") + return + } + var k, l, n, r int + for _, m := range results.Mutants { + switch m.Status() { + case mutant.Killed: + k++ + case mutant.Lived: + l++ + case mutant.NotCovered: + n++ + case mutant.Runnable: + r++ + } + } + elapsed := durafmt.Parse(results.Elapsed).LimitFirstN(2) + notCovered := fgHiYellow(n) + if r > 0 { + runnable := fgGreen(r) + rCoverage := float64(r) / float64(r+n) * 100 + log.Infoln("") + log.Infof("Dry run completed in %s\n", elapsed.String()) + log.Infof("Runnable: %s, Not covered: %s\n", runnable, notCovered) + log.Infof("Mutant coverage: %.2f%%\n", rCoverage) + return + } + tEfficacy := float64(k) / float64(k+l) * 100 + rCoverage := float64(k+l) / float64(k+l+n) * 100 + killed := fgGreen(k) + lived := fgRed(l) + log.Infoln("") + log.Infof("Mutation testing completed in %s\n", elapsed.String()) + log.Infof("Killed: %s, Lived: %s, Not covered: %s\n", killed, lived, notCovered) + log.Infof("Test efficacy: %.2f%%\n", tEfficacy) + log.Infof("Mutant coverage: %.2f%%\n", rCoverage) +} + +// Mutant logs a mutant.Mutant. +// It reports the mutant.Status, the mutant.Type and its position. +// This function uses the log package in gremlins to write to the +// chosen io.Writer, so it is necessary to call log.Init before +// the report generation. +func Mutant(m mutant.Mutant) { + status := m.Status().String() + switch m.Status() { + case mutant.Killed, mutant.Runnable: + status = fgGreen(m.Status()) + case mutant.Lived: + status = fgRed(m.Status()) + case mutant.NotCovered: + status = fgHiYellow(m.Status()) + } + log.Infof("%s%s %s at %s\n", padding(m.Status()), status, m.Type(), m.Position()) +} + +func padding(s mutant.Status) string { + var pad string + padLen := 12 - len(s.String()) + for i := 0; i < padLen; i++ { + pad += " " + } + return pad +} diff --git a/report/report_test.go b/report/report_test.go new file mode 100644 index 00000000..dd1f66cb --- /dev/null +++ b/report/report_test.go @@ -0,0 +1,186 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package report_test + +import ( + "bytes" + "github.com/google/go-cmp/cmp" + "github.com/k3rn31/gremlins/log" + "github.com/k3rn31/gremlins/mutant" + "github.com/k3rn31/gremlins/report" + "go/token" + "testing" + "time" +) + +func TestReport(t *testing.T) { + t.Run("it reports findings in normal run", func(t *testing.T) { + out := &bytes.Buffer{} + log.Init(out) + defer log.Reset() + + mutants := []mutant.Mutant{ + stubMutant{mutant.Lived, mutant.ConditionalsNegation}, + stubMutant{mutant.Killed, mutant.ConditionalsNegation}, + stubMutant{mutant.NotCovered, mutant.ConditionalsNegation}, + } + data := report.Results{ + Mutants: mutants, + Elapsed: (2 * time.Minute) + (22 * time.Second) + (123 * time.Millisecond), + } + + report.Do(data) + + got := out.String() + + want := "\n" + + // Limit the time reporting to the first two units (millis are excluded) + "Mutation testing completed in 2 minutes 22 seconds\n" + + "Killed: 1, Lived: 1, Not covered: 1\n" + + "Test efficacy: 50.00%\n" + + "Mutant coverage: 66.67%\n" + + if !cmp.Equal(got, want) { + t.Errorf(cmp.Diff(want, got)) + } + }) + + t.Run("it reports findings in dry-run", func(t *testing.T) { + out := &bytes.Buffer{} + log.Init(out) + defer log.Reset() + + mutants := []mutant.Mutant{ + stubMutant{mutant.Runnable, mutant.ConditionalsNegation}, + stubMutant{mutant.Runnable, mutant.ConditionalsNegation}, + stubMutant{mutant.NotCovered, mutant.ConditionalsNegation}, + } + data := report.Results{ + Mutants: mutants, + Elapsed: (2 * time.Minute) + (22 * time.Second) + (123 * time.Millisecond), + } + + report.Do(data) + + got := out.String() + + want := "\n" + + // Limit the time reporting to the first two units (millis are excluded) + "Dry run completed in 2 minutes 22 seconds\n" + + "Runnable: 2, Not covered: 1\n" + + "Mutant coverage: 66.67%\n" + + if !cmp.Equal(got, want) { + t.Errorf(cmp.Diff(want, got)) + } + }) + t.Run("it reports nothing if no result", func(t *testing.T) { + out := &bytes.Buffer{} + log.Init(out) + defer log.Reset() + + var mutants []mutant.Mutant + data := report.Results{ + Mutants: mutants, + } + + report.Do(data) + + got := out.String() + + want := "\n" + + "No results to report.\n" + + if !cmp.Equal(got, want) { + t.Errorf(cmp.Diff(want, got)) + } + }) +} + +func TestMutantLog(t *testing.T) { + out := &bytes.Buffer{} + defer out.Reset() + log.Init(out) + defer log.Reset() + + m := stubMutant{mutant.Lived, mutant.ConditionalsBoundary} + report.Mutant(m) + m = stubMutant{mutant.Killed, mutant.ConditionalsBoundary} + report.Mutant(m) + m = stubMutant{mutant.NotCovered, mutant.ConditionalsBoundary} + report.Mutant(m) + m = stubMutant{mutant.Runnable, mutant.ConditionalsBoundary} + report.Mutant(m) + + got := out.String() + + want := "" + + " LIVED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + + " KILLED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + + " NOT COVERED CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + + " RUNNABLE CONDITIONALS_BOUNDARY at aFolder/aFile.go:12:3\n" + + if !cmp.Equal(got, want) { + t.Errorf(cmp.Diff(got, want)) + } +} + +type stubMutant struct { + status mutant.Status + mutantType mutant.Type +} + +func (s stubMutant) Type() mutant.Type { + return s.mutantType +} + +func (s stubMutant) SetType(_ mutant.Type) { + panic("implement me") +} + +func (s stubMutant) Status() mutant.Status { + return s.status +} + +func (s stubMutant) SetStatus(_ mutant.Status) { + panic("implement me") +} + +func (s stubMutant) Position() token.Position { + return token.Position{ + Filename: "aFolder/aFile.go", + Offset: 0, + Line: 12, + Column: 3, + } +} + +func (s stubMutant) Pos() token.Pos { + return 123 +} + +func (s stubMutant) SetWorkdir(_ string) { + panic("implement me") +} + +func (s stubMutant) Apply() error { + panic("implement me") +} + +func (s stubMutant) Rollback() error { + panic("implement me") +}