From 4391f1175dcc545e6d7d5ed9e8044f28fb84a413 Mon Sep 17 00:00:00 2001 From: Gregory Petrosyan Date: Tue, 30 May 2023 00:19:18 +0300 Subject: [PATCH] Adjust number of checks performed based on the test deadline See #35, #38. --- engine.go | 77 +++++++++++++++++++++++++++++++------------- engine_test.go | 3 +- shrink.go | 7 ++-- shrink_test.go | 2 +- statemachine_test.go | 2 +- 5 files changed, 62 insertions(+), 29 deletions(-) diff --git a/engine.go b/engine.go index 0ee46bc..ef58c40 100644 --- a/engine.go +++ b/engine.go @@ -26,6 +26,9 @@ const ( invalidChecksMult = 10 exampleMaxTries = 1000 + maxTestTimeout = 24 * time.Hour + shrinkStepBound = 10 * time.Second // can be improved by taking average checkOnce runtime into account + tracebackLen = 32 tracebackStop = "pgregory.net/rapid.checkOnce" runtimePrefix = "runtime." @@ -84,13 +87,33 @@ func assertValidRange(min int, max int) { } } +func checkDeadline(t *testing.T) time.Time { + if t == nil { + return time.Now().Add(maxTestTimeout) // convenience + } + d, ok := t.Deadline() + if !ok { + return time.Now().Add(maxTestTimeout) + } + return d +} + +func shrinkDeadline(deadline time.Time) time.Time { + d := time.Now().Add(flags.shrinkTime) + max := deadline.Add(-shrinkStepBound) // account for the fact that shrink deadline is checked before the step + if d.After(max) { + d = max + } + return d +} + // Check fails the current test if rapid can find a test case which falsifies prop. // // Property is falsified in case of a panic or a call to // [*T.Fatalf], [*T.Fatal], [*T.Errorf], [*T.Error], [*T.FailNow] or [*T.Fail]. func Check(t *testing.T, prop func(*T)) { t.Helper() - checkTB(t, prop) + checkTB(t, checkDeadline(t), prop) } // MakeCheck is a convenience function for defining subtests suitable for @@ -110,7 +133,7 @@ func Check(t *testing.T, prop func(*T)) { func MakeCheck(prop func(*T)) func(*testing.T) { return func(t *testing.T) { t.Helper() - checkTB(t, prop) + checkTB(t, checkDeadline(t), prop) } } @@ -154,7 +177,7 @@ func checkFuzz(tb tb, prop func(*T), input []byte) { } } -func checkTB(tb tb, prop func(*T)) { +func checkTB(tb tb, deadline time.Time, prop func(*T)) { tb.Helper() checks := flags.checks @@ -163,11 +186,11 @@ func checkTB(tb tb, prop func(*T)) { } start := time.Now() - valid, invalid, seed, buf, err1, err2 := doCheck(tb, flags.failfile, checks, baseSeed(), prop) + valid, invalid, earlyExit, seed, buf, err1, err2 := doCheck(tb, flags.failfile, deadline, checks, baseSeed(), prop) dt := time.Since(start) if err1 == nil && err2 == nil { - if valid == checks { + if valid == checks || (earlyExit && valid > 0) { tb.Logf("[rapid] OK, passed %v tests (%v)", valid, dt) } else { tb.Errorf("[rapid] only generated %v valid tests from %v total (%v)", valid, valid+invalid, dt) @@ -206,7 +229,7 @@ func checkTB(tb tb, prop func(*T)) { } } -func doCheck(tb tb, failfile string, checks int, seed uint64, prop func(*T)) (int, int, uint64, []uint64, *testError, *testError) { +func doCheck(tb tb, failfile string, deadline time.Time, checks int, seed uint64, prop func(*T)) (int, int, bool, uint64, []uint64, *testError, *testError) { tb.Helper() assertf(!tb.Failed(), "check function called with *testing.T which has already failed") @@ -214,13 +237,13 @@ func doCheck(tb tb, failfile string, checks int, seed uint64, prop func(*T)) (in if failfile != "" { buf, err1, err2 := checkFailFile(tb, failfile, prop) if err1 != nil || err2 != nil { - return 0, 0, 0, buf, err1, err2 + return 0, 0, false, 0, buf, err1, err2 } } - valid, invalid, seed, err1 := findBug(tb, checks, seed, prop) + valid, invalid, earlyExit, seed, err1 := findBug(tb, deadline, checks, seed, prop) if err1 == nil { - return valid, invalid, 0, nil, nil, nil + return valid, invalid, earlyExit, 0, nil, nil, nil } s := newRandomBitStream(seed, true) @@ -228,13 +251,13 @@ func doCheck(tb tb, failfile string, checks int, seed uint64, prop func(*T)) (in t.Logf("[rapid] trying to reproduce the failure") err2 := checkOnce(t, prop) if !sameError(err1, err2) { - return valid, invalid, seed, s.data, err1, err2 + return valid, invalid, false, seed, s.data, err1, err2 } t.Logf("[rapid] trying to minimize the failing test case") - buf, err3 := shrink(tb, s.recordedBits, err2, prop) + buf, err3 := shrink(tb, shrinkDeadline(deadline), s.recordedBits, err2, prop) - return valid, invalid, seed, buf, err2, err3 + return valid, invalid, false, seed, buf, err2, err3 } func checkFailFile(tb tb, failfile string, prop func(*T)) ([]uint64, *testError, *testError) { @@ -269,7 +292,7 @@ func checkFailFile(tb tb, failfile string, prop func(*T)) ([]uint64, *testError, return buf, err1, err2 } -func findBug(tb tb, checks int, seed uint64, prop func(*T)) (int, int, uint64, *testError) { +func findBug(tb tb, deadline time.Time, checks int, seed uint64, prop func(*T)) (int, int, bool, uint64, *testError) { tb.Helper() var ( @@ -279,35 +302,45 @@ func findBug(tb tb, checks int, seed uint64, prop func(*T)) (int, int, uint64, * invalid = 0 ) + var total time.Duration for valid < checks && invalid < checks*invalidChecksMult { - seed += uint64(valid) + uint64(invalid) + iter := valid + invalid + if iter > 0 && time.Until(deadline) < total/time.Duration(iter)*5 { + if t.shouldLog() { + t.Logf("[rapid] early exit after test #%v (%v)", iter, total) + } + return valid, invalid, true, 0, nil + } + + seed += uint64(iter) r.init(seed) - var start time.Time + start := time.Now() if t.shouldLog() { - t.Logf("[rapid] test #%v start (seed %v)", valid+invalid+1, seed) - start = time.Now() + t.Logf("[rapid] test #%v start (seed %v)", iter+1, seed) } err := checkOnce(t, prop) + dt := time.Since(start) + total += dt if err == nil { if t.shouldLog() { - t.Logf("[rapid] test #%v OK (%v)", valid+invalid+1, time.Since(start)) + t.Logf("[rapid] test #%v OK (%v)", iter+1, dt) } valid++ } else if err.isInvalidData() { if t.shouldLog() { - t.Logf("[rapid] test #%v invalid (%v)", valid+invalid+1, time.Since(start)) + t.Logf("[rapid] test #%v invalid (%v)", iter+1, dt) } invalid++ } else { if t.shouldLog() { - t.Logf("[rapid] test #%v failed: %v", valid+invalid+1, err) + t.Logf("[rapid] test #%v failed: %v", iter+1, err) } - return valid, invalid, seed, err + return valid, invalid, false, seed, err } } - return valid, invalid, 0, nil + return valid, invalid, false, 0, nil } func checkOnce(t *T, prop func(*T)) (err *testError) { diff --git a/engine_test.go b/engine_test.go index 369985b..f0b1cad 100644 --- a/engine_test.go +++ b/engine_test.go @@ -79,9 +79,10 @@ func BenchmarkCheckOverhead(b *testing.B) { f := func(t *T) { g.Draw(t, "") } + deadline := checkDeadline(nil) b.ResetTimer() for i := 0; i < b.N; i++ { - checkTB(b, f) + checkTB(b, deadline, f) } } diff --git a/shrink.go b/shrink.go index 936b499..52a1589 100644 --- a/shrink.go +++ b/shrink.go @@ -31,7 +31,7 @@ const ( labelSortGroups = "sort_groups" ) -func shrink(tb tb, rec recordedBits, err *testError, prop func(*T)) ([]uint64, *testError) { +func shrink(tb tb, deadline time.Time, rec recordedBits, err *testError, prop func(*T)) ([]uint64, *testError) { rec.prune() s := &shrinker{ @@ -44,7 +44,7 @@ func shrink(tb tb, rec recordedBits, err *testError, prop func(*T)) ([]uint64, * cache: map[string]struct{}{}, } - buf, err := s.shrink() + buf, err := s.shrink(deadline) if flags.debugvis { name := fmt.Sprintf("vis-%v.html", strings.Replace(tb.Name(), "/", "_", -1)) @@ -82,7 +82,7 @@ func (s *shrinker) debugf(verbose_ bool, format string, args ...any) { } } -func (s *shrinker) shrink() (buf []uint64, err *testError) { +func (s *shrinker) shrink(deadline time.Time) (buf []uint64, err *testError) { defer func() { if r := recover(); r != nil { buf, err = s.rec.data, r.(*testError) @@ -90,7 +90,6 @@ func (s *shrinker) shrink() (buf []uint64, err *testError) { }() i := 0 - deadline := time.Now().Add(flags.shrinkTime) for shrinks := -1; s.shrinks > shrinks && time.Now().Before(deadline); i++ { shrinks = s.shrinks diff --git a/shrink_test.go b/shrink_test.go index c36198a..f0dae56 100644 --- a/shrink_test.go +++ b/shrink_test.go @@ -210,7 +210,7 @@ func checkShrink(t *testing.T, prop func(*T), draws ...any) { t.Run(strconv.Itoa(i), func(t *testing.T) { t.Helper() - _, _, seed, buf, err1, err2 := doCheck(t, "", 100, baseSeed(), prop) + _, _, _, seed, buf, err1, err2 := doCheck(t, "", checkDeadline(nil), 100, baseSeed(), prop) if seed != 0 && err1 == nil && err2 == nil { t.Fatalf("shrink test did not fail (seed %v)", seed) } diff --git a/statemachine_test.go b/statemachine_test.go index 683ac00..c0f59cc 100644 --- a/statemachine_test.go +++ b/statemachine_test.go @@ -235,6 +235,6 @@ func TestStateMachine_DiscardGarbage(t *testing.T) { func BenchmarkCheckQueue(b *testing.B) { for i := 0; i < b.N; i++ { - _, _, _, _, _, _ = doCheck(b, "", 100, baseSeed(), queueTest) + _, _, _, _, _, _, _ = doCheck(b, "", checkDeadline(nil), 100, baseSeed(), queueTest) } }