Adjust number of checks performed based on the test deadline

See #35, #38.

engine.go

@@ -26,6 +26,9 @@ const (
 	invalidChecksMult = 10
 	exampleMaxTries   = 1000
 
+	maxTestTimeout  = 24 * time.Hour
+	shrinkStepBound = 10 * time.Second // can be improved by taking average checkOnce runtime into account
+
 	tracebackLen  = 32
 	tracebackStop = "pgregory.net/rapid.checkOnce"
 	runtimePrefix = "runtime."
@@ -84,13 +87,33 @@ func assertValidRange(min int, max int) {
 	}
 }
 
+func checkDeadline(t *testing.T) time.Time {
+	if t == nil {
+		return time.Now().Add(maxTestTimeout) // convenience
+	}
+	d, ok := t.Deadline()
+	if !ok {
+		return time.Now().Add(maxTestTimeout)
+	}
+	return d
+}
+
+func shrinkDeadline(deadline time.Time) time.Time {
+	d := time.Now().Add(flags.shrinkTime)
+	max := deadline.Add(-shrinkStepBound) // account for the fact that shrink deadline is checked before the step
+	if d.After(max) {
+		d = max
+	}
+	return d
+}
+
 // Check fails the current test if rapid can find a test case which falsifies prop.
 //
 // Property is falsified in case of a panic or a call to
 // [*T.Fatalf], [*T.Fatal], [*T.Errorf], [*T.Error], [*T.FailNow] or [*T.Fail].
 func Check(t *testing.T, prop func(*T)) {
 	t.Helper()
-	checkTB(t, prop)
+	checkTB(t, checkDeadline(t), prop)
 }
 
 // MakeCheck is a convenience function for defining subtests suitable for
@@ -110,7 +133,7 @@ func Check(t *testing.T, prop func(*T)) {
 func MakeCheck(prop func(*T)) func(*testing.T) {
 	return func(t *testing.T) {
 		t.Helper()
-		checkTB(t, prop)
+		checkTB(t, checkDeadline(t), prop)
 	}
 }
 
@@ -154,7 +177,7 @@ func checkFuzz(tb tb, prop func(*T), input []byte) {
 	}
 }
 
-func checkTB(tb tb, prop func(*T)) {
+func checkTB(tb tb, deadline time.Time, prop func(*T)) {
 	tb.Helper()
 
 	checks := flags.checks
@@ -163,11 +186,11 @@ func checkTB(tb tb, prop func(*T)) {
 	}
 
 	start := time.Now()
-	valid, invalid, seed, buf, err1, err2 := doCheck(tb, flags.failfile, checks, baseSeed(), prop)
+	valid, invalid, earlyExit, seed, buf, err1, err2 := doCheck(tb, flags.failfile, deadline, checks, baseSeed(), prop)
 	dt := time.Since(start)
 
 	if err1 == nil && err2 == nil {
-		if valid == checks {
+		if valid == checks || (earlyExit && valid > 0) {
 			tb.Logf("[rapid] OK, passed %v tests (%v)", valid, dt)
 		} else {
 			tb.Errorf("[rapid] only generated %v valid tests from %v total (%v)", valid, valid+invalid, dt)
@@ -206,35 +229,35 @@ func checkTB(tb tb, prop func(*T)) {
 	}
 }
 
-func doCheck(tb tb, failfile string, checks int, seed uint64, prop func(*T)) (int, int, uint64, []uint64, *testError, *testError) {
+func doCheck(tb tb, failfile string, deadline time.Time, checks int, seed uint64, prop func(*T)) (int, int, bool, uint64, []uint64, *testError, *testError) {
 	tb.Helper()
 
 	assertf(!tb.Failed(), "check function called with *testing.T which has already failed")
 
 	if failfile != "" {
 		buf, err1, err2 := checkFailFile(tb, failfile, prop)
 		if err1 != nil || err2 != nil {
-			return 0, 0, 0, buf, err1, err2
+			return 0, 0, false, 0, buf, err1, err2
 		}
 	}
 
-	valid, invalid, seed, err1 := findBug(tb, checks, seed, prop)
+	valid, invalid, earlyExit, seed, err1 := findBug(tb, deadline, checks, seed, prop)
 	if err1 == nil {
-		return valid, invalid, 0, nil, nil, nil
+		return valid, invalid, earlyExit, 0, nil, nil, nil
 	}
 
 	s := newRandomBitStream(seed, true)
 	t := newT(tb, s, flags.verbose, nil)
 	t.Logf("[rapid] trying to reproduce the failure")
 	err2 := checkOnce(t, prop)
 	if !sameError(err1, err2) {
-		return valid, invalid, seed, s.data, err1, err2
+		return valid, invalid, false, seed, s.data, err1, err2
 	}
 
 	t.Logf("[rapid] trying to minimize the failing test case")
-	buf, err3 := shrink(tb, s.recordedBits, err2, prop)
+	buf, err3 := shrink(tb, shrinkDeadline(deadline), s.recordedBits, err2, prop)
 
-	return valid, invalid, seed, buf, err2, err3
+	return valid, invalid, false, seed, buf, err2, err3
 }
 
 func checkFailFile(tb tb, failfile string, prop func(*T)) ([]uint64, *testError, *testError) {
@@ -269,7 +292,7 @@ func checkFailFile(tb tb, failfile string, prop func(*T)) ([]uint64, *testError,
 	return buf, err1, err2
 }
 
-func findBug(tb tb, checks int, seed uint64, prop func(*T)) (int, int, uint64, *testError) {
+func findBug(tb tb, deadline time.Time, checks int, seed uint64, prop func(*T)) (int, int, bool, uint64, *testError) {
 	tb.Helper()
 
 	var (
@@ -279,35 +302,45 @@ func findBug(tb tb, checks int, seed uint64, prop func(*T)) (int, int, uint64, *
 		invalid = 0
 	)
 
+	var total time.Duration
 	for valid < checks && invalid < checks*invalidChecksMult {
-		seed += uint64(valid) + uint64(invalid)
+		iter := valid + invalid
+		if iter > 0 && time.Until(deadline) < total/time.Duration(iter)*5 {
+			if t.shouldLog() {
+				t.Logf("[rapid] early exit after test #%v (%v)", iter, total)
+			}
+			return valid, invalid, true, 0, nil
+		}
+
+		seed += uint64(iter)
 		r.init(seed)
-		var start time.Time
+		start := time.Now()
 		if t.shouldLog() {
-			t.Logf("[rapid] test #%v start (seed %v)", valid+invalid+1, seed)
-			start = time.Now()
+			t.Logf("[rapid] test #%v start (seed %v)", iter+1, seed)
 		}
 
 		err := checkOnce(t, prop)
+		dt := time.Since(start)
+		total += dt
 		if err == nil {
 			if t.shouldLog() {
-				t.Logf("[rapid] test #%v OK (%v)", valid+invalid+1, time.Since(start))
+				t.Logf("[rapid] test #%v OK (%v)", iter+1, dt)
 			}
 			valid++
 		} else if err.isInvalidData() {
 			if t.shouldLog() {
-				t.Logf("[rapid] test #%v invalid (%v)", valid+invalid+1, time.Since(start))
+				t.Logf("[rapid] test #%v invalid (%v)", iter+1, dt)
 			}
 			invalid++
 		} else {
 			if t.shouldLog() {
-				t.Logf("[rapid] test #%v failed: %v", valid+invalid+1, err)
+				t.Logf("[rapid] test #%v failed: %v", iter+1, err)
 			}
-			return valid, invalid, seed, err
+			return valid, invalid, false, seed, err
 		}
 	}
 
-	return valid, invalid, 0, nil
+	return valid, invalid, false, 0, nil
 }
 
 func checkOnce(t *T, prop func(*T)) (err *testError) {

engine_test.go

@@ -79,9 +79,10 @@ func BenchmarkCheckOverhead(b *testing.B) {
 	f := func(t *T) {
 		g.Draw(t, "")
 	}
+	deadline := checkDeadline(nil)
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		checkTB(b, f)
+		checkTB(b, deadline, f)
 	}
 }

shrink.go

@@ -31,7 +31,7 @@ const (
 	labelSortGroups          = "sort_groups"
 )
 
-func shrink(tb tb, rec recordedBits, err *testError, prop func(*T)) ([]uint64, *testError) {
+func shrink(tb tb, deadline time.Time, rec recordedBits, err *testError, prop func(*T)) ([]uint64, *testError) {
 	rec.prune()
 
 	s := &shrinker{
@@ -44,7 +44,7 @@ func shrink(tb tb, rec recordedBits, err *testError, prop func(*T)) ([]uint64, *
 		cache:   map[string]struct{}{},
 	}
 
-	buf, err := s.shrink()
+	buf, err := s.shrink(deadline)
 
 	if flags.debugvis {
 		name := fmt.Sprintf("vis-%v.html", strings.Replace(tb.Name(), "/", "_", -1))
@@ -82,15 +82,14 @@ func (s *shrinker) debugf(verbose_ bool, format string, args ...any) {
 	}
 }
 
-func (s *shrinker) shrink() (buf []uint64, err *testError) {
+func (s *shrinker) shrink(deadline time.Time) (buf []uint64, err *testError) {
 	defer func() {
 		if r := recover(); r != nil {
 			buf, err = s.rec.data, r.(*testError)
 		}
 	}()
 
 	i := 0
-	deadline := time.Now().Add(flags.shrinkTime)
 	for shrinks := -1; s.shrinks > shrinks && time.Now().Before(deadline); i++ {
 		shrinks = s.shrinks
 

shrink_test.go

@@ -210,7 +210,7 @@ func checkShrink(t *testing.T, prop func(*T), draws ...any) {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			t.Helper()
 
-			_, _, seed, buf, err1, err2 := doCheck(t, "", 100, baseSeed(), prop)
+			_, _, _, seed, buf, err1, err2 := doCheck(t, "", checkDeadline(nil), 100, baseSeed(), prop)
 			if seed != 0 && err1 == nil && err2 == nil {
 				t.Fatalf("shrink test did not fail (seed %v)", seed)
 			}

statemachine_test.go

@@ -235,6 +235,6 @@ func TestStateMachine_DiscardGarbage(t *testing.T) {
 
 func BenchmarkCheckQueue(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		_, _, _, _, _, _ = doCheck(b, "", 100, baseSeed(), queueTest)
+		_, _, _, _, _, _, _ = doCheck(b, "", checkDeadline(nil), 100, baseSeed(), queueTest)
 	}
 }