[dev.fuzz] internal/fuzz: make minimization tests more reliable

* Introduced -fuzzminimizetime flag to control the number of time or
  the number of calls to spend minimizing. Defaults to 60s. Only works
  for unrecoverable crashes for now.
* Moved the count (used by -fuzztime=1000x) into shared
  memory. Calling workerClient.fuzz resets it, but it will remain
  after the worker processes crashes. workerClient.minimize resets it
  once before restarting the worker the first time, but the total
  number of runs should still be limited during minimization, even
  after multiple terminations and restarts.
* Renamed fuzzArgs.Count to Limit to avoid confusion.
* Several other small fixes and refactorings.

Change-Id: I03faa4c94405041f6dfe48568e5ead502f8dbbd2
Reviewed-on: https://go-review.googlesource.com/c/go/+/320171
Trust: Jay Conrod <jayconrod@google.com>
Trust: Katie Hockman <katie@golang.org>
Run-TryBot: Jay Conrod <jayconrod@google.com>
Reviewed-by: Katie Hockman <katie@golang.org>

4 files changed, 21 insertions, 12 deletions

diff --git a/src/cmd/go/internal/test/flagdefs.go b/src/cmd/go/internal/test/flagdefs.go
index 5a666aa1f9..3148074d57 100644
--- a/src/cmd/go/internal/test/flagdefs.go
+++ b/src/cmd/go/internal/test/flagdefs.go
@@ -20,6 +20,7 @@ var passFlagToTest = map[string]bool{
 	"cpuprofile":           true,
 	"failfast":             true,
 	"fuzz":                 true,
+	"fuzzminimizetime":     true,
 	"fuzztime":             true,
 	"list":                 true,
 	"memprofile":           true,
diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go
index 6a7b2a608b..e3eca9249b 100644
--- a/src/cmd/go/internal/test/testflag.go
+++ b/src/cmd/go/internal/test/testflag.go
@@ -69,6 +69,7 @@ func init() {
 	cf.Bool("short", false, "")
 	cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "")
 	cf.String("fuzztime", "", "")
+	cf.String("fuzzminimizetime", "", "")
 	cf.StringVar(&testTrace, "trace", "", "")
 	cf.BoolVar(&testV, "v", false, "")
 	cf.Var(&testShuffle, "shuffle", "")
diff --git a/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt
index ca2b389321..cba91a99cf 100644
--- a/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt
+++ b/src/cmd/go/testdata/script/test_fuzz_mutate_crash.txt
@@ -185,22 +185,25 @@ func FuzzWithTwoTypes(f *testing.F) {
 }
 
 func FuzzInt(f *testing.F) {
+	f.Add(0)
 	f.Fuzz(func(t *testing.T, a int) {
-		if a > 200 && a < 250 {
+		if a != 0 {
 			panic("this input caused a crash!")
 		}
 	})
 }
 
 func FuzzUint(f *testing.F) {
+	f.Add(uint(0))
 	f.Fuzz(func(t *testing.T, a uint) {
-		if a > 200 && a < 250 {
+		if a != 0 {
 			panic("this input caused a crash!")
 		}
 	})
 }
 
 func FuzzBool(f *testing.F) {
+	f.Add(false)
 	f.Fuzz(func(t *testing.T, a bool) {
 		if a {
 			panic("this input caused a crash!")
@@ -218,22 +221,25 @@ func FuzzFloat(f *testing.F) {
 }
 
 func FuzzByte(f *testing.F) {
+	f.Add(byte(0))
 	f.Fuzz(func(t *testing.T, a byte) {
-		if a > 50 {
+		if a != 0 {
 			panic("this input caused a crash!")
 		}
 	})
 }
 
 func FuzzRune(f *testing.F) {
+	f.Add(rune(0))
 	f.Fuzz(func(t *testing.T, a rune) {
-		if a > 50 {
+		if a != 0 {
 			panic("this input caused a crash!")
 		}
 	})
 }
 
 func FuzzString(f *testing.F) {
+	f.Add("")
 	f.Fuzz(func(t *testing.T, a string) {
 		if a != "" {
 			panic("this input caused a crash!")
diff --git a/src/cmd/go/testdata/script/test_fuzz_mutator.txt b/src/cmd/go/testdata/script/test_fuzz_mutator.txt
index c92be50a8e..9098d52f5b 100644
--- a/src/cmd/go/testdata/script/test_fuzz_mutator.txt
+++ b/src/cmd/go/testdata/script/test_fuzz_mutator.txt
@@ -10,6 +10,9 @@
 
 [short] skip
 
+# TODO(b/181800488): remove -parallel=1, here and below. For now, when a
+# crash is found, all workers keep running, wasting resources and reducing
+# the number of executions available to the minimizer, increasing flakiness.
 go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz
 go run check_logs.go fuzz fuzz.worker
 
@@ -20,7 +23,7 @@ stdout FAIL
 stdout 'mutator found enough unique mutations'
 
 # Test that minimization is working for recoverable errors.
-! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=1000x minimizer_test.go
+! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
 ! stdout '^ok'
 stdout 'got the minimum size!'
 stdout 'contains a letter'
@@ -33,7 +36,7 @@ go run check_testdata.go FuzzMinimizerRecoverable 50
 ! go test -run=FuzzMinimizerRecoverable minimizer_test.go
 
 # Test that minimization is working for non-recoverable errors.
-! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=1000x minimizer_test.go
+! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
 ! stdout '^ok'
 stdout 'got the minimum size!'
 stdout 'contains a letter'
@@ -42,9 +45,9 @@ stdout FAIL
 # Check that the bytes written to testdata are of length 50 (the minimum size)
 go run check_testdata.go FuzzMinimizerNonrecoverable 50
 
-# Test that minimization can be cancelled by fuzztime and the latest crash will
-# still be logged and written to testdata.
-! go test -fuzz=FuzzNonMinimizable -run=FuzzNonMinimizable -parallel=1 -fuzztime=5s minimizer_test.go
+# Test that minimization can be cancelled by fuzzminimizetime and the latest
+# crash will still be logged and written to testdata.
+! go test -fuzz=FuzzNonMinimizable -run=FuzzNonMinimizable -parallel=1 -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go
 ! stdout '^ok'
 stdout 'testdata[/\\]corpus[/\\]FuzzNonMinimizable[/\\]'
 ! stdout 'got the minimum size!'  # it shouldn't have had enough time to minimize it
@@ -108,7 +111,6 @@ package fuzz_test
 import (
 	"bytes"
 	"testing"
-	"time"
 )
 
 func FuzzMinimizerRecoverable(f *testing.F) {
@@ -155,7 +157,6 @@ func FuzzNonMinimizable(f *testing.F) {
 		if len(b) == 20 {
 			t.Log("got the minimum size!")
 		}
-		time.Sleep(4 * time.Second)
 	})
 }
 
@@ -301,7 +302,7 @@ func main() {
 		os.Exit(1)
 	}
 	if want, got := numBytes, len(s); want != got {
-		fmt.Fprintf(os.Stderr, "want %d bytes, got %d", want, got)
+		fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got)
 		os.Exit(1)
 	}
 }