diff options
author | Katie Hockman <katie@golang.org> | 2021-05-07 13:23:11 -0400 |
---|---|---|
committer | Katie Hockman <katie@golang.org> | 2021-05-19 16:19:59 +0000 |
commit | 2212a1a339c7ac72ff2133855c97ae097444cb5c (patch) | |
tree | 61882c45dba124e44cc2e797661af8572481fe18 /src | |
parent | b9417ffd2797753219aa404abcb848e7c7c8bfd8 (diff) | |
download | go-git-2212a1a339c7ac72ff2133855c97ae097444cb5c.tar.gz |
[dev.fuzz] cmd/go/internal/test,testing: add documentation about fuzzing
Change-Id: Id43f7f75d6033a2c35bacd1cc0b8e3fbcaf69316
Reviewed-on: https://go-review.googlesource.com/c/go/+/317973
Trust: Katie Hockman <katie@golang.org>
Run-TryBot: Katie Hockman <katie@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>
Diffstat (limited to 'src')
-rw-r--r-- | src/cmd/go/alldocs.go | 76 | ||||
-rw-r--r-- | src/cmd/go/internal/test/test.go | 78 | ||||
-rw-r--r-- | src/cmd/go/main.go | 1 | ||||
-rw-r--r-- | src/testing/testing.go | 56 |
4 files changed, 175 insertions, 36 deletions
diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 0e1119aa0c..2a439bd904 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -53,6 +53,7 @@ // private configuration for downloading non-public code // testflag testing flags // testfunc testing functions +// fuzz fuzzing // vcs controlling version control with GOVCS // // Use "go help <topic>" for more information about that topic. @@ -1377,8 +1378,8 @@ // // 'Go test' recompiles each package along with any files with names matching // the file pattern "*_test.go". -// These additional files can contain test functions, benchmark functions, and -// example functions. See 'go help testfunc' for more. +// These additional files can contain test functions, benchmark functions, fuzz +// targets and example functions. See 'go help testfunc' for more. // Each listed package causes the execution of a separate test binary. // Files whose names begin with "_" (including "_test.go") or "." are ignored. // @@ -1447,6 +1448,8 @@ // so a successful package test result will be cached and reused // regardless of -timeout setting. // +// Run 'go help fuzz' for details around how the go command handles fuzz targets. +// // In addition to the build flags, the flags handled by 'go test' itself are: // // -args @@ -2609,7 +2612,8 @@ // (for example, -benchtime 100x). // // -count n -// Run each test and benchmark n times (default 1). +// Run each test, benchmark, and fuzz targets' seed corpora n times +// (default 1). // If -cpu is set, run n times for each GOMAXPROCS value. // Examples are always run once. // @@ -2638,32 +2642,51 @@ // Sets -cover. // // -cpu 1,2,4 -// Specify a list of GOMAXPROCS values for which the tests or -// benchmarks should be executed. The default is the current value +// Specify a list of GOMAXPROCS values for which the tests, benchmarks or +// fuzz targets should be executed. The default is the current value // of GOMAXPROCS. // // -failfast // Do not start new tests after the first test failure. // +// -fuzz name +// Run the fuzz target with the given regexp. Must match exactly one fuzz +// target. This is an experimental feature. +// +// -fuzztime t +// Run enough iterations of the fuzz test to take t, specified as a +// time.Duration (for example, -fuzztime 1h30s). The default is to run +// forever. +// The special syntax Nx means to run the fuzz test N times +// (for example, -fuzztime 100x). +// +// -keepfuzzing +// Keep running the fuzz target if a crasher is found. +// // -list regexp -// List tests, benchmarks, or examples matching the regular expression. -// No tests, benchmarks or examples will be run. This will only -// list top-level tests. No subtest or subbenchmarks will be shown. +// List tests, benchmarks, fuzz targets, or examples matching the regular +// expression. No tests, benchmarks, fuzz targets, or examples will be run. +// This will only list top-level tests. No subtest or subbenchmarks will be +// shown. // // -parallel n -// Allow parallel execution of test functions that call t.Parallel. +// Allow parallel execution of test functions that call t.Parallel, and +// f.Fuzz functions that call t.Parallel when running the seed corpus. // The value of this flag is the maximum number of tests to run -// simultaneously; by default, it is set to the value of GOMAXPROCS. +// simultaneously. While fuzzing, the value of this flag is the +// maximum number of workers to run the fuzz function simultaneously, +// regardless of whether t.Parallel has been called; by default, it is set +// to the value of GOMAXPROCS. // Note that -parallel only applies within a single test binary. // The 'go test' command may run tests for different packages // in parallel as well, according to the setting of the -p flag // (see 'go help build'). // // -run regexp -// Run only those tests and examples matching the regular expression. -// For tests, the regular expression is split by unbracketed slash (/) -// characters into a sequence of regular expressions, and each part -// of a test's identifier must match the corresponding element in +// Run only those tests, examples, and fuzz targets matching the regular +// expression. For tests, the regular expression is split by unbracketed +// slash (/) characters into a sequence of regular expressions, and each +// part of a test's identifier must match the corresponding element in // the sequence, if any. Note that possible parents of matches are // run too, so that -run=X/Y matches and runs and reports the result // of all tests matching X, even those without sub-tests matching Y, @@ -2830,6 +2853,10 @@ // // func BenchmarkXxx(b *testing.B) { ... } // +// A fuzz target is one named FuzzXxx and should have the signature, +// +// func FuzzXxx(f *testing.F) { ... } +// // An example function is similar to a test function but, instead of using // *testing.T to report success or failure, prints output to os.Stdout. // If the last comment in the function starts with "Output:" then the output @@ -2869,11 +2896,30 @@ // // The entire test file is presented as the example when it contains a single // example function, at least one other function, type, variable, or constant -// declaration, and no test or benchmark functions. +// declaration, and no fuzz targets or test or benchmark functions. // // See the documentation of the testing package for more information. // // +// Fuzzing +// +// By default, go test will build and run the fuzz targets using the target's seed +// corpus only. Any generated corpora in $GOCACHE that were previously written by +// the fuzzing engine will not be run by default. +// +// When -fuzz is set, the binary will be instrumented for coverage. After all +// tests, examples, benchmark functions, and the seed corpora for all fuzz targets +// have been run, go test will begin to fuzz the specified fuzz target. +// Note that this feature is experimental. +// +// -run can be used for testing a single seed corpus entry for a fuzz target. The +// regular expression value of -run can be in the form $target/$name, where $target +// is the name of the fuzz target, and $name is the name of the file (ignoring file +// extensions) to run. For example, -run=FuzzFoo/497b6f87. +// +// See https://golang.org/s/draft-fuzzing-design for more details. +// +// // Controlling version control with GOVCS // // The 'go get' command can run version control commands like git diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 290f6fd348..d5afae782b 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -60,8 +60,8 @@ followed by detailed output for each failed package. 'Go test' recompiles each package along with any files with names matching the file pattern "*_test.go". -These additional files can contain test functions, benchmark functions, and -example functions. See 'go help testfunc' for more. +These additional files can contain test functions, benchmark functions, fuzz +targets and example functions. See 'go help testfunc' for more. Each listed package causes the execution of a separate test binary. Files whose names begin with "_" (including "_test.go") or "." are ignored. @@ -130,6 +130,8 @@ A cached test result is treated as executing in no time at all, so a successful package test result will be cached and reused regardless of -timeout setting. +Run 'go help fuzz' for details around how the go command handles fuzz targets. + In addition to the build flags, the flags handled by 'go test' itself are: -args @@ -206,7 +208,8 @@ control the execution of any test: (for example, -benchtime 100x). -count n - Run each test and benchmark n times (default 1). + Run each test, benchmark, and fuzz targets' seed corpora n times + (default 1). If -cpu is set, run n times for each GOMAXPROCS value. Examples are always run once. @@ -235,32 +238,51 @@ control the execution of any test: Sets -cover. -cpu 1,2,4 - Specify a list of GOMAXPROCS values for which the tests or - benchmarks should be executed. The default is the current value + Specify a list of GOMAXPROCS values for which the tests, benchmarks or + fuzz targets should be executed. The default is the current value of GOMAXPROCS. -failfast Do not start new tests after the first test failure. + -fuzz name + Run the fuzz target with the given regexp. Must match exactly one fuzz + target. This is an experimental feature. + + -fuzztime t + Run enough iterations of the fuzz test to take t, specified as a + time.Duration (for example, -fuzztime 1h30s). The default is to run + forever. + The special syntax Nx means to run the fuzz test N times + (for example, -fuzztime 100x). + + -keepfuzzing + Keep running the fuzz target if a crasher is found. + -list regexp - List tests, benchmarks, or examples matching the regular expression. - No tests, benchmarks or examples will be run. This will only - list top-level tests. No subtest or subbenchmarks will be shown. + List tests, benchmarks, fuzz targets, or examples matching the regular + expression. No tests, benchmarks, fuzz targets, or examples will be run. + This will only list top-level tests. No subtest or subbenchmarks will be + shown. -parallel n - Allow parallel execution of test functions that call t.Parallel. + Allow parallel execution of test functions that call t.Parallel, and + f.Fuzz functions that call t.Parallel when running the seed corpus. The value of this flag is the maximum number of tests to run - simultaneously; by default, it is set to the value of GOMAXPROCS. + simultaneously. While fuzzing, the value of this flag is the + maximum number of workers to run the fuzz function simultaneously, + regardless of whether t.Parallel has been called; by default, it is set + to the value of GOMAXPROCS. Note that -parallel only applies within a single test binary. The 'go test' command may run tests for different packages in parallel as well, according to the setting of the -p flag (see 'go help build'). -run regexp - Run only those tests and examples matching the regular expression. - For tests, the regular expression is split by unbracketed slash (/) - characters into a sequence of regular expressions, and each part - of a test's identifier must match the corresponding element in + Run only those tests, examples, and fuzz targets matching the regular + expression. For tests, the regular expression is split by unbracketed + slash (/) characters into a sequence of regular expressions, and each + part of a test's identifier must match the corresponding element in the sequence, if any. Note that possible parents of matches are run too, so that -run=X/Y matches and runs and reports the result of all tests matching X, even those without sub-tests matching Y, @@ -430,6 +452,10 @@ A benchmark function is one named BenchmarkXxx and should have the signature, func BenchmarkXxx(b *testing.B) { ... } +A fuzz target is one named FuzzXxx and should have the signature, + + func FuzzXxx(f *testing.F) { ... } + An example function is similar to a test function but, instead of using *testing.T to report success or failure, prints output to os.Stdout. If the last comment in the function starts with "Output:" then the output @@ -469,12 +495,34 @@ Here is another example where the ordering of the output is ignored: The entire test file is presented as the example when it contains a single example function, at least one other function, type, variable, or constant -declaration, and no test or benchmark functions. +declaration, and no fuzz targets or test or benchmark functions. See the documentation of the testing package for more information. `, } +var HelpFuzz = &base.Command{ + UsageLine: "fuzz", + Short: "fuzzing", + Long: ` +By default, go test will build and run the fuzz targets using the target's seed +corpus only. Any generated corpora in $GOCACHE that were previously written by +the fuzzing engine will not be run by default. + +When -fuzz is set, the binary will be instrumented for coverage. After all +tests, examples, benchmark functions, and the seed corpora for all fuzz targets +have been run, go test will begin to fuzz the specified fuzz target. +Note that this feature is experimental. + +-run can be used for testing a single seed corpus entry for a fuzz target. The +regular expression value of -run can be in the form $target/$name, where $target +is the name of the fuzz target, and $name is the name of the file (ignoring file +extensions) to run. For example, -run=FuzzFoo/497b6f87. + +See https://golang.org/s/draft-fuzzing-design for more details. +`, +} + var ( testBench string // -bench flag testC bool // -c flag diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 02174a56ff..452673dd34 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -80,6 +80,7 @@ func init() { modfetch.HelpPrivate, test.HelpTestflag, test.HelpTestfunc, + test.HelpFuzz, modget.HelpVCS, } } diff --git a/src/testing/testing.go b/src/testing/testing.go index 63dcc62597..6b710d26d5 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -34,7 +34,7 @@ // its -bench flag is provided. Benchmarks are run sequentially. // // For a description of the testing flags, see -// https://golang.org/cmd/go/#hdr-Testing_flags +// https://golang.org/cmd/go/#hdr-Testing_flags. // // A sample benchmark function looks like this: // func BenchmarkRandInt(b *testing.B) { @@ -132,6 +132,27 @@ // example function, at least one other function, type, variable, or constant // declaration, and no test or benchmark functions. // +// Fuzzing +// +// Functions of the form +// func FuzzXxx(*testing.F) +// are considered fuzz targets, and are executed by the "go test" command. When +// the -fuzz flag is provided, the functions will be fuzzed. +// +// For a description of the testing flags, see +// https://golang.org/cmd/go/#hdr-Testing_flags. +// +// For a description of fuzzing, see golang.org/s/draft-fuzzing-design. +// +// A sample fuzz target looks like this: +// func FuzzBytesCmp(f *testing.F) { +// f.Fuzz(func(t *testing.T, a, b []byte) { +// if bytes.HasPrefix(a, b) && !bytes.Contains(a, b) { +// t.Error("HasPrefix is true, but Contains is false") +// } +// }) +// } +// // Skipping // // Tests or benchmarks may be skipped at run time with a call to @@ -144,6 +165,21 @@ // ... // } // +// The Skip method of *T can be used in a fuzz target if the input is invalid, +// but should not be considered a crash. For example: +// +// func FuzzJSONMarshalling(f *testing.F) { +// f.Fuzz(func(t *testing.T, b []byte) { +// var v interface{} +// if err := json.Unmarshal(b, &v); err != nil { +// t.Skip() +// } +// if _, err := json.Marshal(v); err != nil { +// t.Error("Marshal: %v", err) +// } +// }) +// } +// // Subtests and Sub-benchmarks // // The Run methods of T and B allow defining subtests and sub-benchmarks, @@ -163,17 +199,25 @@ // of the top-level test and the sequence of names passed to Run, separated by // slashes, with an optional trailing sequence number for disambiguation. // -// The argument to the -run and -bench command-line flags is an unanchored regular +// The argument to the -run, -bench, and -fuzz command-line flags is an unanchored regular // expression that matches the test's name. For tests with multiple slash-separated // elements, such as subtests, the argument is itself slash-separated, with // expressions matching each name element in turn. Because it is unanchored, an // empty expression matches any string. // For example, using "matching" to mean "whose name contains": // -// go test -run '' # Run all tests. -// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". -// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". -// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -run '' # Run all tests. +// go test -run Foo # Run top-level tests matching "Foo", such as "TestFooBar". +// go test -run Foo/A= # For top-level tests matching "Foo", run subtests matching "A=". +// go test -run /A=1 # For all top-level tests, run subtests matching "A=1". +// go test -fuzz FuzzFoo # Fuzz the target matching "FuzzFoo" +// +// The -run argument can also be used to run a specific value in the seed +// corpus, for debugging. For example: +// go test -run=FuzzFoo/9ddb952d9814 +// +// The -fuzz and -run flags can both be set, in order to fuzz a target but +// skip the execution of all other tests. // // Subtests can also be used to control parallelism. A parent test will only // complete once all of its subtests complete. In this example, all tests are |