diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-05-15 10:20:33 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-05-15 10:28:57 +0000 |
commit | d17ea114e5ef69ad5d5d7413280a13e6428098aa (patch) | |
tree | 2c01a75df69f30d27b1432467cfe7c1467a498da /chromium/testing | |
parent | 8c5c43c7b138c9b4b0bf56d946e61d3bbc111bec (diff) | |
download | qtwebengine-chromium-d17ea114e5ef69ad5d5d7413280a13e6428098aa.tar.gz |
BASELINE: Update Chromium to 67.0.3396.47
Change-Id: Idcb1341782e417561a2473eeecc82642dafda5b7
Reviewed-by: Michal Klocek <michal.klocek@qt.io>
Diffstat (limited to 'chromium/testing')
26 files changed, 977 insertions, 1259 deletions
diff --git a/chromium/testing/BUILD.gn b/chromium/testing/BUILD.gn index 1477b17320e..70b420157c0 100644 --- a/chromium/testing/BUILD.gn +++ b/chromium/testing/BUILD.gn @@ -30,10 +30,13 @@ group("test_scripts_shared") { ] } -group("run_gtest_perf_test") { +group("run_perf_test") { data = [ "//testing/scripts/common.py", "//testing/scripts/run_gtest_perf_test.py", + "//testing/scripts/run_performance_tests.py", + "//testing/scripts/run_performance_tests_wrapper.py", + "//testing/scripts/run_telemetry_benchmark_as_googletest.py", "//tools/perf/generate_legacy_perf_dashboard_json.py", ] diff --git a/chromium/testing/buildbot/filters/BUILD.gn b/chromium/testing/buildbot/filters/BUILD.gn index 62c90dc449f..3f7b2b76584 100644 --- a/chromium/testing/buildbot/filters/BUILD.gn +++ b/chromium/testing/buildbot/filters/BUILD.gn @@ -20,7 +20,7 @@ source_set("ash_unittests_filters") { testonly = true data = [ - "//testing/buildbot/filters/ash_unittests_mash.filter", + "//testing/buildbot/filters/mash.ash_unittests.filter", ] } @@ -31,7 +31,6 @@ source_set("browser_tests_filters") { "//testing/buildbot/filters/mash.browser_tests.filter", "//testing/buildbot/filters/mojo.fyi.network_browser_tests.filter", "//testing/buildbot/filters/browser_tests_cros_asan.filter", - "//testing/buildbot/filters/site-per-process.browser_tests.filter", "//testing/buildbot/filters/viz.browser_tests.filter", ] } @@ -43,7 +42,6 @@ source_set("content_browsertests_filters") { "//testing/buildbot/filters/cast-linux.content_browsertests.filter", "//testing/buildbot/filters/mojo.fyi.network_content_browsertests.filter", "//testing/buildbot/filters/site-per-process.content_browsertests.filter", - "//testing/buildbot/filters/mojo.fyi.viz.content_browsertests.filter", "//testing/buildbot/filters/viz.content_browsertests.filter", ] } @@ -56,22 +54,6 @@ source_set("content_unittests_filters") { ] } -source_set("interactive_ui_tests_filters") { - testonly = true - - data = [ - "//testing/buildbot/filters/site-per-process.interactive_ui_tests.filter", - ] -} - -source_set("services_unittests_filters") { - testonly = true - - data = [ - "//testing/buildbot/filters/win10.services_unittests.filter", - ] -} - source_set("unit_tests_filters") { testonly = true diff --git a/chromium/testing/libfuzzer/README.md b/chromium/testing/libfuzzer/README.md index 99a7aa94d93..df023d7cbce 100644 --- a/chromium/testing/libfuzzer/README.md +++ b/chromium/testing/libfuzzer/README.md @@ -10,7 +10,7 @@ *** This directory contains integration between [libFuzzer] and Chromium. -libFuzzer is an in-process coverage-driven evolutionary fuzzer. It helps +LibFuzzer is an in-process coverage-driven evolutionary fuzzing engine. It helps engineers to uncover potential security & stability problems earlier. *** note @@ -25,11 +25,11 @@ ClusterFuzz fuzzing system. Cover bug: [crbug.com/539572]. ## Documentation * [Getting Started Guide] walks you through all the steps necessary to create -your fuzzer and submit it to ClusterFuzz. -* [Efficient Fuzzer Guide] explains how to measure fuzzer effectiveness and +your fuzz target and submit it to ClusterFuzz. +* [Efficient Fuzzer Guide] explains how to measure fuzz target effectiveness and ways to improve it. * [Guide to libprotobuf-mutator] walks through the steps necessary to create a -fuzzer that libFuzzer gives mutated protobufs to as input (for developers +fuzz target that libFuzzer gives mutated protobufs to as input (for developers already familiar with libFuzzer). * [ClusterFuzz Integration] describes integration between ClusterFuzz and libFuzzer. @@ -39,7 +39,7 @@ libFuzzer. ## Trophies * [ClusterFuzz Bugs] - issues found and automatically filed by ClusterFuzz. -* [Manual Bugs] - issues that were filed manually after running fuzzers. +* [Manual Bugs] - issues that were filed manually after running fuzz targets. * [Pdfium Bugs] - bugs found in pdfium by manual fuzzing. * [OSS Trophies] - bugs found with libFuzzer in open-source projects. diff --git a/chromium/testing/libfuzzer/clusterfuzz.md b/chromium/testing/libfuzzer/clusterfuzz.md index 56b720536f2..c46c4df6334 100644 --- a/chromium/testing/libfuzzer/clusterfuzz.md +++ b/chromium/testing/libfuzzer/clusterfuzz.md @@ -1,34 +1,34 @@ # libFuzzer and ClusterFuzz Integration ClusterFuzz is a distributed fuzzing infrastructure that automatically -executes libFuzzer tests on scale. +executes libFuzzer powered fuzzer tests on scale. Googlers can read more [here](https://goto.google.com/clusterfuzz). ## Status Links * [Buildbot] - status of all libFuzzer builds. -* [ClusterFuzz Fuzzer Status] - fuzzing metrics, links to crashes and coverage +* [ClusterFuzz Fuzzer Status] - fuzzing metrics, links to crashes and coverage reports. -* [ClusterFuzz libFuzzer Logs] - individual fuzzer run logs. -* [Corpus GCS Bucket] - current corpus for each fuzzer. Can be used to upload -bootstrapped corpus. +* [ClusterFuzz libFuzzer Logs] - individual fuzz target run logs. +* [Corpus GCS Bucket] - current corpus for each fuzz target. Can be used to +upload bootstrapped corpus. ## Integration Details The integration between libFuzzer and ClusterFuzz consists of: * Build rules definition in [fuzzer_test.gni]. -* [Buildbot] that automatically discovers fuzzers using `gn refs` facility, -builds fuzzers with multiple sanitizers and uploads binaries to a special -GCS bucket. Build bot recipe is defined in [chromium_libfuzzer.py]. -* ClusterFuzz downloads new binaries once a day and runs fuzzers continuously. -* Fuzzer run logs are uploaded to [ClusterFuzz libFuzzer Logs] GCS bucket. -* Fuzzing corpus is maintained for each fuzzer in [Corpus GCS Bucket]. Once a day, -corpus is minimized to reduce number of duplicates and/or reduce effect of -parasitic coverage. -* [ClusterFuzz Fuzzer Status] displays fuzzer runtime -metrics as well as provides links to crashes and coverage reports. +* [Buildbot] that automatically discovers fuzz targets using `gn refs`, builds +fuzz targets with multiple sanitizers and uploads binaries to a GCS bucket. +Recipe is defined in [chromium_libfuzzer.py]. +* ClusterFuzz downloads builds and runs fuzz targets continuously. +* Fuzz target run logs are uploaded to [ClusterFuzz libFuzzer Logs] GCS bucket. +* Fuzzing corpus is maintained for each fuzz target in [Corpus GCS Bucket]. Once +a day, the corpus is minimized to reduce number of duplicates and/or reduce +effect of parasitic coverage. +* [ClusterFuzz Fuzzer Status] displays fuzzer runtime metrics as well as +provides links to crashes and coverage reports. ## Corpus @@ -41,7 +41,7 @@ ClusterFuzz uses these files for fuzzing but doesn't delete/overwrite them. These corpus files are frequently modified during fuzzing sessions and can be deleted during corpus minimization. -A fuzzer has two input corpus directories, seed and general, but its output +A fuzz target has two input corpus directories, seed and general, but its output goes into general corpus directory. Seed corpus is read-only. diff --git a/chromium/testing/libfuzzer/efficient_fuzzer.md b/chromium/testing/libfuzzer/efficient_fuzzer.md index bbde9ab8703..04b5dc733b0 100644 --- a/chromium/testing/libfuzzer/efficient_fuzzer.md +++ b/chromium/testing/libfuzzer/efficient_fuzzer.md @@ -1,39 +1,124 @@ -# Efficient Fuzzer +# Efficient Fuzzer Guide -This document describes ways to determine your fuzzer efficiency and ways +This document describes ways to determine efficiency of a fuzz target and ways to improve it. ## Overview -Being a coverage-driven fuzzer, libFuzzer considers a certain input *interesting* -if it results in new code coverage. The set of all interesting inputs is called +Being a coverage-driven fuzzing engine, libFuzzer considers a certain input +*interesting* if it results in new code coverage, i.e. it reaches a code that +has not been reached before. The set of all interesting inputs is called *corpus*. Items in corpus are constantly mutated in search of new interesting inputs. Corpus can be shared across fuzzer runs and grows over time as new code is reached. -The following things are extremely effective for improving fuzzer efficiency, so we -*strongly recommend* them for any fuzzer: +There are several metrics you should look at to determine effectiveness of your +fuzz target: + +* [Execution Speed](#Execution-Speed) +* [Code Coverage](#Code-Coverage) +* [Corpus Size](#Corpus-Size) + +You can collect these metrics manually or take them from [ClusterFuzz status] +pages after a fuzz target is checked in Chromium repository. + +The following things are extremely useful for improving fuzzing efficiency, so +we *strongly recommend* them for any fuzz target: * [Seed Corpus](#Seed-Corpus) * [Fuzzer Dictionary](#Fuzzer-Dictionary) -There are several metrics you should look at to determine your fuzzer effectiveness: +There are other ways that are useful in some cases, but not always applicable: +* [Custom Options](#Custom-Options) +* [Custom Build](#Custom-Build) -* [Fuzzer Speed](#Fuzzer-Speed) -* [Corpus Size](#Corpus-Size) -* [Code Coverage](#Code-Coverage) -You can collect these metrics manually or take them from [ClusterFuzz status] -pages after a fuzzer is checked in Chromium repository. +## Execution Speed + +Fuzz target speed is calculated in executions per second. It is printed while a +fuzz target is running: + +``` +#19346 NEW cov: 2815 bits: 1082 indir: 43 units: 150 exec/s: 19346 L: 62 +``` + +Because libFuzzer performs randomized mutations, it is critical to have it run +as fast as possible to navigate through the large search space efficiently and +find interesting code paths. You should try to get to at least 1,000 exec/s from +your fuzz target locally before submitting it to the Chromium repository. + + +### Initialization/Cleanup + +Try to keep `LLVMFuzzerTestOneInput` function as simple as possible. If your +fuzzing function is too complex, it can bring down fuzzer execution speed OR it +can target very specific usecases and fail to account for unexpected scenarios. + +Prefer to use static initialization and shared resources rather than performing +setup and teardown on every single input. Checkout example on +[startup initialization] in libFuzzer documentation. + +You can skip freeing static resources. However, all resources allocated within +`LLVMFuzzerTestOneInput` function should be de-allocated since this function is +called millions of times during a fuzzing session. Otherwise, we will hit OOMs +frequently and reduce overall fuzzing efficiency. + + +### Memory Usage + +Avoid allocation of dynamic memory wherever possible. Memory instrumentation +works faster for stack-based and static objects, than for heap allocated ones. + +It is always a good idea to try different variants for your fuzz target locally, +and then submit the fastest implementation. + + +## Code Coverage + +[ClusterFuzz status] page provides source-level coverage report for fuzz targets +from recent runs. Looking at the report might provide an insight on how to +improve code coverage of a fuzz target. + +You can also generate source-level coverage report locally by running the +[coverage script] stored in Chromium repository. The script provides detailed +instructions as well as an usage example. + +Note that code coverage of a fuzz target **depends heavily** on the corpus +provided when running the target, i.e. code coverage report generated by a fuzz +target launched without any corpus would not make much sense. + +We encourage you to try out the [coverage script], as it usually generates a better +code coverage visualization compared to the coverage report hosted on +ClusterFuzz. +*NOTE: This is an experimental feature and an active area of work. We are +working on improving this process.* + + +## Corpus Size + +After running for a while, a fuzz target would reach a plateau and may stop +discovering new interesting inputs. Corpus for a reasonably complex target +should contain hundreds (if not thousands) of items. + +Too small of a corpus size indicates that fuzz target is hitting a code barrier +and is unable to get past it. Common cases of such issues include: checksums, +magic numbers, etc. The easiest way to diagnose this problem is to generate and +analyze a [coverage report](#Coverage). To fix the issue, you can: + +* Change the code (e.g. disable crc checks while fuzzing), see [Custom Build](#Custom-Build). +* Prepare or improve [seed corpus](#Seed-Corpus). +* Prepare or improve [fuzzer dictionary](#Fuzzer-Dictionary). +* Add [custom options](#Custom-Options). + ## Seed Corpus -Seed corpus is a set of *valid* and *interesting* inputs that serve as starting points -for a fuzzer. If one is not provided, a fuzzer would have to guess these inputs -from scratch, which can take an indefinite amount of time depending of the size -of inputs. +Seed corpus is a set of *valid* and *interesting* inputs that serve as starting +points for a fuzz target. If one is not provided, a fuzzing engine would have to +guess these inputs from scratch, which can take an indefinite amount of time +depending on size of the inputs and complexity of the target format. Seed corpus works especially well for strictly defined file formats and data transmission protocols. @@ -41,20 +126,23 @@ transmission protocols. * For file format parsers, add valid files from your test suite. * For protocol parsers, add valid raw streams from test suite into separate files. -Other examples include a graphics library seed corpus, which would be a variety of -small PNG/JPG/GIF files. +Other examples include a graphics library seed corpus, which would be a variety +of small PNG/JPG/GIF files. -If you are running the fuzzer locally, you can pass a corpus directory as an argument: +If you are running a fuzz target locally, you can pass a corpus directory as an +argument: ``` ./out/libfuzzer/my_fuzzer ~/tmp/my_fuzzer_corpus ``` -While libFuzzer can start with an empty corpus, most fuzzers require a seed corpus -to be useful. The fuzzer would store all the interesting items it finds in that directory. +The fuzzer would store all the interesting inputs it finds in that directory. + +While libFuzzer can start with an empty corpus, seed corpus is always useful and +in many cases is able to increase code coverage by an order of magnitude. ClusterFuzz uses seed corpus defined in Chromium source repository. You need to -add a `seed_corpus` attribute to your fuzzer definition in BUILD.gn file: +add a `seed_corpus` attribute to your `fuzzer_test` definition in BUILD.gn file: ``` fuzzer_test("my_protocol_fuzzer") { @@ -77,9 +165,9 @@ fuzzer_test("my_protocol_fuzzer") { All files found in these directories and their subdirectories will be archived into a `<my_fuzzer_name>_seed_corpus.zip` output archive. -If you can't store seed corpus in Chromium repository (e.g. it is too large, has -licensing issues, etc), you can upload the corpus to Google Cloud Storage bucket -used by ClusterFuzz: +If you can't store seed corpus in Chromium repository (e.g. it is too large, +cannot be open sourced, etc), you can upload the corpus to Google Cloud Storage +bucket used by ClusterFuzz: 1) Go to [Corpus GCS Bucket]. 2) Open directory named `<my_fuzzer_name>_static`. If the directory does not @@ -91,22 +179,46 @@ Alternative and faster way is to use [gsutil] command line tool: gsutil -m rsync <path_to_corpus> gs://clusterfuzz-corpus/libfuzzer/<my_fuzzer_name>_static ``` +### Corpus Minimization + +It's important to minimize seed corpus to a *small set of interesting inputs* +before uploading. The reason being that seed corpus is synced to all fuzzing +bots for every iteration, so it is important to keep it small both for fuzzing +efficiency and to prevent our bots from running out of disk space. + +The minimization can be done using `-merge=1` option of libFuzzer: + +```bash +# Create an empty directory. +mkdir seed_corpus_minimized + +# Run the fuzzer with -merge=1 flag. +./my_fuzzer -merge=1 ./seed_corpus_minimized ./seed_corpus +``` + +After running the command above, `seed_corpus_minimized` directory will contain +a minimized corpus that gives the same code coverage as the initial +`seed_corpus` directory. + + + + ## Fuzzer Dictionary -It is very useful to provide fuzzer a set of *common words or values* that you -expect to find in the input. Adding a dictionary highly improves the efficiency of -finding new units and works especially well in certain usecases (e.g. fuzzing file -format decoders). +It is very useful to provide fuzz target with a set of *common words or values* +that you expect to find in the input. Adding a dictionary highly improves the +efficiency of finding new units and works especially well in certain usecases +(e.g. fuzzing file format decoders or text based protocols like XML). -To add a dictionary, first create a dictionary file. Dictionary file is a flat text file -where tokens are listed one per line in the format of name="value". The -alphanumeric name is ignored and can be omitted, although it is a convenient -way to document the meaning of a particular token. The value must appear in -quotes, with hex escaping (\xNN) applied to all non-printable, high-bit, or -otherwise problematic characters (\\ and \" shorthands are recognized too). -This syntax is similar to the one used by [AFL] fuzzing engine (-x option). +To add a dictionary, first create a dictionary file. This is a flat text file +where tokens are listed one per line in the format of `name="value"`, where +`name` is optional and can be omitted, although it is a convenient way to +document the meaning of a particular token. The value must appear in quotes, +with hex escaping (\xNN) applied to all non-printable, high-bit, or otherwise +problematic characters (\\ and \" shorthands are recognized too). This syntax is +similar to the one used by [AFL] fuzzing engine (-x option). -An examples dictionary looks like: +An example dictionary looks like: ``` # Lines starting with '#' and empty lines are ignored. @@ -121,19 +233,19 @@ kw3="\xF7\xF8" "foo\x0Abar" ``` -Make sure to test your dictionary by running your fuzzer locally: +Make sure to test your dictionary by running your fuzz target locally: ```bash ./out/libfuzzer/my_protocol_fuzzer -dict=<path_to_dict> <path_to_corpus> ``` -If the dictionary is effective, you should see new units discovered in fuzzer output. +If the dictionary is effective, you should see new units discovered in the +output. To submit a dictionary to Chromium repository: -1) Add the dictionary file in the same directory as your fuzz target, with name -`<my_fuzzer>.dict`. -2) Add `dict` attribute to fuzzer definition in BUILD.gn file: +1) Add the dictionary file in the same directory as your fuzz target +2) Add `dict` attribute to `fuzzer_test` definition in BUILD.gn file: ``` fuzzer_test("my_protocol_fuzzer") { @@ -145,146 +257,15 @@ fuzzer_test("my_protocol_fuzzer") { The dictionary will be used automatically by ClusterFuzz once it picks up a new revision build. -### Corpus Minimization - -It's important to minimize seed corpus to a *small set of interesting inputs* before -uploading. The reason being that seed corpus is synced to all fuzzing bots for every -iteration, so it is important to keep it small both for fuzzing efficiency and to prevent -our bots from running out of disk space (should not exceed 1 Gb). - -The minimization can be done using `-merge=1` option of libFuzzer: - -```bash -# Create an empty directory. -mkdir seed_corpus_minimized - -# Run the fuzzer with -merge=1 flag. -./my_fuzzer -merge=1 ./seed_corpus_minimized ./seed_corpus -``` - -After running the command above, `seed_corpus_minimized` directory will contain -a minimized corpus that gives the same code coverage as the initial -`seed_corpus` directory. - -## Fuzzer Speed - -Fuzzer speed is calculated in executions per second. It is printed while the fuzzer -is running: - -``` -#19346 NEW cov: 2815 bits: 1082 indir: 43 units: 150 exec/s: 19346 L: 62 -``` - -Because libFuzzer performs randomized mutations, it is critical to have it run as -fast as possible to navigate the large search space efficiently and find interesting -code paths. You should try to get to at least 1,000 exec/s from your fuzzer runs -locally before submitting the fuzzer to Chromium repository. Profile the fuzzer -using any standard tool to see where it spends its time. - - -### Initialization/Cleanup - -Try to keep `LLVMFuzzerTestOneInput` function as simple as possible. If your fuzzing -function is too complex, it can bring down fuzzer execution speed OR it might target -very specific usecases and fail to account for unexpected scenarios. - -Prefer to use static initialization and shared resources rather than bringing the -environment up and down on every single run. Otherwise, it will slow down -fuzzer speed on every run and its ability to find new interesting inputs. -Checkout example on [startup initialization] in libFuzzer documentation. - -Fuzzers don't have to shutdown gracefully. We either kill them or they crash -because memory sanitizer tool found a problem. You can skip freeing static -resources. - -All resources allocated within `LLVMFuzzerTestOneInput` function should be -de-allocated since this function is called millions of times during a fuzzing session. -Otherwise, we will hit OOMs frequently and reduce overall fuzzing efficiency. - - -### Memory Usage - -Avoid allocation of dynamic memory wherever possible. Memory instrumentation -works faster for stack-based and static objects, than for heap allocated ones. - -It is always a good idea to try different variants for your fuzzer locally, and then -submit the fastest implementation. - -### Maximum Testcase Length - -You can control the maximum length of a test input using `-max_len` parameter -(see [custom options](#Custom-Options)). This parameter can often significantly -improve execution speed. Beware that you might miss coverage and unexpected -scenarios happening from longer size inputs. - -1) Define which `-max_len` value is reasonable for your target. For example, it -may be useless to fuzz an image decoder with too small value of testcase length. - -2) Increase the value defined on previous step. Check its influence on execution -speed of fuzzer. If speed doesn't drop significantly for long inputs, it is fine -to have some bigger value for `-max_len` or even skip it completely. - -In general, bigger `-max_len` value gives better coverage which is the main -priority for fuzzing. However, low execution speed may result in waste of -fuzzing resources and being unable to find interesting inputs in reasonable time. -If large inputs make the fuzzer too slow, you should adjust the value of `-max_len` -and find a trade-off between coverage and execution speed. - -*Note:* ClusterFuzz runs two different fuzzing engines (**LibFuzzer** and -**AFL**) using the same target functions. AFL doesn't support `-max_len` -parameter and may provide input of any length to the target. If your target has -an input length limit that you would like to *strictly enforce*, it's -recommended to add a sanity check to the beginning of your target function: - -``` -if (size > kSizeLimit) - return 0; -``` - -For more information check out the discussion in [issue 638836]. - - -## Code Coverage - -[ClusterFuzz status] page provides fuzzer source-level coverage report from -recent runs. Looking at the report might provide an insight to improve fuzzer -coverage. - -You can also generate source-level coverage report locally by running the -[coverage script] stored in Chromium repository. The script provides detailed -instructions as well as an usage example. - -We encourage you to try out the script, as it usually generates a better code -coverage visualization compared to the coverage report hosted on ClusterFuzz. -*NOTE: This is an experimental feature and an active area of work. We are -working on improving this process.* - - -## Corpus Size - -After running for a while, the fuzzer would reach a plateau and won't discover -new interesting inputs. Corpus for a reasonably complex functionality should -contain hundreds (if not thousands) of items. - -Too small of a corpus size indicates fuzzer is hitting a code barrier and is unable -to get past it. Common cases of such issues include: checksums, magic numbers, -etc. The easiest way to diagnose this problem is to generate and analyze a -[coverage report](#Coverage). To fix the issue, you can: - -* Change the code (e.g. disable crc checks while fuzzing). -* Prepare or improve [seed corpus](#Seed-Corpus). -* Prepare or improve [fuzzer dictionary](#Fuzzer-Dictionary). -* Add [custom options](#Custom-Options). - -### Custom Options +## Custom Options Custom options help to fine tune libFuzzer execution parameters and will also override the default values used by ClusterFuzz. Please read [libFuzzer options] page for detailed documentation on how these work. -Add the options needed in `libfuzzer_options` attribute to your fuzzer definition in -BUILD.gn file: +Add the options needed in `libfuzzer_options` attribute to your `fuzzer_test` +definition in BUILD.gn file: ``` fuzzer_test("my_protocol_fuzzer") { @@ -299,6 +280,18 @@ fuzzer_test("my_protocol_fuzzer") { Please note that `dict` parameter should be provided [separately](#Fuzzer-Dictionary). All other options can be passed using `libfuzzer_options` property. + +## Custom Build + +If you need to change the code being tested by your fuzz target, you may use an +`#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` macro in your target code. + +Note that patching target code is not a preferred way of improving corresponding +fuzz target, but in some cases that might be the only way possible, e.g. when +there is no intended API to disable checksum verification, or when target code +uses random generator that affects reproducibility of crashes. + + [AFL]: http://lcamtuf.coredump.cx/afl/ [ClusterFuzz status]: clusterfuzz.md#Status-Links [Corpus GCS Bucket]: https://console.cloud.google.com/storage/clusterfuzz-corpus/libfuzzer diff --git a/chromium/testing/libfuzzer/fuzzers/BUILD.gn b/chromium/testing/libfuzzer/fuzzers/BUILD.gn index 1038fd0d507..61f6006f0eb 100644 --- a/chromium/testing/libfuzzer/fuzzers/BUILD.gn +++ b/chromium/testing/libfuzzer/fuzzers/BUILD.gn @@ -49,24 +49,6 @@ fuzzer_test("snappy_fuzzer") { ] } -fuzzer_test("string_tokenizer_fuzzer") { - sources = [ - "string_tokenizer_fuzzer.cc", - ] - deps = [ - "//base", - ] -} - -fuzzer_test("string_to_int_fuzzer") { - sources = [ - "string_to_int_fuzzer.cc", - ] - deps = [ - "//base", - ] -} - fuzzer_test("template_url_parser_fuzzer") { sources = [ "template_url_parser_fuzzer.cc", @@ -80,18 +62,6 @@ fuzzer_test("template_url_parser_fuzzer") { libfuzzer_options = [ "max_len=4096" ] } -fuzzer_test("url_parse_fuzzer") { - sources = [ - "url_parse_fuzzer.cc", - ] - deps = [ - "//base", - "//base:i18n", - "//url:url", - ] - dict = "dicts/generated/url_parse_fuzzer.dict" -} - fuzzer_test("url_parse_proto_fuzzer") { sources = [ "url_parse_proto_fuzzer.cc", @@ -130,15 +100,6 @@ fuzzer_test("libsrtp_fuzzer") { libfuzzer_options = [ "max_len=1500" ] } -fuzzer_test("base_json_reader_fuzzer") { - sources = [ - "base_json_reader_fuzzer.cc", - ] - deps = [ - "//base", - ] -} - libpng_seed_corpuses = [ "//components/viz/test/data", "//third_party/WebKit/LayoutTests/images/png-suite/samples", diff --git a/chromium/testing/libfuzzer/fuzzers/base_json_reader_fuzzer.cc b/chromium/testing/libfuzzer/fuzzers/base_json_reader_fuzzer.cc deleted file mode 100644 index 7b3e44891b7..00000000000 --- a/chromium/testing/libfuzzer/fuzzers/base_json_reader_fuzzer.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <stddef.h> -#include <stdint.h> - -#include <string> - -#include "base/json/json_reader.h" -#include "base/values.h" - -int error_code, error_line, error_column; -std::string error_message; - -// Entry point for LibFuzzer. -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - if (size < 1) - return 0; - - const std::string input_string(reinterpret_cast<const char*>(data), size - 1); - const int options = data[size - 1]; - base::JSONReader::ReadAndReturnError(input_string, options, &error_code, - &error_message, &error_line, - &error_column); - return 0; -} diff --git a/chromium/testing/libfuzzer/fuzzers/dicts/generated/url_parse_fuzzer.dict b/chromium/testing/libfuzzer/fuzzers/dicts/generated/url_parse_fuzzer.dict deleted file mode 100644 index 302f5901edb..00000000000 --- a/chromium/testing/libfuzzer/fuzzers/dicts/generated/url_parse_fuzzer.dict +++ /dev/null @@ -1,403 +0,0 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This file has been generated with testing/libfuzzer/dictionary_generator.py -# using url_parse_fuzzer binary and RFC 3986. -"DNS" -"text" -"TCP" -"\"%D3%81%87%A4%95%81@%C2%85%81%83%88\"." -"[RFC2234]" -"F.," -"FORCE" -"SOCIETY" -"implementation" -"TASK" -"cache" -"WINS," -"D.1." -"to" -"only" -"HTML" -"SPONSORED" -"[RFC1630]." -"D.," -"[RFC1123]" -"resources" -"(STD" -"[RFC1808]," -"string" -"returning" -"==" -"H" -"HEREIN" -"[BCP35]" -"SP)" -"SCTP)" -"(NUL)" -"THE" -"(URI):" -"REPRESENTS" -"resource" -"A.," -"EXPRESS" -"list" -"(%2E)," -"WILL" -"J." -"INCLUDING" -"segment." -"[RFC2732]" -"(URL)\"," -"set" -"HTTP" -"IANA" -"INFORMATION" -"(%41-%5A" -"[RFC2518]" -"M." -"direct" -"(IDNA)\"," -"Only" -"Version" -"are" -"allowed." -"\"X\"" -"(SP)." -"2DIGIT" -"section" -"BUT" -"\"UTF-8," -"3" -"version" -"[RFC1034]" -"probably" -"[RFC2732]." -"metadata" -"Y.," -"C" -"WWW\"" -"FOR" -"0X" -"S" -"address" -"INPUT" -"[" -"P." -"WWW:" -"AND" -"WWW" -"[BCP35]." -"MA" -"\"AS" -"\"%\"" -"NOT" -"ANY" -"[RFC1808]" -"WARRANTY" -"useful" -"[RFC1737]." -"[STD63]," -"\"HTTP\"" -"(MIME)" -"TELNET" -"[RFC1630]" -"S." -"D.2." -"B.," -"[RFC2234]." -"[RFC2234]," -"BCP" -"[STD63];" -"use" -"LATIN" -"from" -"C." -"0" -"WARRANTIES" -"(MHTML)\"," -"ENGINEERING" -"URI;" -"few" -"(DNS)." -"expected" -"USENET" -"type" -"empty" -"XML" -"URL?\"," -"W3C/MIT" -"F" -"CA" -"STD:" -"SMTP" -"[RFC2141]," -"N" -"A)," -"NOTE:" -"CR" -"MHTML" -"must" -"ANY)," -"ALL" -"[STD63]" -"RIGHTS" -"HE/SHE" -"SP" -"[BCP19]" -"value" -"INFRINGE" -"while" -"KATAKANA" -"US-ASCII" -"W3C/IETF" -"loop" -"J.," -"2E:" -"L." -"have" -"%61-%7A)," -"is" -"PARTICULAR" -"thus" -"URI," -"parse" -"STEP" -"MIME" -"UTF-8" -"in" -"failed" -"LF" -"binary" -"ISO/IEC" -"\"A" -"(%5F)," -")" -"HTTP," -"get" -"\"A\"," -"[RFC2141]" -"BUFFER" -"ABNF" -"[RFC2557]." -"I." -"WARRANTIES," -"URN" -"EBCDIC" -"A" -"used" -"http" -"may" -"IP" -"IS" -"after" -"L" -"Q" -"'A'" -"running" -"HEXDIG" -"such" -"EBCDIC," -"data" -"[ASCII]" -"a" -"P" -"[ASCII]." -"M.," -"Names" -"the" -"[RFC0952]." -"[RFC3490]" -"US-ASCII." -"2C:" -"THAT" -"E.," -"(%2D)," -"\"URL:\"" -"WITH" -"BY" -"[UCS]," -"tables" -"[UCS]" -"TO" -"BNF" -"internal" -"P.," -"ORGANIZATION" -"\"HTTP" -"URI." -"it," -"D" -"format" -"URL" -"(0" -"URI\"" -"URI" -"K." -"URI:" -"T" -"D.W." -"not" -"R." -"LIMITED" -"\"%3A\")" -"name" -"OF" -"B." -"[RFC1736]" -"(R)," -"IPR" -"[RFC1738];" -"OUTPUT" -"LALR" -"OR" -"STD" -"[RFC3513]" -"because" -"bytes" -"DNS," -"back" -"(URI)" -"*DIGIT" -"[RFC2046]" -"[RFC3305]" -"W3C" -"E." -"for" -"space" -"ABNF\"," -"[RFC1535]." -"DQUOTE" -"I" -"does" -"'F'" -"[RFC2396]" -"be" -"K.," -"DISCLAIM" -"G" -"(UTF-16)," -"This" -"M" -"INTERNET" -"RFC" -"X3.4," -"base" -"(T):" -"IMPLIED," -"by" -"\"URL\"" -"on" -"DIGIT" -"(ABNF)" -"WEBDAV\"," -"of" -"could" -"R.," -"(ABNF:" -"S.," -"1*4HEXDIG" -"CAPITAL" -"number" -"one" -"ISO" -"FITNESS" -"\"%7E\"" -"open" -"ANSI" -"[BCP19]," -"\"%C3%80\"," -"IETF" -"support" -"\"URN" -"[RFC1123]." -"long" -"[RFC0952]" -":" -"was" -"[RFC3513]." -"[RFC2718]" -"B" -"N." -"that" -"IDNA" -"OCTET" -"but" -"R" -"POSIX" -"LETTER" -"CONTRIBUTOR," -"[RFC1738]" -"(C)" -"with" -"\"URI\"" -"16" -"default" -"double" -"\"URN\"" -"[RFC2557]" -"up" -"TCP," -"PURPOSE." -"MERCHANTABILITY" -"1)" -"IS\"" -"\"IANA" -"(URN)" -"and" -"USE" -"false" -"(IF" -"USA" -"URL," -"an" -"To" -"as" -"(%7E)" -"at" -"file" -"need" -"any" -"\"%E3%82%A2\"." -"physical" -"1*HEXDIG" -"no" -"[RFC1737]" -"-" -"invalid" -"A." -"application" -"valid" -"take" -"which" -"test" -"[RFC2732]," -"you" -"=" -"GRAVE" -"<URI>" -"[RFC2396]," -"2B:" -"period," -"UDP," -"[RFC1535]" -"T." -"(UCS)\"," -"U" -"A-F." -"T.," -"]" -"[RFC2718]." -"D." -"persistent" -"traditional" -"L.," -"As" -"IMPLIED" -"(URL)" -"ALPHA" -"[RFC3305]." -"H.," -"\"MIME" - diff --git a/chromium/testing/libfuzzer/fuzzers/string_to_int_fuzzer.cc b/chromium/testing/libfuzzer/fuzzers/string_to_int_fuzzer.cc deleted file mode 100644 index ca42f678528..00000000000 --- a/chromium/testing/libfuzzer/fuzzers/string_to_int_fuzzer.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <stddef.h> -#include <stdint.h> - -#include <string> -#include <vector> - -#include "base/strings/string_number_conversions.h" - -// Entry point for LibFuzzer. -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - std::string input(reinterpret_cast<const char*>(data), size); - int out_int; - base::StringToInt(input, &out_int); - unsigned out_uint; - base::StringToUint(input, &out_uint); - int64_t out_int64; - base::StringToInt64(input, &out_int64); - uint64_t out_uint64; - base::StringToUint64(input, &out_uint64); - size_t out_size; - base::StringToSizeT(input, &out_size); - double out_double; - base::StringToDouble(input, &out_double); - base::HexStringToInt(input, &out_int); - base::HexStringToUInt(input, &out_uint); - base::HexStringToInt64(input, &out_int64); - base::HexStringToUInt64(input, &out_uint64); - - std::vector<uint8_t> out_bytes; - base::HexStringToBytes(input, &out_bytes); - - base::HexEncode(data, size); - return 0; -} diff --git a/chromium/testing/libfuzzer/fuzzers/string_tokenizer_fuzzer.cc b/chromium/testing/libfuzzer/fuzzers/string_tokenizer_fuzzer.cc deleted file mode 100644 index c41b8aecc4f..00000000000 --- a/chromium/testing/libfuzzer/fuzzers/string_tokenizer_fuzzer.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <stddef.h> -#include <stdint.h> - -#include <string> - -#include "base/strings/string_tokenizer.h" - -void GetAllTokens(base::StringTokenizer& t) { - while (t.GetNext()) { - (void)t.token(); - } -} - -// Entry point for LibFuzzer. -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - if (size < 1) { - return 0; - } - - // Allow quote_chars and options to be set. Otherwise full coverage - // won't be possible since IsQuote, FullGetNext and other functions - // won't be called. - size_t pattern_size = data[0]; - - if (pattern_size > size - 1) { - return 0; - } - - std::string pattern(reinterpret_cast<const char*>(data + 1), - pattern_size); - - std::string input( - reinterpret_cast<const char*>(data + 1 + pattern_size), - size - pattern_size - 1); - - - base::StringTokenizer t(input, pattern); - GetAllTokens(t); - - base::StringTokenizer t_quote(input, pattern); - t_quote.set_quote_chars("\""); - GetAllTokens(t_quote); - - base::StringTokenizer t_options(input, pattern); - t_options.set_options(base::StringTokenizer::RETURN_DELIMS); - GetAllTokens(t_options); - - - base::StringTokenizer t_quote_and_options(input, pattern); - t_quote_and_options.set_quote_chars("\""); - t_quote_and_options.set_options(base::StringTokenizer::RETURN_DELIMS); - GetAllTokens(t_quote_and_options); - - return 0; -} diff --git a/chromium/testing/libfuzzer/fuzzers/url_parse_fuzzer.cc b/chromium/testing/libfuzzer/fuzzers/url_parse_fuzzer.cc deleted file mode 100644 index e2e814da6e1..00000000000 --- a/chromium/testing/libfuzzer/fuzzers/url_parse_fuzzer.cc +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <stddef.h> -#include <stdint.h> - -#include "base/at_exit.h" -#include "base/i18n/icu_util.h" -#include "url/gurl.h" - -struct TestCase { - TestCase() { - CHECK(base::i18n::InitializeICU()); - } - - // used by ICU integration. - base::AtExitManager at_exit_manager; -}; - -TestCase* test_case = new TestCase(); - -// Entry point for LibFuzzer. -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - GURL url(std::string(reinterpret_cast<const char*>(data), size)); - return 0; -} diff --git a/chromium/testing/libfuzzer/getting_started.md b/chromium/testing/libfuzzer/getting_started.md index 94eb007acc0..fc6adc07765 100644 --- a/chromium/testing/libfuzzer/getting_started.md +++ b/chromium/testing/libfuzzer/getting_started.md @@ -6,9 +6,9 @@ This document will walk you through: -* setting up your build enviroment. -* creating your first fuzzer. -* running the fuzzer and verifying its vitals. +* setting up your build environment. +* creating your first fuzz target. +* running the fuzz target and verifying its vitals. ## Configure Build @@ -29,18 +29,18 @@ Supported sanitizer configurations are: | GN Argument | Description | |--------------|----| | `is_asan=true` | enables [Address Sanitizer] to catch problems like buffer overruns. | -| `is_msan=true` | enables [Memory Sanitizer] to catch problems like uninitialed reads<sup>\[[*](reference.md#MSan)\]</sup>. | +| `is_msan=true` | enables [Memory Sanitizer] to catch problems like uninitialized reads<sup>\[[*](reference.md#MSan)\]</sup>. | | `is_ubsan_security=true` | enables [Undefined Behavior Sanitizer] to catch<sup>\[[*](reference.md#UBSan)\]</sup> undefined behavior like integer overflow. | | | it is possible to run libfuzzer without any sanitizers; *probably not what you want*.| - Fuzzers are built with minimal symbols by default, regardless of the value of -`is_debug` and `symbol_level`. However if you want to run the fuzzer under a +Fuzz targets are built with minimal symbols by default, regardless of the value +of `is_debug` and `symbol_level`. However if you want to run fuzz target under a debugger you can re-enable them by setting `sanitizer_keep_symbols=true`. To get the exact GN configuration that are used on our builders, see [Build Config]. -## Write Fuzzer Function +## Write Fuzz Target Create a new `<my_fuzzer>.cc` file and define a `LLVMFuzzerTestOneInput` function: @@ -54,10 +54,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { } ``` -*Note*: You should create the fuzzer file `<my_fuzzer>.cc next to the code that is being -tested and in the same directory as your other unit tests. Please do not use -`testing/libfuzzer/fuzzers` directory, this was a directory used for initial sample fuzzers and -is no longer recommended for any new fuzzers. +*Note*: You should create the fuzz target file `<my_fuzzer>.cc` next to the code +that is being tested and in the same directory as your other unit tests. Please +do not use `testing/libfuzzer/fuzzers` directory, this was a directory used for +initial sample fuzz targets and is no longer recommended for landing new fuzz +targets. [quic_stream_factory_fuzzer.cc] is a good example of real-world fuzz target. @@ -73,7 +74,7 @@ fuzzer_test("my_fuzzer") { } ``` -## Build and Run Fuzzer Locally +## Build and Run Fuzz Target Locally Build with ninja as usual and run: @@ -82,24 +83,29 @@ ninja -C out/libfuzzer url_parse_fuzzer ./out/libfuzzer/url_parse_fuzzer ``` -Your fuzzer should produce output like this: +Your fuzz target should produce output like this: ``` -INFO: Seed: 1787335005 -INFO: -max_len is not provided, using 64 -INFO: PreferSmall: 1 -#0 READ units: 1 exec/s: 0 -#1 INITED cov: 2361 bits: 95 indir: 29 units: 1 exec/s: 0 -#2 NEW cov: 2710 bits: 359 indir: 36 units: 2 exec/s: 0 L: 64 MS: 0 +INFO: Seed: 1511722356 +INFO: Loaded 2 modules (115485 guards): 22572 [0x7fe8acddf560, 0x7fe8acdf5610), 92913 [0xaa05d0, 0xafb194), +INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes +INFO: A corpus is not provided, starting from an empty corpus +#2 INITED cov: 961 ft: 48 corp: 1/1b exec/s: 0 rss: 48Mb +#3 NEW cov: 986 ft: 70 corp: 2/104b exec/s: 0 rss: 48Mb L: 103/103 MS: 1 InsertRepeatedBytes- +#4 NEW cov: 989 ft: 74 corp: 3/106b exec/s: 0 rss: 48Mb L: 2/103 MS: 1 InsertByte- +#6 NEW cov: 991 ft: 76 corp: 4/184b exec/s: 0 rss: 48Mb L: 78/103 MS: 2 CopyPart-InsertRepeatedBytes- ``` -The `... NEW ...` line appears when libFuzzer finds new and interesting inputs. -The efficient fuzzer should be able to finds lots of them rather quickly. -The `... pulse ...` line will appear periodically to show the current status. +* `... NEW ...` line appears when libFuzzer finds new and interesting inputs. +* an efficient fuzz target should be able to finds lots of them rather quickly. +* `... pulse ...` line will appear periodically to show the current status. + +For more information about libFuzzer's output, please refer to [its own +documentation]. ### Symbolize Stacktrace -If your fuzzer crashes when running locally and you see non-symbolized +If your fuzz target crashes when running locally and you see non-symbolized stacktrace, make sure that you have directory containing `llvm-symbolizer` binary added in `$PATH`. The symbolizer binary is included in Chromium's Clang package located at `third_party/llvm-build/Release+Asserts/bin/` directory. @@ -112,35 +118,42 @@ $ ASAN_OPTIONS=external_symbolizer_path=/my/local/llvm/build/llvm-symbolizer \ ./fuzzer ./crash-input ``` -The same approach works with other sanitizers (e.g. `MSAN_OPTIONS`, `UBSAN_OPTIONS`, etc). +The same approach works with other sanitizers (e.g. `MSAN_OPTIONS`, +`UBSAN_OPTIONS`, etc). -## Improving Your Fuzzer +## Improving Your Fuzz Target -Your fuzzer may immediately discover interesting (i.e. crashing) inputs. +Your fuzz target may immediately discover interesting (i.e. crashing) inputs. To make it more efficient, several small steps can take you really far: * Create seed corpus. Add `seed_corpus = "src/fuzz-testcases/"` attribute -to your fuzzer targets and add example files in appropriate folder. Read more -in [Seed Corpus] section of efficient fuzzer guide. +to your fuzzer target and add example files in appropriate folder. Read more +in [Seed Corpus] section of the [Efficient Fuzzer Guide]. *Make sure corpus files are appropriately licensed.* * Create mutation dictionary. With a `dict = "protocol.dict"` attribute and -`key=value` dicitionary file format, mutations can be more effective. -See [Fuzzer Dictionary] section of efficient fuzzer guide. -* Specify maximum testcase length. By default libFuzzer uses `-max_len=64` - (or takes the longest testcase in a corpus). ClusterFuzz takes -random value in range from `1` to `10000` for each fuzzing session and passes -that value to libFuzzers. If corpus contains testcases of size greater than -`max_len`, libFuzzer will use only first `max_len` bytes of such testcases. -See [Maximum Testcase Length] section of the efficient fuzzer guide. - -## Disable noisy error message logging - -If the code that you are a fuzzing generates lot of error messages when +`key=value` dictionary file format, mutations can be more effective. +See [Fuzzer Dictionary] section of the [Efficient Fuzzer Guide]. +* Specify testcase length limits. By default, libFuzzer uses `-max_len=4096` +or takes the longest testcase in the corpus if `-max_len` is not specified. +ClusterFuzz uses different strategies for different fuzzing sessions, including +different random values. Also, ClusterFuzz uses different fuzzing engines (e.g. +AFL that doesn't have `-max_len` option). If your target has an input length +limit that you would like to *strictly enforce*, add a sanity check to the +beginning of your target function: + +```cpp +if (size < kMinInputLength || size > kMaxInputLength) + return 0; +``` + +### Disable noisy error message logging + +If the code that you are fuzzing generates lot of error messages when encountering incorrect or invalid data, then you need to silence those errors -in the fuzzer. Otherwise, fuzzer will be slow and inefficient. +in the fuzz target. Otherwise, fuzz target will be slow and inefficient. -If the target uses the Chromium logging APIs, the best way to do that is to -override the environment used for logging in your fuzzer: +If the target uses Chromium logging APIs, the best way to do that is to +override the environment used for logging in your fuzz target: ```cpp struct Environment { @@ -152,20 +165,135 @@ struct Environment { Environment* env = new Environment(); ``` -## Submitting Fuzzer to ClusterFuzz +## Mutating Multiple Inputs + +By default, a fuzzing engine such as libFuzzer mutates a single input referenced +by `uint8_t* data, size_t size`. However, quite often an API under fuzz testing +accepts multiple arguments of various types rather than a single buffer. There +are three approaches for such cases: + +### 1) libprotobuf-mutator + +If you need to mutate multiple inputs of various types and length, please see +[Getting Started with libprotobuf-mutator in Chromium]. That approach allows +to mutate multiple inputs independently. + +**Caveats:** This approach requires an extra effort, but works with APIs and +data structures of any complexity. + +### 2) hash-based argument + +Another frequent case of an API under fuzz testing is a function that accepts a +buffer with data and some integer value meaning a bitwise combination of flags. +For such cases, we recommend to calculate a hash value from `(data, size)` and +use that value for fuzzing of an additional integer argument, for example: + +```cpp +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + std::string str = std::string(reinterpret_cast<const char*>(data), size); + std::size_t data_hash = std::hash<std::string>()(str); + APIToBeFuzzed(data, size, data_hash); + return 0; +} + +``` + +**Caveats:** Hash value derived from the data would be a random value rather +than a meaningful value controlled by fuzzing engine, i.e. a single bit mutation +would result in a completely different hash value that might lead to a new code +coverage, but the next mutation would generate another hash value and trigger +another code path, without providing a real guidance to the fuzzing engine. + +### 3) bytes taken from (data, size) + +You can extract one or more bytes from the data provided by fuzzing engine and +use that value for fuzzing other arguments of the target API or making other +decisions (e.g. number of iterations or attempts for calling some function). +Note that those bytes should not be used as data for any other arguments, e.g.: + +```cpp +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // Don't forget to enforce minimal data length. + if (size < 1) + return 0; + + // Extract single byte for fuzzing "flags" value. + uint8_t flags = data[0]; + + // Wrong, there is a bias between flags and API input. + APIToBeFuzzed(data, size, flags); + + // Good, API input and flags are independent. + APIToBeFuzzed(data + 1, size - 1, flags); + + return 0; +} +``` + +This approach addresses the problem of the *hash-based argument* approach, but +has its own **caveats**: + +* If you extract any bytes from the input (either first or last ones), you +cannot use valid samples as seed corpus. In that case, you'll have to generate +seed corpus manually, i.e. append necessary bytes to the valid sample inputs. + +* Imagine that `APIToBeFuzzed()` had a bug, something like the following: + +```cpp +void APIToBeFuzzed(uint8_t* buffer, size_t length, uint8_t options) { + ... + if (options == 0x66) { + // Yes, looks ridiculous, but things like that did happen in the real world. + *(buffer - 1) = -1; + } + ... +} +``` + +assuming we used the fuzz target listed above, neither ASan nor other santizers +would detect a buffer underwrite vulnerability, as the byte addressed by +`buffer - 1` is actually a mapped memory allocated inside the fuzzing engine as +`data[0]`. + +To avoid issues like that one, we would have to allocate a separate buffer and +copy API input into it, or use a container object e.g.: + +```cpp +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // Don't forget to enforce minimal data length. + if (size < 1) + return 0; + + // Extract single byte for fuzzing flags value. + uint8_t flags = data[0]; + + // Put API input into a separate container. + std::vector<uint8_t> buffer(data + 1, data + size); + + APIToBeFuzzed(buffer.data(), buffer.size(), flags); + + return 0; +} +``` + +There is [base::FuzzedDataProvider] class that might be helpful for writing +fuzz targets using that approach. + + +## Submitting Fuzz Target to ClusterFuzz ClusterFuzz builds and executes all `fuzzer_test` targets in the Chromium -repository. It is extremely important to submit a fuzzer into Chromium +repository. It is extremely important to land a fuzz target into Chromium repository so that ClusterFuzz can run it at scale. Do not rely on just -running fuzzing locally in your own environment, as it will catch far less -issues. It's crucial to run fuzzers continuously forever for catching +running fuzzers locally in your own environment, as it will catch far less +issues. It's crucial to run fuzz targets continuously forever for catching regressions and improving code coverage over time. ## Next Steps -* After your fuzzer is submitted, you should check [ClusterFuzz status] page in -a day or two. -* Check the [Efficient Fuzzer Guide] to better understand your fuzzer +* After your fuzz target is landed, you should check [ClusterFuzz status] page +in a day or two. +* Check the [Efficient Fuzzer Guide] to better understand your fuzz target performance and for optimization hints. @@ -173,10 +301,12 @@ performance and for optimization hints. [ClusterFuzz status]: clusterfuzz.md#Status-Links [Efficient Fuzzer Guide]: efficient_fuzzer.md [Fuzzer Dictionary]: efficient_fuzzer.md#Fuzzer-Dictionary -[Maximum Testcase Length]: efficient_fuzzer.md#Maximum-Testcase-Length [Memory Sanitizer]: http://clang.llvm.org/docs/MemorySanitizer.html [Seed Corpus]: efficient_fuzzer.md#Seed-Corpus [Undefined Behavior Sanitizer]: http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html [crbug/598448]: https://bugs.chromium.org/p/chromium/issues/detail?id=598448 [quic_stream_factory_fuzzer.cc]: https://cs.chromium.org/chromium/src/net/quic/chromium/quic_stream_factory_fuzzer.cc [Build Config]: reference.md#Builder-configurations +[its own documentation]: http://llvm.org/docs/LibFuzzer.html#output +[Getting Started with libprotobuf-mutator in Chromium]: libprotobuf-mutator.md +[base::FuzzedDataProvider]: https://cs.chromium.org/chromium/src/base/test/fuzzed_data_provider.h diff --git a/chromium/testing/libfuzzer/proto/lpm_interface.h b/chromium/testing/libfuzzer/proto/lpm_interface.h new file mode 100644 index 00000000000..22f2ea11e1f --- /dev/null +++ b/chromium/testing/libfuzzer/proto/lpm_interface.h @@ -0,0 +1,16 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Header file that includes libfuzzer_macro.h from libprotobuf-mutator. Useful +// for inclusion in fuzz targets that can't include headers from third_party/. + +#ifndef TESTING_LIBFUZZER_PROTO_LPM_INTERFACE_H_ +#define TESTING_LIBFUZZER_PROTO_LPM_INTERFACE_H_ + +#include "third_party/libprotobuf-mutator/src/src/libfuzzer/libfuzzer_macro.h" + +// Silence logging from the protobuf library. +protobuf_mutator::protobuf::LogSilencer log_silencer; + +#endif // TESTING_LIBFUZZER_PROTO_LPM_INTERFACE_H_ diff --git a/chromium/testing/libfuzzer/reference.md b/chromium/testing/libfuzzer/reference.md index aa6480e3b98..04b3459cba6 100644 --- a/chromium/testing/libfuzzer/reference.md +++ b/chromium/testing/libfuzzer/reference.md @@ -1,6 +1,6 @@ # libFuzzer Integration Reference -## Additional sanitizer configuration +## Additional Sanitizer Configuration ### MSan @@ -79,11 +79,13 @@ Following arguments are supported: | Argument | Description | |----------|-------------| -| sources | **required** list of fuzzer test source files | -| deps | fuzzer dependencies | -| additional_configs | additional GN configurations to be used for compilation | -| dict | a dictionary file for the fuzzer | -| libfuzzer_options | runtime options file for the fuzzer. See [Fuzzer Runtime Options](#Fuzzer-Runtime-Options) | +| `sources` | **required** list of fuzzer test source files | +| `deps` | fuzzer dependencies | +| `additional_configs` | additional GN configurations to be used for compilation | +| `dict` | a dictionary file for the fuzzer | +| `libfuzzer_options` | runtime options file for the fuzzer. See [Fuzzer Runtime Options](#Fuzzer-Runtime-Options) | +| `seed_corpus` | single directory containing test inputs, parsed recursively | +| `seed_corpuses` | multiple directories with the same purpose as `seed_corpus` | ## Fuzzer Runtime Options diff --git a/chromium/testing/scripts/check_network_annotation_auditor.py b/chromium/testing/scripts/check_network_annotation_auditor.py new file mode 100755 index 00000000000..bc9c06bf7e9 --- /dev/null +++ b/chromium/testing/scripts/check_network_annotation_auditor.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""//testing/scripts wrapper for the network traffic annotation auditor +checks.""" + +import json +import os +import sys + + +import common + + +def main_run(args): + command_line = [ + sys.executable, + os.path.join(common.SRC_DIR, 'tools', 'traffic_annotation', 'scripts', + 'traffic_annotation_auditor_tests.py'), + '--build-path', + os.path.join(args.paths['checkout'], 'out', args.build_config_fs), + ] + rc = common.run_command(command_line) + + json.dump({ + 'valid': True, + 'failures': ['Please refer to stdout for errors.'] if rc else [], + }, args.output) + + return rc + + +def main_compile_targets(args): + json.dump(['chrome', 'remoting/host:host', 'remoting/client:client'], + args.output) + + +if __name__ == '__main__': + funcs = { + 'run': main_run, + 'compile_targets': main_compile_targets, + } + sys.exit(common.run_script(sys.argv[1:], funcs)) diff --git a/chromium/testing/scripts/check_network_annotations.py b/chromium/testing/scripts/check_network_annotations.py index e4cdf089d87..1edccf138a4 100755 --- a/chromium/testing/scripts/check_network_annotations.py +++ b/chromium/testing/scripts/check_network_annotations.py @@ -14,13 +14,14 @@ import common def main_run(args): - rc = common.run_command([ + command_line = [ sys.executable, os.path.join(common.SRC_DIR, 'tools', 'traffic_annotation', 'scripts', 'check_annotations.py'), '--build-path', os.path.join(args.paths['checkout'], 'out', args.build_config_fs), - ]) + ] + rc = common.run_command(command_line) json.dump({ 'valid': True, diff --git a/chromium/testing/scripts/run_gtest_perf_test.py b/chromium/testing/scripts/run_gtest_perf_test.py index f2923c483bd..83198f5cc16 100755 --- a/chromium/testing/scripts/run_gtest_perf_test.py +++ b/chromium/testing/scripts/run_gtest_perf_test.py @@ -82,6 +82,25 @@ def main(): args, rest_args = parser.parse_known_args() + rc, charts, output_json = execute_perf_test(args, rest_args) + + # TODO(eakuefner): Make isolated_script_test_perf_output mandatory after + # flipping flag in swarming. + if args.isolated_script_test_perf_output: + filename = args.isolated_script_test_perf_output + else: + filename = args.isolated_script_test_chartjson_output + # Write the returned encoded json to a the charts output file + with open(filename, 'w') as f: + f.write(charts) + + with open(args.isolated_script_test_output, 'w') as fp: + json.dump(output_json, fp) + + return rc + + +def execute_perf_test(args, rest_args): env = os.environ.copy() # Assume we want to set up the sandbox environment variables all the # time; doing so is harmless on non-Linux platforms and is needed @@ -124,28 +143,17 @@ def main(): results_processor = ( generate_legacy_perf_dashboard_json.LegacyResultsProcessor()) charts = results_processor.GenerateJsonResults(tempfile_path) - # TODO(eakuefner): Make isolated_script_test_perf_output mandatory after - # flipping flag in swarming. - if args.isolated_script_test_perf_output: - filename = args.isolated_script_test_perf_output - else: - filename = args.isolated_script_test_chartjson_output - # Write the returned encoded json to a the charts output file - with open(filename, 'w') as f: - f.write(charts) except Exception: traceback.print_exc() rc = 1 valid = (rc == 0) failures = [] if valid else ['(entire test suite)'] - with open(args.isolated_script_test_output, 'w') as fp: - json.dump({ - 'valid': valid, - 'failures': failures, - }, fp) - - return rc + output_json = { + 'valid': valid, + 'failures': failures, + } + return rc, charts, output_json # This is not really a "script test" so does not need to manually add # any additional compile targets. diff --git a/chromium/testing/scripts/run_performance_tests.py b/chromium/testing/scripts/run_performance_tests.py index f19dbb09574..da21a8bbb56 100755 --- a/chromium/testing/scripts/run_performance_tests.py +++ b/chromium/testing/scripts/run_performance_tests.py @@ -27,7 +27,11 @@ followed by a subsequent Python script. It could be generalized to invoke an arbitrary executable. It currently runs several benchmarks. The benchmarks it will execute are -based on the shard it is running on and the sharding_map_path( +based on the shard it is running on and the sharding_map_path. + +If this is executed with a non-telemetry perf test, the flag --non-telemetry +has to be passed in to the script so the script knows it is running +an executable and not the run_benchmark command. The results of running the benchmark are put in separate directories per benchmark. Two files will be present in each directory; perf_results.json, which @@ -51,16 +55,31 @@ import traceback import common import run_telemetry_benchmark_as_googletest +import run_gtest_perf_test # Current whitelist of benchmarks outputting histograms BENCHMARKS_TO_OUTPUT_HISTOGRAMS = [ 'dummy_benchmark.histogram_benchmark_1', + 'blink_perf.bindings', + 'blink_perf.canvas', + 'blink_perf.css', + 'blink_perf.dom', + 'blink_perf.events', + 'blink_perf.image_decoder', + 'blink_perf.layout', + 'blink_perf.owp_storage', + 'blink_perf.paint', + 'blink_perf.parser', + 'blink_perf.shadow_dom', + 'blink_perf.svg', + 'memory.top_10_mobile' ] # We currently have two different sharding schemes for android -# vs desktop. -CURRENT_DESKTOP_NUM_SHARDS = 5 -CURRENT_ANDROID_NUM_SHARDS = 21 +# vs desktop. When we are running at capacity we will have 26 +# desktop shards and 39 android. +CURRENT_DESKTOP_NUM_SHARDS = 26 +CURRENT_ANDROID_NUM_SHARDS = 39 def get_sharding_map_path(total_shards, testing): # Determine if we want to do a test run of the benchmarks or run the @@ -81,6 +100,21 @@ def get_sharding_map_path(total_shards, testing): 'benchmark_bot_map.json') +def write_results( + perf_test_name, perf_results, json_test_results, isolated_out_dir, encoded): + benchmark_path = os.path.join(isolated_out_dir, perf_test_name) + + os.makedirs(benchmark_path) + with open(os.path.join(benchmark_path, 'perf_results.json'), 'w') as f: + # non telemetry perf results are already json encoded + if encoded: + f.write(perf_results) + else: + json.dump(perf_results, f) + with open(os.path.join(benchmark_path, 'test_results.json'), 'w') as f: + json.dump(json_test_results, f) + + def execute_benchmark(benchmark, isolated_out_dir, args, rest_args, is_reference): # While we are between chartjson and histogram set we need @@ -100,7 +134,7 @@ def execute_benchmark(benchmark, isolated_out_dir, # Need to append output format. per_benchmark_args = (rest_args[:1] + [benchmark] + rest_args[1:] + [output_format]) - benchmark_path = None + benchmark_name = benchmark if is_reference: # Need to parse out the browser to replace browser flag with # reference build so we run it reference build as well @@ -113,9 +147,7 @@ def execute_benchmark(benchmark, isolated_out_dir, # Now we need to add in the rest of the reference build args per_benchmark_args.append('--max-failures=5') per_benchmark_args.append('--output-trace-tag=_ref') - benchmark_path = os.path.join(isolated_out_dir, benchmark + '.reference') - else: - benchmark_path = os.path.join(isolated_out_dir, benchmark) + benchmark_name = benchmark + '.reference' # We don't care exactly what these are. In particular, the perf results # could be any format (chartjson, legacy, histogram). We just pass these @@ -124,11 +156,8 @@ def execute_benchmark(benchmark, isolated_out_dir, run_telemetry_benchmark_as_googletest.run_benchmark( args, per_benchmark_args, is_histograms)) - os.makedirs(benchmark_path) - with open(os.path.join(benchmark_path, 'perf_results.json'), 'w') as f: - json.dump(perf_results, f) - with open(os.path.join(benchmark_path, 'test_results.json'), 'w') as f: - json.dump(json_test_results, f) + write_results( + benchmark_name, perf_results, json_test_results, isolated_out_dir, False) return rc @@ -148,38 +177,53 @@ def main(): parser.add_argument( '--isolated-script-test-filter', type=str, required=False) parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') + # TODO(eyaich) We could potentially assume this based on shards == 1 since + # benchmarks will always have multiple shards. + parser.add_argument('--non-telemetry', + help='Type of perf test', type=bool, default=False) parser.add_argument('--testing', help='Testing instance', type=bool, default=False) args, rest_args = parser.parse_known_args() isolated_out_dir = os.path.dirname(args.isolated_script_test_output) - # First determine what shard we are running on to know how to - # index into the bot map to get list of benchmarks to run. - total_shards = None - shard_index = None - - env = os.environ.copy() - if 'GTEST_TOTAL_SHARDS' in env: - total_shards = env['GTEST_TOTAL_SHARDS'] - if 'GTEST_SHARD_INDEX' in env: - shard_index = env['GTEST_SHARD_INDEX'] - - if not (total_shards or shard_index): - raise Exception('Shard indicators must be present for perf tests') - - sharding_map_path = get_sharding_map_path(total_shards, args.testing or False) - with open(sharding_map_path) as f: - sharding_map = json.load(f) - sharding = None - sharding = sharding_map[shard_index]['benchmarks'] - return_code = 0 - - for benchmark in sharding: - return_code = (execute_benchmark( - benchmark, isolated_out_dir, args, rest_args, False) or return_code) - return_code = (execute_benchmark( - benchmark, isolated_out_dir, args, rest_args, True) or return_code) + if args.non_telemetry: + # For non telemetry tests the benchmark name is the name of the executable. + benchmark_name = rest_args[0] + return_code, charts, output_json = run_gtest_perf_test.execute_perf_test( + args, rest_args) + + write_results(benchmark_name, charts, output_json, isolated_out_dir, True) + else: + # First determine what shard we are running on to know how to + # index into the bot map to get list of benchmarks to run. + total_shards = None + shard_index = None + + env = os.environ.copy() + if 'GTEST_TOTAL_SHARDS' in env: + total_shards = env['GTEST_TOTAL_SHARDS'] + if 'GTEST_SHARD_INDEX' in env: + shard_index = env['GTEST_SHARD_INDEX'] + + if not (total_shards or shard_index): + raise Exception('Shard indicators must be present for perf tests') + + sharding_map_path = get_sharding_map_path( + total_shards, args.testing or False) + with open(sharding_map_path) as f: + sharding_map = json.load(f) + sharding = None + sharding = sharding_map[shard_index]['benchmarks'] + return_code = 0 + + for benchmark in sharding: + return_code = (execute_benchmark( + benchmark, isolated_out_dir, args, rest_args, False) or return_code) + # We ignore the return code of the reference build since we do not + # monitor it. + execute_benchmark(benchmark, isolated_out_dir, args, rest_args, True) + return return_code # This is not really a "script test" so does not need to manually add diff --git a/chromium/testing/scripts/run_performance_tests_wrapper.py b/chromium/testing/scripts/run_performance_tests_wrapper.py new file mode 100755 index 00000000000..4924fb6e6bc --- /dev/null +++ b/chromium/testing/scripts/run_performance_tests_wrapper.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This script is a wrapper script used during migration of performance +tests to use the new recipe. See crbug.com/757933 + +Non-telemetry tests now will all run with this script. The flag +--migrated-test will indicate if this test is using the new recipe or not. +By default this script runs the legacy testing/scripts/run_gtest_perf_test.py. + +""" + +import argparse +import os +import subprocess +import sys + +SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath( + __file__)))) + +GTEST = os.path.join(SRC_DIR, 'testing', 'scripts', 'run_gtest_perf_test.py') +PERF = os.path.join(SRC_DIR, 'testing', 'scripts', 'run_performance_tests.py') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--migrated-test', type=bool, default=False) + + args, rest_args = parser.parse_known_args() + + if args.migrated_test: + return subprocess.call([sys.executable, PERF] + rest_args) + else: + return subprocess.call([sys.executable, GTEST] + rest_args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/chromium/testing/test.gni b/chromium/testing/test.gni index f9f8c911cab..0f929858382 100644 --- a/chromium/testing/test.gni +++ b/chromium/testing/test.gni @@ -10,6 +10,7 @@ if (is_android) { import("//build/config/android/config.gni") import("//build/config/android/rules.gni") import("//build/config/sanitizers/sanitizers.gni") + import("//build/config/android/extract_unwind_tables.gni") } if (is_fuchsia) { @@ -100,6 +101,21 @@ template("test") { "write_asset_list", "use_native_activity", ] + + # Adds the unwind tables from unstripped binary as an asset file in the + # apk, if |add_unwind_tables_in_apk| is specified by the test. + if (defined(invoker.add_unwind_tables_in_apk) && + invoker.add_unwind_tables_in_apk) { + _unwind_table_asset_name = "${target_name}_unwind_assets" + unwind_table_asset(_unwind_table_asset_name) { + testonly = true + library_target = _library_target + deps = [ + ":$_library_target", + ] + } + } + shared_library(_library_target) { # Configs will always be defined since we set_defaults in BUILDCONFIG.gn. configs = [] # Prevent list overwriting warning. @@ -129,6 +145,10 @@ template("test") { # Add the Java classes so that each target does not have to do it. deps += [ "//base/test:test_support_java" ] + if (defined(_unwind_table_asset_name)) { + deps += [ ":${_unwind_table_asset_name}" ] + } + # TODO(agrieve): Remove this data_dep once bots don't build the _apk # target (post-GYP). # It's a bit backwards for the apk to depend on the runner script, since diff --git a/chromium/testing/trigger_scripts/PRESUBMIT.py b/chromium/testing/trigger_scripts/PRESUBMIT.py index c666fc5e18c..17e0e443273 100644 --- a/chromium/testing/trigger_scripts/PRESUBMIT.py +++ b/chromium/testing/trigger_scripts/PRESUBMIT.py @@ -39,9 +39,9 @@ def PostUploadHook(cl, change, output_api): return output_api.EnsureCQIncludeTrybotsAreAdded( cl, [ - 'master.tryserver.chromium.linux:linux_optional_gpu_tests_rel', - 'master.tryserver.chromium.mac:mac_optional_gpu_tests_rel', - 'master.tryserver.chromium.win:win_optional_gpu_tests_rel', - 'master.tryserver.chromium.android:android_optional_gpu_tests_rel', + 'luci.chromium.try:linux_optional_gpu_tests_rel', + 'luci.chromium.try:mac_optional_gpu_tests_rel', + 'luci.chromium.try:win_optional_gpu_tests_rel', + 'luci.chromium.try:android_optional_gpu_tests_rel', ], 'Automatically added optional GPU tests to run on CQ.') diff --git a/chromium/testing/trigger_scripts/base_test_triggerer.py b/chromium/testing/trigger_scripts/base_test_triggerer.py index 98f19e0edc9..598bdea66e3 100755 --- a/chromium/testing/trigger_scripts/base_test_triggerer.py +++ b/chromium/testing/trigger_scripts/base_test_triggerer.py @@ -74,12 +74,6 @@ class BaseTestTriggerer(object): list, to either affect the trigger command, or what the bot runs. """ - - - - assert '--' in all_args, ( - 'Malformed trigger command; -- argument expected but not found') - dash_ind = all_args.index('--') bot_args = ['--dump-json', temp_file] if total_shards > 1: bot_args.append('--env') @@ -92,13 +86,18 @@ class BaseTestTriggerer(object): bot_args.append('--dimension') bot_args.append(key) bot_args.append(val) - return self.append_additional_args( - all_args[:dash_ind] + bot_args + all_args[dash_ind:]) - - def append_additional_args(self, args): + if '--' in all_args: + dash_ind = all_args.index('--') + additional_args = all_args[:dash_ind] + bot_args + all_args[dash_ind:] + else: + additional_args = all_args + bot_args + return self.append_additional_args(additional_args, shard_index) + + def append_additional_args(self, args, shard_index): """ Gives subclasses ability to append additional args if necessary - Base class just returns given get.""" + Base class just returns given args.""" + del shard_index # unused return args def parse_bot_configs(self, args): diff --git a/chromium/testing/trigger_scripts/perf_device_trigger.py b/chromium/testing/trigger_scripts/perf_device_trigger.py index f24670cf262..c9568e6e416 100755 --- a/chromium/testing/trigger_scripts/perf_device_trigger.py +++ b/chromium/testing/trigger_scripts/perf_device_trigger.py @@ -40,8 +40,18 @@ class PerfDeviceTriggerer(base_test_triggerer.BaseTestTriggerer): def __init__(self): super(PerfDeviceTriggerer, self).__init__() - def append_additional_args(self, args): - return args + def append_additional_args(self, args, shard_index): + # Append a tag to the swarming task with the shard number + # so we can query for the last bot that ran a specific shard. + tag = 'shard:%d' % shard_index + shard_tag = ['--tag', tag] + # Need to append this before the dash if present so it gets fed to + # the swarming task itself. + if '--' in args: + dash_ind = args.index('--') + return args[:dash_ind] + shard_tag + args[dash_ind:] + else: + return args + shard_tag def select_config_indices(self, args, verbose): # For perf we want to trigger a job for every valid config since diff --git a/chromium/testing/trigger_scripts/perf_device_trigger_unittest.py b/chromium/testing/trigger_scripts/perf_device_trigger_unittest.py index f6eb576b442..d40293e4c95 100755 --- a/chromium/testing/trigger_scripts/perf_device_trigger_unittest.py +++ b/chromium/testing/trigger_scripts/perf_device_trigger_unittest.py @@ -120,9 +120,13 @@ class UnitTest(unittest.TestCase): def test_shard_env_vars_and_bot_id(self): triggerer = self.basic_setup() self.assertTrue(self.list_contains_sublist( - triggerer._swarming_runs[0], ['--bot', 'build1'])) + triggerer._swarming_runs[0], ['id', 'build1'])) self.assertTrue(self.list_contains_sublist( - triggerer._swarming_runs[1], ['--bot', 'build2'])) + triggerer._swarming_runs[1], ['id', 'build2'])) + self.assertTrue(self.list_contains_sublist( + triggerer._swarming_runs[0], ['--tag', 'shard:0'])) + self.assertTrue(self.list_contains_sublist( + triggerer._swarming_runs[1], ['--tag', 'shard:1'])) self.assertTrue(self.list_contains_sublist( triggerer._swarming_runs[0], ['--env', 'GTEST_SHARD_INDEX', '0'])) self.assertTrue(self.list_contains_sublist( diff --git a/chromium/testing/variations/README.md b/chromium/testing/variations/README.md index 044bc07b8c6..3a644615b01 100644 --- a/chromium/testing/variations/README.md +++ b/chromium/testing/variations/README.md @@ -50,7 +50,7 @@ Each *study configuration* is a dictionary containing `platforms` and `experiments`. `platforms` is an array of strings, indicating the targetted platforms. The -strings may be `android`, `chromeos`, `ios`, `linux`, `mac`, or `windows`. +strings may be `android`, `chromeos`, `ios`, `linux`, `mac`, or `win`. `experiments` is an array containing the *experiments*. diff --git a/chromium/testing/variations/fieldtrial_testing_config.json b/chromium/testing/variations/fieldtrial_testing_config.json index 2bc941dff8a..0a957757631 100644 --- a/chromium/testing/variations/fieldtrial_testing_config.json +++ b/chromium/testing/variations/fieldtrial_testing_config.json @@ -1,23 +1,4 @@ { - "AbusiveExperienceEnforce": [ - { - "platforms": [ - "android", - "chromeos", - "linux", - "mac", - "win" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "AbusiveExperienceEnforce" - ] - } - ] - } - ], "AccountConsistencyVariations": [ { "platforms": [ @@ -27,9 +8,9 @@ ], "experiments": [ { - "name": "DiceFixAuthErrors", + "name": "DicePrepareMigration", "params": { - "method": "dice_fix_auth_errors" + "method": "dice_prepare_migration_new_endpoint" }, "enable_features": [ "AccountConsistency" @@ -88,79 +69,6 @@ ] } ], - "AndroidInProductHelpChromeHomeExpand": [ - { - "platforms": [ - "android" - ], - "experiments": [ - { - "name": "Tracking", - "params": { - "availability": ">=5", - "event_1": "name:bottom_sheet_expanded_from_button;comparator:<5; window:360;storage:360", - "event_2": "name:chrome_home_cold_start_iph_trigger;comparator:==0;window:30;storage:30", - "event_trigger": "name:chrome_home_cold_start_iph_trigger;comparator:<4;window:360;storage:360", - "event_used": "name:bottom_sheet_expanded_from_swipe;comparator:<5;window:360;storage:360", - "session_rate": "==0" - }, - "enable_features": [ - "IPH_ChromeHomeExpand" - ] - } - ] - } - ], - "AndroidInProductHelpChromeHomeMenuHeader": [ - { - "platforms": [ - "android" - ], - "experiments": [ - { - "name": "Tracking", - "params": { - "availability": "any", - "event_1": "name:bottom_sheet_expanded;comparator:<50;window:680;storage:680", - "event_3": "name:chrome_home_non_home_content_shown;comparator:<20;window:680;storage:680", - "event_trigger": "name:chrome_home_menu_header_iph_trigger;comparator:<100;window:680;storage:680", - "event_used": "name:chrome_home_menu_header_clicked;comparator:<20;window:680;storage:680", - "session_rate": "any", - "session_rate_impact": "none" - }, - "enable_features": [ - "IPH_ChromeHomeMenuHeader" - ] - } - ] - } - ], - "AndroidInProductHelpChromeHomePullToRefresh": [ - { - "platforms": [ - "android" - ], - "experiments": [ - { - "name": "Tracking", - "params": { - "availability": ">=5", - "event_1": "name:bottom_sheet_expanded_from_button;comparator:<5; window:360;storage:360", - "event_2": "name:chrome_home_pull_to_refresh_trigger;comparator:==0;window:30;storage:30", - "event_3": "name:pull_to_refresh;comparator:>3;window:30;storage:30", - "event_trigger": "name:chrome_home_pull_to_refresh_trigger;comparator:<4;window:360;storage:360", - "event_used": "name:bottom_sheet_expanded_from_swipe;comparator:<5;window:360;storage:360", - "session_rate": "any", - "x_iph-timeout-duration-ms": "5000" - }, - "enable_features": [ - "ChromeHomePersistentIph", - "IPH_ChromeHomePullToRefresh" - ] - } - ] - } - ], "AndroidInProductHelpContextualSearchPromotePanelOpen": [ { "platforms": [ @@ -168,9 +76,9 @@ ], "experiments": [ { - "name": "PromotePanelOpeningExperiment2", + "name": "PromotePanelOpeningExperiment", "params": { - "availability": ">=30", + "availability": ">=14", "event_1": "name:contextual_search_promote_panel_open_iph_trigger;comparator:<2;window:680;storage:680", "event_2": "name:contextual_search_panel_opened;comparator:<3;window:90;storage:90", "event_trigger": "name:contextual_search_promote_panel_open_iph_trigger;comparator:==0;window:90;storage:680", @@ -191,9 +99,9 @@ ], "experiments": [ { - "name": "PromoteTapExperiment2", + "name": "PromoteTapExperiment", "params": { - "availability": ">=30", + "availability": ">=14", "event_1": "name:contextual_search_panel_opened;comparator:>=2;window:90;storage:90", "event_2": "name:contextual_search_promote_tap_iph_trigger;comparator:<2;window:680;storage:680", "event_trigger": "name:contextual_search_promote_tap_iph_trigger;comparator:==0;window:90;storage:680", @@ -214,9 +122,9 @@ ], "experiments": [ { - "name": "WebSearchExperiment2", + "name": "WebSearchExperiment", "params": { - "availability": ">=30", + "availability": ">=14", "event_1": "name:web_search_performed;comparator:>=1;window:90;storage:90", "event_2": "name:contextual_search_web_search_iph_trigger;comparator:<2;window:680;storage:680", "event_3": "name:contextual_search_panel_opened;comparator:>0;window:680;storage:680", @@ -464,28 +372,6 @@ ] } ], - "AutofillGooglePayBranding": [ - { - "platforms": [ - "android", - "chromeos", - "ios", - "linux", - "mac", - "win" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "AutofillCreditCardDropdownGooglePayBranding", - "AutofillUpstreamUseGooglePayBranding", - "AutofillUseNewSettingsNameInDropdown" - ] - } - ] - } - ], "AutofillProfileCleanup": [ { "platforms": [ @@ -529,7 +415,7 @@ ] } ], - "AutofillUpstreamRequestCvcIfMissing": [ + "AutofillSingleClick": [ { "platforms": [ "chromeos", @@ -539,18 +425,14 @@ ], "experiments": [ { - "name": "EnabledWithGoogleBranding_Launched", - "enable_features": [ - "AutofillUpstreamRequestCvcIfMissing" - ] + "name": "Enabled" } ] } ], - "AutofillUpstreamSendDetectedValues": [ + "AutofillUpstreamRequestCvcIfMissing": [ { "platforms": [ - "android", "chromeos", "linux", "mac", @@ -558,9 +440,9 @@ ], "experiments": [ { - "name": "Enabled", + "name": "EnabledWithGoogleBranding_Launched", "enable_features": [ - "AutofillUpstreamSendDetectedValues" + "AutofillUpstreamRequestCvcIfMissing" ] } ] @@ -792,10 +674,9 @@ ] } ], - "CheckerImaging": [ + "CertificateTransparencyLogAuditing": [ { "platforms": [ - "android", "chromeos", "linux", "mac", @@ -803,22 +684,29 @@ ], "experiments": [ { - "name": "CheckerImaging", + "name": "Enabled", "enable_features": [ - "CheckerImaging" + "CertificateTransparencyLogAuditing" ] } ] } ], - "ChildAccountDetection": [ + "CheckerImaging": [ { "platforms": [ + "android", + "chromeos", + "linux", + "mac", "win" ], "experiments": [ { - "name": "Disabled" + "name": "CheckerImaging", + "enable_features": [ + "CheckerImaging" + ] } ] } @@ -841,52 +729,16 @@ ] } ], - "ChromeHomeBottomNavLabels": [ - { - "platforms": [ - "android" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "ChromeHomeBottomNavLabels" - ] - } - ] - } - ], - "ChromeHomeInactivitySheetExpansion": [ - { - "platforms": [ - "android" - ], - "experiments": [ - { - "name": "Enabled_3_Hours", - "params": { - "time_since_backgrounded_in_mins": "180" - }, - "enable_features": [ - "ChromeHomeInactivitySheetExpansion" - ] - } - ] - } - ], - "ChromeHomeSuggestionsMemoryReduction": [ + "ChromeModernDesign": [ { "platforms": [ "android" ], "experiments": [ { - "name": "DropAllButFirstThumbnail", + "name": "Modern", "enable_features": [ - "ChromeHomeDropAllButFirstThumbnail" - ], - "disable_features": [ - "ChromeHomeDestroySuggestions" + "ChromeModernDesign" ] } ] @@ -1252,41 +1104,6 @@ ] } ], - "DisallowFetchForDocWrittenScriptsInMainFrame": [ - { - "platforms": [ - "android", - "chromeos", - "linux", - "mac", - "win" - ], - "experiments": [ - { - "name": "DocumentWriteScriptBlockGroup_20161208_Launch", - "params": { - "disallowFetchForDocWrittenScriptsInMainFrame": "false", - "disallowFetchForDocWrittenScriptsInMainFrameOnSlowConnections": "true" - } - } - ] - } - ], - "DispatchSafetyNetCheckOffThread": [ - { - "platforms": [ - "android" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "DispatchSafetyNetCheckOffThread" - ] - } - ] - } - ], "DownloadHomeMoreButton": [ { "platforms": [ @@ -1367,6 +1184,24 @@ ] } ], + "EnableCastLocalMedia": [ + { + "platforms": [ + "linux", + "mac", + "win", + "chromeos" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "EnableCastLocalMedia" + ] + } + ] + } + ], "EnableCsrssLockdown": [ { "platforms": [ @@ -1520,6 +1355,25 @@ ] } ], + "GCMTokenInvalidAfterDays": [ + { + "platforms": [ + "linux", + "mac", + "win", + "chromeos", + "ios" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "GCMTokenInvalidAfterDays" + ] + } + ] + } + ], "GaiaPasswordReuse": [ { "platforms": [ @@ -1765,6 +1619,22 @@ ] } ], + "IncompatibleApplicationsWarning": [ + { + "platforms": [ + "win" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "IncompatibleApplicationsWarning", + "ModuleDatabase" + ] + } + ] + } + ], "InstanceID": [ { "platforms": [ @@ -1837,45 +1707,25 @@ ] } ], - "KeepAliveRendererForKeepaliveRequests": [ - { - "platforms": [ - "android", - "chromeos", - "linux", - "mac", - "win" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "KeepAliveRendererForKeepaliveRequests" - ] - } - ] - } - ], - "LazyParseCSS": [ + "LocalScreenCasting": [ { "platforms": [ "linux", "mac", "win", - "android", "chromeos" ], "experiments": [ { "name": "Enabled", "enable_features": [ - "LazyParseCSS" + "LocalScreenCasting" ] } ] } ], - "LowPriorityIframes": [ + "LowPriorityIframes2": [ { "platforms": [ "linux", @@ -1972,9 +1822,10 @@ ] } ], - "MojoCdm": [ + "ModernMediaControls": [ { "platforms": [ + "android", "chromeos", "linux", "mac", @@ -1984,7 +1835,7 @@ { "name": "Enabled", "enable_features": [ - "MojoCdm" + "UseModernMediaControls" ] } ] @@ -2024,6 +1875,40 @@ ] } ], + "NTPArticleSuggestionsExpandableHeader": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "NTPArticleSuggestionsExpandableHeader" + ] + } + ] + } + ], + "NTPArticleSuggestionsNowStream": [ + { + "platforms": [ + "android", + "ios" + ], + "experiments": [ + { + "name": "Enabled", + "params": { + "content_suggestions_backend": "https://chromefeedcontentsuggestions-pa.googleapis.com/v2/suggestions/fetch" + }, + "enable_features": [ + "NTPArticleSuggestions" + ] + } + ] + } + ], "NTPBreakingNewsPush": [ { "platforms": [ @@ -2172,6 +2057,25 @@ ] } ], + "NavigationMojoResponse": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "win" + ], + "experiments": [ + { + "name": "NavigationMojoResponse", + "enable_features": [ + "NavigationMojoResponse" + ] + } + ] + } + ], "NetAdaptiveProxyConnectionTimeout": [ { "platforms": [ @@ -2324,6 +2228,21 @@ ] } ], + "OfflinePagesDescriptivePendingStatus": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "OfflinePagesDescriptivePendingStatus" + ] + } + ] + } + ], "OfflinePagesPrefetchingSuggestions": [ { "platforms": [ @@ -2343,41 +2262,54 @@ "OmniboxBundledExperimentV1": [ { "platforms": [ - "android", "chromeos", - "ios", "linux", "mac", "win" ], "experiments": [ { - "name": "UIExperiments", + "name": "DesktopExperiments", "params": { "UIVerticalMargin": "8" }, "enable_features": [ + "OmniboxBreakWordsAtUnderscores", + "OmniboxDisplayTitleForCurrentUrl", + "OmniboxTailSuggestions", "OmniboxUIExperimentShowSuggestionFavicons", "OmniboxUIExperimentSwapTitleAndUrl", - "OmniboxUIExperimentVerticalMargin" + "OmniboxUIExperimentVerticalMargin", + "ZeroSuggestRedirectToChrome", + "ZeroSuggestSwapTitleAndUrl" ] } ] - } - ], - "OmniboxCombinedSpeculativeServiceWorker": [ + }, { "platforms": [ "android" ], "experiments": [ { - "name": "PrewarmingWithServiceWorker2", + "name": "AndroidExperiments", "enable_features": [ - "OmniboxSpeculativeServiceWorkerStartOnQueryInput" - ], - "disable_features": [ - "OmniboxSpareRenderer" + "OmniboxBreakWordsAtUnderscores", + "OmniboxDisplayTitleForCurrentUrl" + ] + } + ] + }, + { + "platforms": [ + "ios" + ], + "experiments": [ + { + "name": "iOSExperiments", + "enable_features": [ + "OmniboxBreakWordsAtUnderscores", + "OmniboxDisplayTitleForCurrentUrl" ] } ] @@ -2578,6 +2510,21 @@ ] } ], + "PasswordManagerSearchSettings": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "PasswordSearchEnabled", + "enable_features": [ + "PasswordSearchMobile" + ] + } + ] + } + ], "PasswordProtectionForEnterprise": [ { "platforms": [ @@ -2796,6 +2743,22 @@ ] } ], + "PreviewsNoScript": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "NoScriptWithWhiteList2", + "enable_features": [ + "NoScriptPreviews", + "OptimizationHints" + ] + } + ] + } + ], "PrintPdfAsImage": [ { "platforms": [ @@ -2946,6 +2909,25 @@ ] } ], + "RemoveNavigationHistory": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "win" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "RemoveNavigationHistory" + ] + } + ] + } + ], "RendererSchedulerWakeUpThrottling": [ { "platforms": [ @@ -3090,23 +3072,12 @@ ], "experiments": [ { - "name": "Enabled_bg_limit_8_4", - "params": { - "bg_limit": "8", - "bg_sub_limit": "4" - }, - "enable_features": [ - "ReportRendererPeakMemoryStats", - "ResourceLoadScheduler" - ] - }, - { - "name": "Enabled_bg_limit_8", + "name": "Enabled_bg_limit_3_1", "params": { - "bg_limit": "8" + "bg_limit": "3", + "bg_sub_limit": "1" }, "enable_features": [ - "ReportRendererPeakMemoryStats", "ResourceLoadScheduler" ] } @@ -3556,6 +3527,27 @@ ] } ], + "SimplifyHttpsIndicator": [ + { + "platforms": [ + "chromeos", + "linux", + "win", + "mac" + ], + "experiments": [ + { + "name": "EvToSecure", + "params": { + "treatment": "ev-to-secure" + }, + "enable_features": [ + "SimplifyHttpsIndicator" + ] + } + ] + } + ], "SocketReadIfReady": [ { "platforms": [ @@ -3653,33 +3645,6 @@ ] } ], - "SubresourceFilter": [ - { - "platforms": [ - "android", - "chromeos", - "linux", - "mac", - "win" - ], - "experiments": [ - { - "name": "LiveRun_BAS_Phishing", - "params": { - "enable_presets": "liverun_on_phishing_sites,liverun_on_better_ads_violating_sites" - }, - "enable_features": [ - "SubresourceFilter", - "SubresourceFilterExperimentalUI" - ] - }, - { - "name": "VerifyFeatureIsStillDisabledByDefault", - "forcing_flag": "suppress-enabling-subresource-filter-from-fieldtrial-testing-config" - } - ] - } - ], "SuperfishInterstitial": [ { "platforms": [ @@ -3715,23 +3680,6 @@ ] } ], - "TabSyncByRecency": [ - { - "platforms": [ - "android", - "chromeos", - "ios", - "linux", - "mac", - "win" - ], - "experiments": [ - { - "name": "Enabled" - } - ] - } - ], "ThrottleDelayable": [ { "platforms": [ @@ -3745,15 +3693,9 @@ { "name": "Enabled", "params": { - "EffectiveConnectionType1": "Slow-2G", - "EffectiveConnectionType2": "2G", - "EffectiveConnectionType3": "3G", - "MaxDelayableRequests1": "8", - "MaxDelayableRequests2": "8", - "MaxDelayableRequests3": "14", - "NonDelayableWeight1": "2.0", - "NonDelayableWeight2": "2.0", - "NonDelayableWeight3": "2.0" + "EffectiveConnectionType1": "3G", + "MaxDelayableRequests1": "14", + "NonDelayableWeight1": "2.0" }, "enable_features": [ "ThrottleDelayable" @@ -4039,6 +3981,24 @@ ] } ], + "UsePdfCompositorServiceForPrint": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "win" + ], + "experiments": [ + { + "name": "UsePdfCompositorServiceForPrint", + "enable_features": [ + "UsePdfCompositorServiceForPrint" + ] + } + ] + } + ], "UserActivityEventLogging": [ { "platforms": [ @@ -4146,6 +4106,26 @@ ] } ], + "VariationsHttpDisabled": [ + { + "platforms": [ + "win", + "mac", + "chromeos", + "linux", + "ios", + "android" + ], + "experiments": [ + { + "name": "VariationsHttpDisabled", + "disable_features": [ + "VariationsHttpRetry" + ] + } + ] + } + ], "VideoCaptureService": [ { "platforms": [ @@ -4164,6 +4144,25 @@ ] } ], + "VideoSurfaceLayer": [ + { + "platforms": [ + "linux", + "mac", + "win", + "chromeos", + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "UseSurfaceLayerForVideo" + ] + } + ] + } + ], "ViewsSimplifiedFullscreenUI": [ { "platforms": [ @@ -4342,6 +4341,21 @@ ] } ], + "WindowService-ChromeOS": [ + { + "platforms": [ + "chromeos" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "Mus" + ] + } + ] + } + ], "WorkStealingInScriptRunner": [ { "platforms": [ |