diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-12 14:27:29 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:35:20 +0000 |
commit | c30a6232df03e1efbd9f3b226777b07e087a1122 (patch) | |
tree | e992f45784689f373bcc38d1b79a239ebe17ee23 /chromium/testing | |
parent | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (diff) | |
download | qtwebengine-chromium-85-based.tar.gz |
BASELINE: Update Chromium to 85.0.4183.14085-based
Change-Id: Iaa42f4680837c57725b1344f108c0196741f6057
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/testing')
31 files changed, 1676 insertions, 1041 deletions
diff --git a/chromium/testing/OWNERS b/chromium/testing/OWNERS index 75f1d8956f6..6e673934653 100644 --- a/chromium/testing/OWNERS +++ b/chromium/testing/OWNERS @@ -1,4 +1,5 @@ dpranke@chromium.org +dpranke@google.com johnchen@chromium.org # These aren't actually great contact points for this directory, but diff --git a/chromium/testing/android/junit/BUILD.gn b/chromium/testing/android/junit/BUILD.gn index 3452c7c4b1b..52a408855da 100644 --- a/chromium/testing/android/junit/BUILD.gn +++ b/chromium/testing/android/junit/BUILD.gn @@ -27,9 +27,9 @@ java_library("junit_test_support") { "java/src/org/chromium/testing/local/TestDir.java", ] deps = [ + "//third_party/android_deps:robolectric_all_java", "//third_party/junit", "//third_party/mockito:mockito_java", - "//third_party/robolectric:robolectric_all_java", ] } diff --git a/chromium/testing/buildbot/filters/BUILD.gn b/chromium/testing/buildbot/filters/BUILD.gn index 7a28f3bb884..a3900d85849 100644 --- a/chromium/testing/buildbot/filters/BUILD.gn +++ b/chromium/testing/buildbot/filters/BUILD.gn @@ -47,7 +47,11 @@ source_set("browser_tests_filters") { data = [ "//testing/buildbot/filters/bfcache.browser_tests.filter", + "//testing/buildbot/filters/chromeos.msan.browser_tests.oobe_negative.filter", + "//testing/buildbot/filters/chromeos.msan.browser_tests.oobe_positive.filter", "//testing/buildbot/filters/code_coverage.browser_tests.filter", + "//testing/buildbot/filters/ozone-linux.wayland_browser_tests.filter", + "//testing/buildbot/filters/ozone-linux.x11_browser_tests.filter", "//testing/buildbot/filters/pixel_browser_tests.filter", "//testing/buildbot/filters/webrtc_functional.browser_tests.filter", ] @@ -65,7 +69,8 @@ source_set("chrome_public_test_apk_filters") { data = [ "//testing/buildbot/filters/bfcache.chrome_public_test_apk.filter", "//testing/buildbot/filters/android.pie_arm64_rel.chrome_public_test_apk.filter", - "//testing/buildbot/filters/android.emulator.chrome_public_test_apk.filter", + "//testing/buildbot/filters/android.emulator_m.chrome_public_test_apk.filter", + "//testing/buildbot/filters/android.emulator_p.chrome_public_test_apk.filter", ] } @@ -84,7 +89,8 @@ source_set("content_browsertests_filters") { data = [ "//testing/buildbot/filters/android.asan.content_browsertests.filter", - "//testing/buildbot/filters/android.emulator.content_browsertests.filter", + "//testing/buildbot/filters/android.emulator_m.content_browsertests.filter", + "//testing/buildbot/filters/android.emulator_p.content_browsertests.filter", "//testing/buildbot/filters/bfcache.content_browsertests.filter", "//testing/buildbot/filters/cast-linux.content_browsertests.filter", "//testing/buildbot/filters/gpu.skiarenderer_vulkan_content_browsertests.filter", @@ -105,7 +111,10 @@ source_set("content_junit_tests_filters") { source_set("content_shell_test_apk_filters") { testonly = true - data = [ "//testing/buildbot/filters/bfcache.content_shell_test_apk.filter" ] + data = [ + "//testing/buildbot/filters/bfcache.content_shell_test_apk.filter", + "//testing/buildbot/filters/android.emulator_m.content_shell_test_apk.filter", + ] } source_set("content_unittests_filters") { @@ -126,14 +135,17 @@ source_set("e2e_sync_integration_tests_filters") { source_set("fuchsia_filters") { data = [ "//testing/buildbot/filters/fuchsia.base_perftests.filter", + "//testing/buildbot/filters/fuchsia.blink_unittests.filter", "//testing/buildbot/filters/fuchsia.components_unittests.filter", "//testing/buildbot/filters/fuchsia.content_unittests.filter", + "//testing/buildbot/filters/fuchsia.headless_browsertests.filter", "//testing/buildbot/filters/fuchsia.mojo_unittests.filter", "//testing/buildbot/filters/fuchsia.net_perftests.filter", "//testing/buildbot/filters/fuchsia.net_unittests.filter", "//testing/buildbot/filters/fuchsia.services_unittests.filter", "//testing/buildbot/filters/fuchsia.storage_unittests.filter", "//testing/buildbot/filters/fuchsia.ui_base_unittests.filter", + "//testing/buildbot/filters/fuchsia.viz_unittests.filter", ] } @@ -164,8 +176,10 @@ source_set("linux_trusty_rel_browser_tests_filters") { } source_set("media_unittests_filters") { - data = - [ "//testing/buildbot/filters/android.emulator.media_unittests.filter" ] + data = [ + "//testing/buildbot/filters/android.emulator_m.media_unittests.filter", + "//testing/buildbot/filters/android.emulator_p.media_unittests.filter", + ] } source_set("net_unittests_filters") { @@ -194,7 +208,7 @@ source_set("viz_unittests_filters") { data = [ "//testing/buildbot/filters/android.emulator.viz_unittests.filter", - "//testing/buildbot/filters/gpu.skiarenderer_dawn_viz_unittests.filter", + "//testing/buildbot/filters/gpu.linux.skiarenderer_dawn_viz_unittests.filter", ] } diff --git a/chromium/testing/libfuzzer/README.md b/chromium/testing/libfuzzer/README.md index 29dc829775d..60bc0a96c16 100644 --- a/chromium/testing/libfuzzer/README.md +++ b/chromium/testing/libfuzzer/README.md @@ -36,6 +36,7 @@ Started Guide]. *** * [Reproducing bugs] found by libFuzzer/AFL and reported by ClusterFuzz. +* [Fuzzing mojo interfaces] using automatically generated libprotobuf-mutator fuzzers. ## Further Reading @@ -65,6 +66,7 @@ Started Guide]. [Creating a fuzz target that expects a protobuf]: libprotobuf-mutator.md [Detailed references]: reference.md [Fuzzing]: https://en.wikipedia.org/wiki/Fuzzing +[Fuzzing mojo interfaces]: ../../mojo/docs/mojolpm.md [Getting Started Guide]: getting_started.md [Guided in-process fuzzing of Chrome components]: https://security.googleblog.com/2016/08/guided-in-process-fuzzing-of-chrome.html [Improving fuzz target efficiency]: efficient_fuzzing.md diff --git a/chromium/testing/libfuzzer/fuzzer_test.gni b/chromium/testing/libfuzzer/fuzzer_test.gni index b46b653564a..9ea466b8692 100644 --- a/chromium/testing/libfuzzer/fuzzer_test.gni +++ b/chromium/testing/libfuzzer/fuzzer_test.gni @@ -63,6 +63,8 @@ template("fuzzer_test") { action(target_name + "_seed_corpus") { script = "//testing/libfuzzer/archive_corpus.py" + testonly = true + args = [ "--output", rebase_path(out, root_build_dir), diff --git a/chromium/testing/libfuzzer/proto/skia_image_filter.proto b/chromium/testing/libfuzzer/proto/skia_image_filter.proto index a495618439c..3448f551a57 100644 --- a/chromium/testing/libfuzzer/proto/skia_image_filter.proto +++ b/chromium/testing/libfuzzer/proto/skia_image_filter.proto @@ -350,7 +350,7 @@ message Paint { enum Style { kFill_Style = 0; kStroke_Style = 1; - kStrokeAndFill_Style = 2; + kStrokeAndFill_Style = 2; // no longer needed/used } enum StrokeCap { diff --git a/chromium/testing/libfuzzer/proto/skia_image_filter_proto_converter.cc b/chromium/testing/libfuzzer/proto/skia_image_filter_proto_converter.cc index f6fece43fa9..46814c562a7 100644 --- a/chromium/testing/libfuzzer/proto/skia_image_filter_proto_converter.cc +++ b/chromium/testing/libfuzzer/proto/skia_image_filter_proto_converter.cc @@ -949,8 +949,7 @@ void Converter::Visit(const Paint& paint) { if (stroke_style_used_) { style = Paint::kFill_Style; - } else if (style == Paint::kStrokeAndFill_Style || - style == Paint::kStroke_Style) { + } else if (style == Paint::kStroke_Style) { stroke_style_used_ = true; // Avoid timeouts. stroke_cap = Paint::kButt_Cap; diff --git a/chromium/testing/merge_scripts/code_coverage/merge_lib.py b/chromium/testing/merge_scripts/code_coverage/merge_lib.py index 92851121635..4b6cda35cc3 100644 --- a/chromium/testing/merge_scripts/code_coverage/merge_lib.py +++ b/chromium/testing/merge_scripts/code_coverage/merge_lib.py @@ -23,7 +23,6 @@ logging.basicConfig( def _call_profdata_tool(profile_input_file_paths, profile_output_file_path, profdata_tool_path, - retries=3, sparse=True): """Calls the llvm-profdata tool. @@ -59,34 +58,6 @@ def _call_profdata_tool(profile_input_file_paths, output = subprocess.check_output(subprocess_cmd, stderr=subprocess.STDOUT) logging.info('Merge succeeded with output: %r', output) except subprocess.CalledProcessError as error: - if len(profile_input_file_paths) > 1 and retries >= 0: - logging.warning('Merge failed with error output: %r', error.output) - - # The output of the llvm-profdata command will include the path of - # malformed files, such as - # `error: /.../default.profraw: Malformed instrumentation profile data` - invalid_profiles = [ - f for f in profile_input_file_paths if f in error.output - ] - - if not invalid_profiles: - logging.info( - 'Merge failed, but wasn\'t able to figure out the culprit invalid ' - 'profiles from the output, so skip retry and bail out.') - raise error - - valid_profiles = list( - set(profile_input_file_paths) - set(invalid_profiles)) - if valid_profiles: - logging.warning( - 'Following invalid profiles are removed as they were mentioned in ' - 'the merge error output: %r', invalid_profiles) - logging.info('Retry merging with the remaining profiles: %r', - valid_profiles) - return invalid_profiles + _call_profdata_tool( - valid_profiles, profile_output_file_path, profdata_tool_path, - retries - 1) - logging.error('Failed to merge profiles, return code (%d), output: %r' % (error.returncode, error.output)) raise error @@ -252,7 +223,8 @@ def merge_profiles(input_dir, input_extension, profdata_tool_path, input_filename_pattern='.*', - sparse=True): + sparse=True, + skip_validation=False): """Merges the profiles produced by the shards using llvm-profdata. Args: @@ -265,6 +237,8 @@ def merge_profiles(input_dir, a valid regex pattern if present. sparse (bool): flag to indicate whether to run llvm-profdata with --sparse. Doc: https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge + skip_validation (bool): flag to skip the _validate_and_convert_profraws + invocation. only applicable when input_extension is .profraw. Returns: The list of profiles that had to be excluded to get the merge to @@ -275,7 +249,12 @@ def merge_profiles(input_dir, input_filename_pattern) invalid_profraw_files = [] counter_overflows = [] - if input_extension == '.profraw': + + if skip_validation: + logging.warning('--skip-validation has been enabled. Skipping conversion ' + 'to ensure that profiles are valid.') + + if input_extension == '.profraw' and not skip_validation: profile_input_file_paths, invalid_profraw_files, counter_overflows = ( _validate_and_convert_profraws(profile_input_file_paths, profdata_tool_path, diff --git a/chromium/testing/merge_scripts/code_coverage/merge_lib_test.py b/chromium/testing/merge_scripts/code_coverage/merge_lib_test.py index 9da452e9c0c..97c1fbefb3a 100755 --- a/chromium/testing/merge_scripts/code_coverage/merge_lib_test.py +++ b/chromium/testing/merge_scripts/code_coverage/merge_lib_test.py @@ -8,13 +8,6 @@ import subprocess import sys import unittest -THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert( - 0, - os.path.abspath( - os.path.join(THIS_DIR, os.pardir, os.pardir, os.pardir, 'third_party', - 'pymock'))) - import mock import merge_lib as merger diff --git a/chromium/testing/merge_scripts/code_coverage/merge_results.py b/chromium/testing/merge_scripts/code_coverage/merge_results.py index d8ea2e098aa..199464b1dbc 100755 --- a/chromium/testing/merge_scripts/code_coverage/merge_results.py +++ b/chromium/testing/merge_scripts/code_coverage/merge_results.py @@ -72,6 +72,15 @@ def _MergeAPIArgumentParser(*args, **kwargs): action='store_true', dest='sparse', help='run llvm-profdata with the sparse flag.') + # (crbug.com/1091310) - IR PGO is incompatible with the initial conversion + # of .profraw -> .profdata that's run to detect validation errors. + # Introducing a bypass flag that'll merge all .profraw directly to .profdata + parser.add_argument( + '--skip-validation', + action='store_true', + help='skip validation for good raw profile data. this will pass all ' + 'raw profiles found to llvm-profdata to be merged. only applicable ' + 'when input extension is .profraw.') return parser @@ -106,7 +115,8 @@ def main(): params.task_output_dir, os.path.join(params.profdata_dir, output_prodata_filename), '.profraw', params.llvm_profdata, - sparse=params.sparse) + sparse=params.sparse, + skip_validation=params.skip_validation) # At the moment counter overflows overlap with invalid profiles, but this is # not guaranteed to remain the case indefinitely. To avoid future conflicts diff --git a/chromium/testing/merge_scripts/code_coverage/merge_results_test.py b/chromium/testing/merge_scripts/code_coverage/merge_results_test.py index 2af89a62881..8094582d8d1 100755 --- a/chromium/testing/merge_scripts/code_coverage/merge_results_test.py +++ b/chromium/testing/merge_scripts/code_coverage/merge_results_test.py @@ -10,11 +10,6 @@ import subprocess import sys import unittest -THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert( - 0, os.path.abspath(os.path.join(THIS_DIR, os.pardir, os.pardir, os.pardir, - 'third_party', 'pymock'))) - import mock import merge_results @@ -55,7 +50,8 @@ class MergeProfilesTest(unittest.TestCase): self.assertEqual( mock_merge.call_args, mock.call(task_output_dir, profdata_file, '.profraw', - 'llvm-profdata', sparse=True), None) + 'llvm-profdata', sparse=True, + skip_validation=False), None) def test_merge_steps_parameters(self): """Test the build-level merge front-end.""" @@ -123,6 +119,45 @@ class MergeProfilesTest(unittest.TestCase): self.assertTrue(mock_validate_and_convert_profraws.called) + @mock.patch.object(merger, '_validate_and_convert_profraws') + def test_profraw_skip_validation(self, mock_validate_and_convert_profraws): + mock_input_dir_walk = [ + ('/b/some/path', ['0', '1', '2', '3'], ['summary.json']), + ('/b/some/path/0', [], + ['output.json', 'default-1.profraw', 'default-2.profraw']), + ('/b/some/path/1', [], + ['output.json', 'default-1.profraw', 'default-2.profraw']), + ] + + with mock.patch.object(os, 'walk') as mock_walk: + with mock.patch.object(os, 'remove'): + mock_walk.return_value = mock_input_dir_walk + with mock.patch.object(subprocess, 'check_output') as mock_exec_cmd: + merger.merge_profiles('/b/some/path', + 'output/dir/default.profdata', + '.profraw', + 'llvm-profdata', + skip_validation=True) + self.assertEqual( + mock.call( + [ + 'llvm-profdata', + 'merge', + '-o', + 'output/dir/default.profdata', + '-sparse=true', + '/b/some/path/0/default-1.profraw', + '/b/some/path/0/default-2.profraw', + '/b/some/path/1/default-1.profraw', + '/b/some/path/1/default-2.profraw' + ], + stderr=-2, + ), mock_exec_cmd.call_args) + + # Skip validation should've passed all profraw files directly, and + # this validate call should not have been invoked. + self.assertFalse(mock_validate_and_convert_profraws.called) + def test_merge_profraw_skip_if_there_is_no_file(self): mock_input_dir_walk = [ @@ -206,50 +241,6 @@ class MergeProfilesTest(unittest.TestCase): # The mock method should only apply when merging .profraw files. self.assertFalse(mock_validate_and_convert_profraws.called) - def test_retry_profdata_merge_failures(self): - mock_input_dir_walk = [ - ('/b/some/path', ['0', '1'], ['summary.json']), - ('/b/some/path/0', [], - ['output.json', 'default-1.profdata', 'default-2.profdata']), - ('/b/some/path/1', [], - ['output.json', 'default-1.profdata', 'default-2.profdata']), - ] - with mock.patch.object(os, 'walk') as mock_walk: - with mock.patch.object(os, 'remove'): - mock_walk.return_value = mock_input_dir_walk - with mock.patch.object(subprocess, 'check_output') as mock_exec_cmd: - invalid_profiles_msg = ( - 'error: /b/some/path/0/default-1.profdata: Malformed ' - 'instrumentation profile data.') - - # Failed on the first merge, but succeed on the second attempt. - mock_exec_cmd.side_effect = [ - subprocess.CalledProcessError( - returncode=1, cmd='dummy cmd', output=invalid_profiles_msg), - None - ] - - merger.merge_profiles('/b/some/path', 'output/dir/default.profdata', - '.profdata', 'llvm-profdata') - - self.assertEqual(2, mock_exec_cmd.call_count) - - # Note that in the second call, /b/some/path/0/default-1.profdata is - # excluded! - self.assertEqual( - mock.call( - [ - 'llvm-profdata', - 'merge', - '-o', - 'output/dir/default.profdata', - '-sparse=true', - '/b/some/path/0/default-2.profdata', - '/b/some/path/1/default-1.profdata', - '/b/some/path/1/default-2.profdata', - ], - stderr=-2, - ), mock_exec_cmd.call_args) @mock.patch('os.remove') def test_mark_invalid_shards(self, mock_rm): @@ -380,7 +371,8 @@ class MergeProfilesTest(unittest.TestCase): self.assertEqual( mock_merge.call_args, mock.call(task_output_dir, profdata_file, '.profraw', - 'llvm-profdata', sparse=expected_outcome), None) + 'llvm-profdata', sparse=expected_outcome, + skip_validation=False), None) if __name__ == '__main__': diff --git a/chromium/testing/scripts/OWNERS b/chromium/testing/scripts/OWNERS index bccebb03dfb..5fe2f0fa096 100644 --- a/chromium/testing/scripts/OWNERS +++ b/chromium/testing/scripts/OWNERS @@ -2,14 +2,18 @@ # and understand the implications of changing these files. dpranke@chromium.org +dpranke@google.com kbr@chromium.org martiniss@chromium.org per-file check_static_initializers.py=thakis@chromium.org per-file check_static_initializers.py=thomasanderson@chromium.org -per-file run-wpt_tests.py=lpz@chromium.org +per-file run_wpt_tests.py=lpz@chromium.org per-file run_wpt_tests.py=robertma@chromium.org +per-file wpt_common.py=lpz@chromium.org +per-file wpt_common.py=robertma@chromium.org +per-file wpt_common.py=rmhasan@chromium.org per-file run_performance_tests.py=johnchen@chromium.org per-file run_performance_tests.py=wenbinzhang@google.com diff --git a/chromium/testing/scripts/get_compile_targets.py b/chromium/testing/scripts/get_compile_targets.py index aed1c569e8a..1e5fcf02a96 100755 --- a/chromium/testing/scripts/get_compile_targets.py +++ b/chromium/testing/scripts/get_compile_targets.py @@ -30,7 +30,8 @@ def main(argv): if filename in ('common.py', 'get_compile_targets.py', 'gpu_integration_test_adapter.py', - 'sizes_common.py'): + 'sizes_common.py', + 'wpt_common.py'): continue with common.temporary_file() as tempfile_path: diff --git a/chromium/testing/scripts/representative_perf_test_data/representatives_frame_times_upper_limit.json b/chromium/testing/scripts/representative_perf_test_data/representatives_frame_times_upper_limit.json index 5b370b87b67..8f6d3038f14 100644 --- a/chromium/testing/scripts/representative_perf_test_data/representatives_frame_times_upper_limit.json +++ b/chromium/testing/scripts/representative_perf_test_data/representatives_frame_times_upper_limit.json @@ -6,7 +6,9 @@ }, "filter_terrain_svg": { "ci_095": 0.377, - "avg": 31.486 + "avg": 31.486, + "experimental": true, + "_comment": "crbug.com/1053614" }, "web_animation_value_type_transform_complex": { "ci_095": 1.783, @@ -55,7 +57,9 @@ }, "css_value_type_shadow": { "ci_095": 1.284, - "avg": 49.386 + "avg": 49.386, + "experimental": true, + "_comment": "crbug.com/1093313" }, "nvidia_vertex_buffer_object": { "ci_095": 3.37, @@ -85,7 +89,9 @@ }, "css_value_type_shadow": { "ci_095": 13.611, - "avg": 61.261 + "avg": 61.261, + "experimental": true, + "_comment": "crbug.com/1093313" }, "animometer_webgl_attrib_arrays": { "ci_095": 0.495, diff --git a/chromium/testing/scripts/run_android_wpt.py b/chromium/testing/scripts/run_android_wpt.py index e657f092f2d..7950d391f79 100755 --- a/chromium/testing/scripts/run_android_wpt.py +++ b/chromium/testing/scripts/run_android_wpt.py @@ -33,6 +33,7 @@ import shutil import sys import common +import wpt_common logger = logging.getLogger(__name__) @@ -40,37 +41,35 @@ SRC_DIR = os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) BUILD_ANDROID = os.path.join(SRC_DIR, 'build', 'android') +BLINK_TOOLS_DIR = os.path.join( + SRC_DIR, 'third_party', 'blink', 'tools') +CATAPULT_DIR = os.path.join(SRC_DIR, 'third_party', 'catapult') +DEFAULT_WPT = os.path.join(wpt_common.WEB_TESTS_DIR, 'external', 'wpt', 'wpt') +PYUTILS = os.path.join(CATAPULT_DIR, 'common', 'py_utils') + +if PYUTILS not in sys.path: + sys.path.append(PYUTILS) + +if BLINK_TOOLS_DIR not in sys.path: + sys.path.append(BLINK_TOOLS_DIR) if BUILD_ANDROID not in sys.path: sys.path.append(BUILD_ANDROID) import devil_chromium +from blinkpy.web_tests.port.android import ( + PRODUCTS, PRODUCTS_TO_EXPECTATION_FILE_PATHS, ANDROID_WEBLAYER, + ANDROID_WEBVIEW, CHROME_ANDROID, ANDROID_DISABLED_TESTS) + from devil import devil_env from devil.android import apk_helper from devil.android import device_utils from devil.android.tools import system_app from devil.android.tools import webview_app -CATAPULT_DIR = os.path.join(SRC_DIR, 'third_party', 'catapult') -PYUTILS = os.path.join(CATAPULT_DIR, 'common', 'py_utils') - -if PYUTILS not in sys.path: - sys.path.append(PYUTILS) - from py_utils.tempfile_ext import NamedTemporaryDirectory -BLINK_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'blink', 'tools') -WEB_TESTS_DIR = os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests') -DEFAULT_WPT = os.path.join(WEB_TESTS_DIR, 'external', 'wpt', 'wpt') - -ANDROID_WEBLAYER = 'android_weblayer' -ANDROID_WEBVIEW = 'android_webview' -CHROME_ANDROID = 'chrome_android' - -# List of supported products. -PRODUCTS = [ANDROID_WEBLAYER, ANDROID_WEBVIEW, CHROME_ANDROID] - class PassThroughArgs(argparse.Action): pass_through_args = [] @@ -107,28 +106,18 @@ def _get_adapter(device): return WPTClankAdapter(device) -class WPTAndroidAdapter(common.BaseIsolatedScriptArgsAdapter): +class WPTAndroidAdapter(wpt_common.BaseWptScriptAdapter): def __init__(self, device): self.pass_through_wpt_args = [] self.pass_through_binary_args = [] self._metadata_dir = None - self._test_apk = None - self._missing_test_apk_arg = None self._device = device super(WPTAndroidAdapter, self).__init__() # Arguments from add_extra_argumentsparse were added so # its safe to parse the arguments and set self._options self.parse_args() - def generate_test_output_args(self, output): - return ['--log-chromium', output] - - def generate_sharding_args(self, total_shards, shard_index): - return ['--total-chunks=%d' % total_shards, - # shard_index is 0-based but WPT's this-chunk to be 1-based - '--this-chunk=%d' % (shard_index + 1)] - @property def rest_args(self): rest_args = super(WPTAndroidAdapter, self).rest_args @@ -169,29 +158,25 @@ class WPTAndroidAdapter(common.BaseIsolatedScriptArgsAdapter): raise NotImplementedError def _maybe_build_metadata(self): - if not self._test_apk: - assert self._missing_test_apk_arg, ( - 'self._missing_test_apk_arg was not set.') - logger.warning('%s was not set, skipping metadata generation.' % - self._missing_test_apk_arg) - return - metadata_builder_cmd = [ sys.executable, - os.path.join(BLINK_TOOLS_DIR, 'build_wpt_metadata.py'), - '--android-apk', - self._test_apk, + os.path.join(wpt_common.BLINK_TOOLS_DIR, 'build_wpt_metadata.py'), + '--android-product', + self.options.product, + '--ignore-default-expectations', '--metadata-output-dir', self._metadata_dir, '--additional-expectations', - os.path.join(WEB_TESTS_DIR, 'android', 'AndroidWPTNeverFixTests') + ANDROID_DISABLED_TESTS, ] metadata_builder_cmd.extend(self._extra_metadata_builder_args()) - common.run_command(metadata_builder_cmd) + return common.run_command(metadata_builder_cmd) def run_test(self): with NamedTemporaryDirectory() as self._metadata_dir, self._install_apks(): - self._maybe_build_metadata() + metadata_command_ret = self._maybe_build_metadata() + if metadata_command_ret != 0: + return metadata_command_ret return super(WPTAndroidAdapter, self).run_test() def _install_apks(self): @@ -202,24 +187,6 @@ class WPTAndroidAdapter(common.BaseIsolatedScriptArgsAdapter): # which was deleted self._metadata_dir = None - def do_post_test_run_tasks(self): - # Move json results into layout-test-results directory - results_dir = os.path.dirname(self.options.isolated_script_test_output) - layout_test_results = os.path.join(results_dir, 'layout-test-results') - os.mkdir(layout_test_results) - shutil.copyfile(self.options.isolated_script_test_output, - os.path.join(layout_test_results, 'full_results.json')) - # create full_results_jsonp.js file which is used to - # load results into the results viewer - with open(self.options.isolated_script_test_output, 'r') as full_results, \ - open(os.path.join( - layout_test_results, 'full_results_jsonp.js'), 'w') as json_js: - json_js.write('ADD_FULL_RESULTS(%s);' % full_results.read()) - # copy layout test results viewer to layout-test-results directory - shutil.copyfile( - os.path.join(WEB_TESTS_DIR, 'fast', 'harness', 'results.html'), - os.path.join(layout_test_results, 'results.html')) - def add_extra_arguments(self, parser): # TODO: |pass_through_args| are broke and need to be supplied by way of # --binary-arg". @@ -282,11 +249,6 @@ class WPTWeblayerAdapter(WPTAndroidAdapter): WEBLAYER_SHELL_PKG = 'org.chromium.weblayer.shell' WEBLAYER_SUPPORT_PKG = 'org.chromium.weblayer.support' - def __init__(self, device): - super(WPTWeblayerAdapter, self).__init__(device) - self._test_apk = self.options.weblayer_shell - self._missing_test_apk_arg = '--weblayer-shell' - @contextlib.contextmanager def _install_apks(self): install_weblayer_shell_as_needed = maybe_install_user_apk( @@ -304,8 +266,7 @@ class WPTWeblayerAdapter(WPTAndroidAdapter): def _extra_metadata_builder_args(self): return [ '--additional-expectations', - os.path.join(WEB_TESTS_DIR, - 'android', 'WeblayerWPTOverrideExpectations')] + PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBLAYER]] def add_extra_arguments(self, parser): super(WPTWeblayerAdapter, self).add_extra_arguments(parser) @@ -327,11 +288,6 @@ class WPTWebviewAdapter(WPTAndroidAdapter): SYSTEM_WEBVIEW_SHELL_PKG = 'org.chromium.webview_shell' - def __init__(self, device): - super(WPTWebviewAdapter, self).__init__(device) - self._test_apk = self.options.system_webview_shell - self._missing_test_apk_arg = '--system-webview-shell' - @contextlib.contextmanager def _install_apks(self): install_shell_as_needed = maybe_install_user_apk( @@ -346,8 +302,7 @@ class WPTWebviewAdapter(WPTAndroidAdapter): def _extra_metadata_builder_args(self): return [ '--additional-expectations', - os.path.join( - WEB_TESTS_DIR, 'android', 'WebviewWPTOverrideExpectations')] + PRODUCTS_TO_EXPECTATION_FILE_PATHS[ANDROID_WEBVIEW]] def add_extra_arguments(self, parser): super(WPTWebviewAdapter, self).add_extra_arguments(parser) @@ -366,11 +321,6 @@ class WPTWebviewAdapter(WPTAndroidAdapter): class WPTClankAdapter(WPTAndroidAdapter): - def __init__(self, device): - super(WPTClankAdapter, self).__init__(device) - self._test_apk = self.options.chrome_apk - self._missing_test_apk_arg = '--chrome-apk' - @contextlib.contextmanager def _install_apks(self): install_clank_as_needed = maybe_install_user_apk( @@ -381,7 +331,7 @@ class WPTClankAdapter(WPTAndroidAdapter): def _extra_metadata_builder_args(self): return [ '--additional-expectations', - os.path.join(WEB_TESTS_DIR, 'android', 'ClankWPTOverrideExpectations')] + PRODUCTS_TO_EXPECTATION_FILE_PATHS[CHROME_ANDROID]] def add_extra_arguments(self, parser): super(WPTClankAdapter, self).add_extra_arguments(parser) diff --git a/chromium/testing/scripts/run_android_wpt.pydeps b/chromium/testing/scripts/run_android_wpt.pydeps index 3f75fc0f319..06416555afa 100644 --- a/chromium/testing/scripts/run_android_wpt.pydeps +++ b/chromium/testing/scripts/run_android_wpt.pydeps @@ -6,8 +6,66 @@ //build/android/pylib/constants/host_paths.py //testing/scripts/common.py //testing/scripts/run_android_wpt.py +//testing/scripts/wpt_common.py //testing/test_env.py //testing/xvfb.py +//third_party/blink/tools/blinkpy/__init__.py +//third_party/blink/tools/blinkpy/common/__init__.py +//third_party/blink/tools/blinkpy/common/checkout/__init__.py +//third_party/blink/tools/blinkpy/common/checkout/git.py +//third_party/blink/tools/blinkpy/common/exit_codes.py +//third_party/blink/tools/blinkpy/common/find_files.py +//third_party/blink/tools/blinkpy/common/host.py +//third_party/blink/tools/blinkpy/common/html_diff.py +//third_party/blink/tools/blinkpy/common/memoized.py +//third_party/blink/tools/blinkpy/common/net/__init__.py +//third_party/blink/tools/blinkpy/common/net/network_transaction.py +//third_party/blink/tools/blinkpy/common/net/results_fetcher.py +//third_party/blink/tools/blinkpy/common/net/web.py +//third_party/blink/tools/blinkpy/common/net/web_test_results.py +//third_party/blink/tools/blinkpy/common/path_finder.py +//third_party/blink/tools/blinkpy/common/read_checksum_from_png.py +//third_party/blink/tools/blinkpy/common/system/__init__.py +//third_party/blink/tools/blinkpy/common/system/executive.py +//third_party/blink/tools/blinkpy/common/system/filesystem.py +//third_party/blink/tools/blinkpy/common/system/path.py +//third_party/blink/tools/blinkpy/common/system/platform_info.py +//third_party/blink/tools/blinkpy/common/system/profiler.py +//third_party/blink/tools/blinkpy/common/system/system_host.py +//third_party/blink/tools/blinkpy/common/system/user.py +//third_party/blink/tools/blinkpy/common/unified_diff.py +//third_party/blink/tools/blinkpy/w3c/__init__.py +//third_party/blink/tools/blinkpy/w3c/wpt_manifest.py +//third_party/blink/tools/blinkpy/web_tests/__init__.py +//third_party/blink/tools/blinkpy/web_tests/breakpad/__init__.py +//third_party/blink/tools/blinkpy/web_tests/breakpad/dump_reader.py +//third_party/blink/tools/blinkpy/web_tests/breakpad/dump_reader_multipart.py +//third_party/blink/tools/blinkpy/web_tests/breakpad/dump_reader_win.py +//third_party/blink/tools/blinkpy/web_tests/builder_list.py +//third_party/blink/tools/blinkpy/web_tests/controllers/__init__.py +//third_party/blink/tools/blinkpy/web_tests/controllers/repaint_overlay.py +//third_party/blink/tools/blinkpy/web_tests/layout_package/__init__.py +//third_party/blink/tools/blinkpy/web_tests/layout_package/bot_test_expectations.py +//third_party/blink/tools/blinkpy/web_tests/layout_package/json_results_generator.py +//third_party/blink/tools/blinkpy/web_tests/models/__init__.py +//third_party/blink/tools/blinkpy/web_tests/models/test_configuration.py +//third_party/blink/tools/blinkpy/web_tests/models/test_expectations.py +//third_party/blink/tools/blinkpy/web_tests/models/test_failures.py +//third_party/blink/tools/blinkpy/web_tests/models/test_run_results.py +//third_party/blink/tools/blinkpy/web_tests/models/typ_types.py +//third_party/blink/tools/blinkpy/web_tests/port/__init__.py +//third_party/blink/tools/blinkpy/web_tests/port/android.py +//third_party/blink/tools/blinkpy/web_tests/port/base.py +//third_party/blink/tools/blinkpy/web_tests/port/driver.py +//third_party/blink/tools/blinkpy/web_tests/port/factory.py +//third_party/blink/tools/blinkpy/web_tests/port/linux.py +//third_party/blink/tools/blinkpy/web_tests/port/server_process.py +//third_party/blink/tools/blinkpy/web_tests/port/win.py +//third_party/blink/tools/blinkpy/web_tests/servers/__init__.py +//third_party/blink/tools/blinkpy/web_tests/servers/apache_http.py +//third_party/blink/tools/blinkpy/web_tests/servers/pywebsocket.py +//third_party/blink/tools/blinkpy/web_tests/servers/server_base.py +//third_party/blink/tools/blinkpy/web_tests/servers/wptserve.py //third_party/catapult/common/py_utils/py_utils/__init__.py //third_party/catapult/common/py_utils/py_utils/cloud_storage.py //third_party/catapult/common/py_utils/py_utils/cloud_storage_global_lock.py @@ -30,7 +88,7 @@ //third_party/catapult/devil/devil/android/constants/chrome.py //third_party/catapult/devil/devil/android/constants/file_system.py //third_party/catapult/devil/devil/android/decorators.py -//third_party/catapult/devil/devil/android/device_blacklist.py +//third_party/catapult/devil/devil/android/device_denylist.py //third_party/catapult/devil/devil/android/device_errors.py //third_party/catapult/devil/devil/android/device_signal.py //third_party/catapult/devil/devil/android/device_temp_file.py @@ -71,4 +129,19 @@ //third_party/catapult/devil/devil/utils/timeout_retry.py //third_party/catapult/devil/devil/utils/watchdog_timer.py //third_party/catapult/devil/devil/utils/zip_utils.py +//third_party/catapult/third_party/typ/typ/__init__.py +//third_party/catapult/third_party/typ/typ/arg_parser.py +//third_party/catapult/third_party/typ/typ/artifacts.py +//third_party/catapult/third_party/typ/typ/expectations_parser.py +//third_party/catapult/third_party/typ/typ/fakes/__init__.py +//third_party/catapult/third_party/typ/typ/fakes/host_fake.py +//third_party/catapult/third_party/typ/typ/host.py +//third_party/catapult/third_party/typ/typ/json_results.py +//third_party/catapult/third_party/typ/typ/pool.py +//third_party/catapult/third_party/typ/typ/printer.py +//third_party/catapult/third_party/typ/typ/python_2_3_compat.py +//third_party/catapult/third_party/typ/typ/runner.py +//third_party/catapult/third_party/typ/typ/stats.py +//third_party/catapult/third_party/typ/typ/test_case.py +//third_party/catapult/third_party/typ/typ/version.py //third_party/catapult/third_party/zipfile/zipfile_2_7_13.py diff --git a/chromium/testing/scripts/run_rendering_benchmark_with_gated_performance.py b/chromium/testing/scripts/run_rendering_benchmark_with_gated_performance.py index db8852291f1..4c58b74944b 100755 --- a/chromium/testing/scripts/run_rendering_benchmark_with_gated_performance.py +++ b/chromium/testing/scripts/run_rendering_benchmark_with_gated_performance.py @@ -124,12 +124,19 @@ class RenderingRepresentativePerfTest(object): self.benchmark = self.options.benchmarks out_dir_path = os.path.dirname(self.options.isolated_script_test_output) - self.output_path = os.path.join( - out_dir_path, self.benchmark, 'test_results.json') - self.results_path = os.path.join( - out_dir_path, self.benchmark, 'perf_results.csv') - re_run_output_dir = os.path.join(out_dir_path, 're_run_failures') + + self.output_path = { + True: os.path.join( + re_run_output_dir, self.benchmark, 'test_results.json'), + False: os.path.join(out_dir_path, self.benchmark, 'test_results.json') + } + self.results_path = { + True: os.path.join( + re_run_output_dir, self.benchmark, 'perf_results.csv'), + False: os.path.join(out_dir_path, self.benchmark, 'perf_results.csv') + } + re_run_test_output = os.path.join(re_run_output_dir, os.path.basename(self.options.isolated_script_test_output)) re_run_test_perf_output = os.path.join(re_run_output_dir, @@ -236,37 +243,38 @@ class RenderingRepresentativePerfTest(object): METRIC_NAME, measured_avg, upper_limit_avg)) def interpret_run_benchmark_results(self, rerun=False): - with open(self.output_path, 'r+') as resultsFile: + with open(self.output_path[rerun], 'r+') as resultsFile: initialOut = json.load(resultsFile) self.result_recorder[rerun].set_tests(initialOut) - with open(self.results_path) as csv_file: + with open(self.results_path[rerun]) as csv_file: csv_obj = csv.DictReader(csv_file) values_per_story = self.parse_csv_results(csv_obj) - # Clearing the result of run_benchmark and write the gated perf results - resultsFile.seek(0) - resultsFile.truncate(0) + if not rerun: + # Clearing the result of run_benchmark and write the gated perf results + resultsFile.seek(0) + resultsFile.truncate(0) self.compare_values(values_per_story, rerun) - def run_perf_tests(self, rerun=False): - self.return_code |= run_performance_tests.main( - self.re_run_args if rerun else self.args) - self.interpret_run_benchmark_results(rerun) + def run_perf_tests(self): + self.return_code |= run_performance_tests.main(self.args) + self.interpret_run_benchmark_results(False) - if not rerun and len(self.result_recorder[rerun].failed_stories) > 0: + if len(self.result_recorder[False].failed_stories) > 0: # For failed stories we run_tests again to make sure it's not a false # positive. print('============ Re_run the failed tests ============') all_failed_stories = '('+'|'.join( - self.result_recorder[rerun].failed_stories)+')' + self.result_recorder[False].failed_stories)+')' # TODO(crbug.com/1055893): Remove the extra chrome categories after # investigation of flakes in representative perf tests. self.re_run_args.extend( ['--story-filter', all_failed_stories, '--pageset-repeat=3', '--extra-chrome-categories=blink,blink_gc,gpu,v8,viz']) - self.run_perf_tests(True) + self.return_code |= run_performance_tests.main(self.re_run_args) + self.interpret_run_benchmark_results(True) for story_name in self.result_recorder[False].failed_stories.copy(): if story_name not in self.result_recorder[True].failed_stories: @@ -283,7 +291,7 @@ class RenderingRepresentativePerfTest(object): self.return_code ) = self.result_recorder[False].get_output(self.return_code) - with open(self.output_path, 'r+') as resultsFile: + with open(self.output_path[False], 'r+') as resultsFile: json.dump(finalOut, resultsFile, indent=4) with open(self.options.isolated_script_test_output, 'w') as outputFile: json.dump(finalOut, outputFile, indent=4) diff --git a/chromium/testing/scripts/run_wpt_tests.py b/chromium/testing/scripts/run_wpt_tests.py index 86ca893abf2..b777ba189b7 100755 --- a/chromium/testing/scripts/run_wpt_tests.py +++ b/chromium/testing/scripts/run_wpt_tests.py @@ -16,26 +16,16 @@ Here's the mapping [isolate script flag] : [wpt flag] import json import os -import shutil import sys import common +import wpt_common -BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools') -WEB_TESTS_DIR = os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests') WPT_METADATA_DIR = "../../wpt_expectations_metadata/" WPT_OVERRIDE_EXPECTATIONS_PATH = ( "../../third_party/blink/web_tests/WPTOverrideExpectations") -class WPTTestAdapter(common.BaseIsolatedScriptArgsAdapter): - - def generate_test_output_args(self, output): - return ['--log-chromium', output] - - def generate_sharding_args(self, total_shards, shard_index): - return ['--total-chunks=%d' % total_shards, - # shard_index is 0-based but WPT's this-chunk to be 1-based - '--this-chunk=%d' % (shard_index + 1)] +class WPTTestAdapter(wpt_common.BaseWptScriptAdapter): @property def rest_args(self): @@ -88,33 +78,11 @@ class WPTTestAdapter(common.BaseIsolatedScriptArgsAdapter): ]) return rest_args - def do_post_test_run_tasks(self): - # Move json results into layout-test-results directory - results_dir = os.path.dirname(self.options.isolated_script_test_output) - layout_test_results = os.path.join(results_dir, 'layout-test-results') - if os.path.exists(layout_test_results): - shutil.rmtree(layout_test_results) - os.mkdir(layout_test_results) - shutil.copyfile(self.options.isolated_script_test_output, - os.path.join(layout_test_results, 'full_results.json')) - # create full_results_jsonp.js file which is used to - # load results into the results viewer - with open(self.options.isolated_script_test_output, 'r') \ - as full_results, \ - open(os.path.join( - layout_test_results, 'full_results_jsonp.js'), 'w') \ - as json_js: - json_js.write('ADD_FULL_RESULTS(%s);' % full_results.read()) - # copy layout test results viewer to layout-test-results directory - shutil.copyfile( - os.path.join(WEB_TESTS_DIR, 'fast', 'harness', 'results.html'), - os.path.join(layout_test_results, 'results.html')) - def main(): # First, generate WPT metadata files. common.run_command([ sys.executable, - os.path.join(BLINK_TOOLS_DIR, 'build_wpt_metadata.py'), + os.path.join(wpt_common.BLINK_TOOLS_DIR, 'build_wpt_metadata.py'), "--metadata-output-dir", WPT_METADATA_DIR, "--additional-expectations", diff --git a/chromium/testing/scripts/wpt_common.py b/chromium/testing/scripts/wpt_common.py new file mode 100644 index 00000000000..ad1ca0eb0db --- /dev/null +++ b/chromium/testing/scripts/wpt_common.py @@ -0,0 +1,205 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import base64 +import json +import os +import shutil +import sys + +import common + +BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools') +WEB_TESTS_DIR = os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests') + +if BLINK_TOOLS_DIR not in sys.path: + sys.path.append(BLINK_TOOLS_DIR) + +from blinkpy.common.host import Host +from blinkpy.web_tests.models import test_failures + +class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter): + """The base class for script adapters that use wptrunner to execute web + platform tests. This contains any code shared between these scripts, such + as integrating output with the results viewer. Subclasses contain other + (usually platform-specific) logic.""" + + def __init__(self): + super(BaseWptScriptAdapter, self).__init__() + host = Host() + self.port = host.port_factory.get() + + def generate_test_output_args(self, output): + return ['--log-chromium', output] + + def generate_sharding_args(self, total_shards, shard_index): + return ['--total-chunks=%d' % total_shards, + # shard_index is 0-based but WPT's this-chunk to be 1-based + '--this-chunk=%d' % (shard_index + 1)] + + def do_post_test_run_tasks(self): + # Move json results into layout-test-results directory + results_dir = os.path.dirname(self.options.isolated_script_test_output) + layout_test_results = os.path.join(results_dir, 'layout-test-results') + if os.path.exists(layout_test_results): + shutil.rmtree(layout_test_results) + os.mkdir(layout_test_results) + + # Perform post-processing of wptrunner output + self.process_wptrunner_output() + + shutil.copyfile(self.options.isolated_script_test_output, + os.path.join(layout_test_results, 'full_results.json')) + # create full_results_jsonp.js file which is used to + # load results into the results viewer + with open(self.options.isolated_script_test_output, 'r') \ + as full_results, \ + open(os.path.join( + layout_test_results, 'full_results_jsonp.js'), 'w') \ + as json_js: + json_js.write('ADD_FULL_RESULTS(%s);' % full_results.read()) + # copy layout test results viewer to layout-test-results directory + shutil.copyfile( + os.path.join(WEB_TESTS_DIR, 'fast', 'harness', 'results.html'), + os.path.join(layout_test_results, 'results.html')) + + def process_wptrunner_output(self): + """Post-process the output generated by wptrunner. + + This output contains a single large json file containing the raw content + or artifacts which need to be extracted into their own files and removed + from the json file (to avoid duplication).""" + output_json = json.load( + open(self.options.isolated_script_test_output, "r")) + test_json = output_json["tests"] + results_dir = os.path.dirname(self.options.isolated_script_test_output) + self._process_test_leaves(results_dir, output_json["path_delimiter"], + test_json, "") + # Write output_json back to the same file after modifying it in memory + with open(self.options.isolated_script_test_output, "w") as output_file: + json.dump(output_json, output_file) + + def _process_test_leaves(self, results_dir, delim, root_node, path_so_far): + """Finds and processes each test leaf below the specified root. + + This will recursively traverse the trie of results in the json output, + keeping track of the path to each test and identifying leaves by the + presence of certain attributes. + + Args: + results_dir: str path to the dir that results are stored + delim: str delimiter to be used for test names + root_node: dict representing the root of the trie we're currently + looking at + path_so_far: str the path to the current root_node in the trie + """ + if "actual" in root_node: + # Found a leaf, process it + if "artifacts" not in root_node: + return + log_artifact = root_node["artifacts"].pop("log", None) + if log_artifact: + artifact_subpath = self._write_log_artifact( + results_dir, path_so_far, log_artifact) + root_node["artifacts"]["actual_text"] = [artifact_subpath] + + screenshot_artifact = root_node["artifacts"].pop("screenshots", + None) + if screenshot_artifact: + screenshot_paths_dict = self._write_screenshot_artifact( + results_dir, path_so_far, screenshot_artifact) + for screenshot_key, path in screenshot_paths_dict.items(): + root_node["artifacts"][screenshot_key] = [path] + + return + + # We're not at a leaf node, continue traversing the trie. + for key in root_node: + # Append the key to the path, separated by the delimiter. However if + # the path is empty, skip the delimiter to avoid a leading slash in + # the path. + new_path = path_so_far + delim + key if path_so_far else key + self._process_test_leaves(results_dir, delim, root_node[key], + new_path) + + def _write_log_artifact(self, results_dir, test_name, log_artifact): + """Writes a log artifact to disk. + + The log artifact contains all the output of a test. It gets written to + the -actual.txt file for the test. + + Args: + results_dir: str path to the directory that results live in + test_name: str name of the test that this artifact is for + log_artifact: list of strings, the log entries for this test from + the json output. + + Returns: + string path to the artifact file that the log was written to, + relative to the directory that the original output is located. + """ + log_artifact_sub_path = ( + os.path.join("layout-test-results", + self.port.output_filename( + test_name, test_failures.FILENAME_SUFFIX_ACTUAL, + ".txt")) + ) + log_artifact_full_path = os.path.join(results_dir, + log_artifact_sub_path) + if not os.path.exists(os.path.dirname(log_artifact_full_path)): + os.makedirs(os.path.dirname(log_artifact_full_path)) + with open(log_artifact_full_path, "w") as artifact_file: + artifact_file.write("\n".join(log_artifact).encode("utf-8")) + + return log_artifact_sub_path + + def _write_screenshot_artifact(self, results_dir, test_name, + screenshot_artifact): + """Write screenshot artifact to disk. + + The screenshot artifact is a list of strings, each of which has the + format <url>:<base64-encoded PNG>. Each url-png pair is a screenshot of + either the test, or one of its refs. We can identify which screenshot is + for the test by comparing the url piece to the test name. + + Args: + results_dir: str path to the directory that results live in + test:name str name of the test that this artifact is for + screenshot_artifact: list of strings, each being a url-png pair as + described above. + + Returns: + A dict mapping the screenshot key (ie: actual, expected) to the + path of the file for that screenshot + """ + result={} + for screenshot_pair in screenshot_artifact: + screenshot_split = screenshot_pair.split(":") + url = screenshot_split[0] + # The url produced by wptrunner will have a leading / which we trim + # away for easier comparison to the test_name below. + if url.startswith("/"): + url = url[1:] + image_bytes = base64.b64decode(screenshot_split[1].strip()) + + screenshot_key = "expected_image" + file_suffix = test_failures.FILENAME_SUFFIX_EXPECTED + if test_name == url: + screenshot_key = "actual_image" + file_suffix = test_failures.FILENAME_SUFFIX_ACTUAL + + screenshot_sub_path = ( + os.path.join("layout-test-results", + self.port.output_filename( + test_name, file_suffix, ".png")) + ) + result[screenshot_key] = screenshot_sub_path + + screenshot_full_path = os.path.join(results_dir,screenshot_sub_path) + if not os.path.exists(os.path.dirname(screenshot_full_path)): + os.makedirs(os.path.dirname(screenshot_full_path)) + # Note: we are writing raw bytes to this file + with open(screenshot_full_path, "wb") as artifact_file: + artifact_file.write(image_bytes) + return result diff --git a/chromium/testing/test.gni b/chromium/testing/test.gni index bd2dd7c21c5..ece76f1503e 100644 --- a/chromium/testing/test.gni +++ b/chromium/testing/test.gni @@ -96,12 +96,12 @@ template("test") { _apk_specific_vars = [ "android_manifest", "android_manifest_dep", + "app_as_shared_lib", "enable_multidex", "product_config_java_packages", "min_sdk_version", "proguard_configs", "proguard_enabled", - "shared_resources", "srcjar_deps", "target_sdk_version", "use_default_launcher", diff --git a/chromium/testing/test_env.py b/chromium/testing/test_env.py index 8734f419579..6e39aa40121 100755 --- a/chromium/testing/test_env.py +++ b/chromium/testing/test_env.py @@ -1,10 +1,11 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Sets environment variables needed to run a chromium unit test.""" +from __future__ import print_function import io import os import signal @@ -13,6 +14,13 @@ import subprocess import sys import time +SIX_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'third_party', + 'six') +sys.path.insert(0, SIX_DIR) + +import six + # This is hardcoded to be src/ relative to this script. ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -167,12 +175,12 @@ def symbolize_snippets_in_json(cmd, env): p = subprocess.Popen(symbolize_command, stderr=subprocess.PIPE, env=env) (_, stderr) = p.communicate() except OSError as e: - print >> sys.stderr, 'Exception while symbolizing snippets: %s' % e + print('Exception while symbolizing snippets: %s' % e, file=sys.stderr) raise if p.returncode != 0: - print >> sys.stderr, "Error: failed to symbolize snippets in JSON:\n" - print >> sys.stderr, stderr + print("Error: failed to symbolize snippets in JSON:\n", file=sys.stderr) + print(stderr, file=sys.stderr) raise subprocess.CalledProcessError(p.returncode, symbolize_command) @@ -339,13 +347,13 @@ def run_executable(cmd, env, stdoutfile=None): # because you need them to reproduce the task properly. env_to_print = extra_env.copy() for env_var_name in ('GTEST_SHARD_INDEX', 'GTEST_TOTAL_SHARDS'): - if env_var_name in env: - env_to_print[env_var_name] = env[env_var_name] + if env_var_name in env: + env_to_print[env_var_name] = env[env_var_name] print('Additional test environment:\n%s\n' 'Command: %s\n' % ( '\n'.join(' %s=%s' % - (k, v) for k, v in sorted(env_to_print.iteritems())), + (k, v) for k, v in sorted(six.iteritems(env_to_print))), ' '.join(cmd))) sys.stdout.flush() env.update(extra_env or {}) @@ -371,7 +379,7 @@ def run_executable(cmd, env, stdoutfile=None): else: return run_command(cmd, env=env, log=False) except OSError: - print >> sys.stderr, 'Failed to start %s' % cmd + print('Failed to start %s' % cmd, file=sys.stderr) raise diff --git a/chromium/testing/test_env_test_script.py b/chromium/testing/test_env_test_script.py index d6cd7197f05..4957ee656d5 100755 --- a/chromium/testing/test_env_test_script.py +++ b/chromium/testing/test_env_test_script.py @@ -5,14 +5,14 @@ """Script for use in test_env unittests.""" -import os +from __future__ import print_function import signal import sys import time -def print_signal(sig, *_): - print 'Signal :{}'.format(sig) +def print_signal(sig, *_args): + print('Signal :{}'.format(sig)) if __name__ == '__main__': diff --git a/chromium/testing/test_env_unittest.py b/chromium/testing/test_env_unittest.py index 76d5766debc..707c65e9fdf 100755 --- a/chromium/testing/test_env_unittest.py +++ b/chromium/testing/test_env_unittest.py @@ -17,20 +17,29 @@ import sys import time import unittest - -TEST_SCRIPT = 'test_env_user_script.py' +HERE = os.path.dirname(os.path.abspath(__file__)) +TEST_SCRIPT = os.path.join(HERE, 'test_env_user_script.py') def launch_process_windows(args): + # The `universal_newlines` option is equivalent to `text` in Python 3. return subprocess.Popen( - [sys.executable, TEST_SCRIPT] + args, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=os.environ.copy(), - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + [sys.executable, TEST_SCRIPT] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=os.environ.copy(), + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + universal_newlines=True) + def launch_process_nonwindows(args): + # The `universal_newlines` option is equivalent to `text` in Python 3. return subprocess.Popen( - [sys.executable, TEST_SCRIPT] + args, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=os.environ.copy()) + [sys.executable, TEST_SCRIPT] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=os.environ.copy(), + universal_newlines=True) def read_subprocess_message(proc, starts_with): @@ -58,7 +67,7 @@ class SignalingWindowsTest(unittest.TestCase): proc = launch_process_windows([]) send_and_wait(proc, signal.CTRL_BREAK_EVENT) sig = read_subprocess_message(proc, 'Signal :') - self.assertEqual(sig, str(signal.SIGBREAK)) + self.assertEqual(sig, str(int(signal.SIGBREAK))) class SignalingNonWindowsTest(unittest.TestCase): @@ -72,13 +81,13 @@ class SignalingNonWindowsTest(unittest.TestCase): proc = launch_process_nonwindows([]) send_and_wait(proc, signal.SIGTERM) sig = read_subprocess_message(proc, 'Signal :') - self.assertEqual(sig, str(signal.SIGTERM)) + self.assertEqual(sig, str(int(signal.SIGTERM))) def test_send_sigint(self): proc = launch_process_nonwindows([]) send_and_wait(proc, signal.SIGINT) sig = read_subprocess_message(proc, 'Signal :') - self.assertEqual(sig, str(signal.SIGINT)) + self.assertEqual(sig, str(int(signal.SIGINT))) if __name__ == '__main__': diff --git a/chromium/testing/test_env_user_script.py b/chromium/testing/test_env_user_script.py index 8cf70adef21..1d94387633d 100755 --- a/chromium/testing/test_env_user_script.py +++ b/chromium/testing/test_env_user_script.py @@ -5,8 +5,13 @@ """Script for use in test_env unittests.""" +import os +import sys + import test_env +HERE = os.path.dirname(os.path.abspath(__file__)) +TEST_SCRIPT = os.path.join(HERE, 'test_env_test_script.py') if __name__ == '__main__': - test_env.run_command(['python', 'test_env_test_script.py']) + test_env.run_command([sys.executable, TEST_SCRIPT]) diff --git a/chromium/testing/trigger_scripts/base_test_triggerer.py b/chromium/testing/trigger_scripts/base_test_triggerer.py index bf7c44b5c01..d16ab381a0d 100755 --- a/chromium/testing/trigger_scripts/base_test_triggerer.py +++ b/chromium/testing/trigger_scripts/base_test_triggerer.py @@ -10,7 +10,7 @@ calls under one logical step. It also gives the subclasses the ability to define their own logic for pruning the configurations they want to trigger jobs on and what configurations to use. -See trigger_multiple_dimensions.py for an example of how to use this base class. +See perf_device_triggerer.py for an example of how to use this base class. """ diff --git a/chromium/testing/trigger_scripts/trigger_multiple_dimensions.py b/chromium/testing/trigger_scripts/trigger_multiple_dimensions.py deleted file mode 100755 index d4d26f55351..00000000000 --- a/chromium/testing/trigger_scripts/trigger_multiple_dimensions.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Custom swarming triggering script. - -This script does custom swarming triggering logic, to allow one bot to -conceptually span multiple Swarming configurations, while lumping all trigger -calls under one logical step. - -The reason this script is needed is to allow seamless upgrades of the GPU, OS -version, or graphics driver. Most Chromium tests, GPU tests in particular, are -triggered with precise values for all of these Swarming dimensions. This ensures -that if a machine is added to the Swarming pool with a slightly different -configuration, tests don't fail for unexpected reasons. - -During an upgrade of the fleet, it's not feasible to take half of the machines -offline. Some experience was gained with this during a recent upgrade of the -GPUs in Chromium's main Windows and Linux NVIDIA bots. In the middle of the -upgrade, only 50% of the capacity was available, and CQ jobs started to time -out. Once the hurdle had been passed in the middle of the upgrade, capacity was -sufficient, but it's crucial that this process remain seamless. - -This script receives multiple machine configurations on the command line in the -form of quoted strings. These strings are JSON dictionaries that represent -entries in the "dimensions" array of the "swarming" dictionary in the -src/testing/buildbot JSON files. The script queries the Swarming pool for the -number of machines of each configuration, and distributes work (shards) among -them using the following algorithm: - -1. If either configuration has machines available (online, not busy at the time -of the query) then distribute shards to them first. - -2. Compute the relative fractions of all of the live (online, not quarantined, -not dead) machines of all configurations. - -3. Distribute the remaining shards probabilistically among these configurations. - -The use of random numbers attempts to avoid the pathology where one -configuration only has a couple of machines, and work is never distributed to it -once all machines are busy. - -This script must have roughly the same command line interface as swarming.py -trigger. It modifies it in the following ways: - * Intercepts the dump-json argument, and creates its own by combining the - results from each trigger call. - * Scans through the multiple-trigger-configs dictionaries. For any key found, - deletes that dimension from the originally triggered task's dimensions. This - is what allows the Swarming dimensions to be replaced. - * On a per-shard basis, adds the Swarming dimensions chosen from the - multiple-trigger-configs list to the dimensions for the shard. - -This script is normally called from the swarming recipe module in tools/build. - -""" - -import argparse -import copy -import json -import os -import random -import subprocess -import sys -import tempfile -import urllib - -import base_test_triggerer - - -class MultiDimensionTestTriggerer(base_test_triggerer.BaseTestTriggerer): - def __init__(self): - super(MultiDimensionTestTriggerer, self).__init__() - - def choose_random_int(self, max_num): - return random.randint(1, max_num) - - def pick_bot_configuration(self, verbose): - # These are the rules used: - # 1. If any configuration has bots available, pick the configuration with - # the most bots available. - # 2. If no configuration has bots available, pick a random configuration - # based on the total number of bots in each configuration. - # - # This method updates bot_statuses_ in case (1), and in both cases, returns - # the index into bot_configs_ that should be used. - if any(status['available'] > 0 for status in self._bot_statuses): - # Case 1. - max_index = 0 - max_val = self._bot_statuses[0]['available'] - for i in xrange(1, len(self._bot_statuses)): - avail = self._bot_statuses[i]['available'] - if avail > max_val: - max_index = i - max_val = avail - self._bot_statuses[max_index]['available'] -= 1 - assert self._bot_statuses[max_index]['available'] >= 0 - if verbose: - print 'Chose bot config %d because bots were available' % (max_index) - return max_index - # Case 2. - # We want to choose a bot uniformly at random from all of the bots specified - # in the bot configs. To do this, we conceptually group the bots into - # buckets, pick a random number between 1 and the total number of bots, and - # figure out which bucket of bots it landed in. - r = self.choose_random_int(self._total_bots) - for i, status in enumerate(self._bot_statuses): - if r <= status['total']: - if verbose: - print 'Chose bot config %d stochastically' % (i) - return i - r -= status['total'] - raise Exception('Should not reach here') - - def select_config_indices(self, args, verbose): - selected_indices = [] - for shard_index in self.indices_to_trigger(args): - selected_indices.append( - (shard_index, self.pick_bot_configuration(verbose))) - return selected_indices - - def prune_test_specific_configs(self, args, verbose): - self.query_swarming_for_bot_configs(verbose) - # This script doesn't know how long individual test shards take to - # run, nor how many Swarming jobs are waiting to run on a - # particular configuration. It can end up scheduling jobs on - # configurations that have very few machines, and backing them up - # to the point where the tasks start expiring. To try to prevent - # this, don't schedule jobs at all on configurations that have - # less than 10% of the total capacity. crbug.com/886985 - MIN_CONFIG_CAPACITY_PERCENTAGE = 0.1 - filtered_bot_configs = [] - filtered_bot_statuses = [] - for i in xrange(len(self._bot_configs)): - config = self._bot_configs[i] - status = self._bot_statuses[i] - if status['total'] >= MIN_CONFIG_CAPACITY_PERCENTAGE * self._total_bots: - filtered_bot_configs.append(config) - filtered_bot_statuses.append(status) - else: - if verbose: - print 'Filtered config because it had too few bots: %s' % str(status) - if len(filtered_bot_configs) == 0: - raise Exception('The bot configurations are too fragmented; no single ' + - 'configuration has even 10% of the total capacity. ' + - 'Distribution will not work well. Failing.') - self._bot_configs = filtered_bot_configs - self._bot_statuses = filtered_bot_statuses - self._total_bots = sum(x['total'] for x in self._bot_statuses) - if verbose: - print 'Total bots after filtering: %d' % (self._total_bots) - -def main(): - # setup args for common contract of base class - parser = base_test_triggerer.BaseTestTriggerer.setup_parser_contract( - argparse.ArgumentParser(description=__doc__)) - args, remaining = parser.parse_known_args() - - triggerer = MultiDimensionTestTriggerer() - return triggerer.trigger_tasks(args, remaining) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/chromium/testing/trigger_scripts/trigger_multiple_dimensions_unittest.py b/chromium/testing/trigger_scripts/trigger_multiple_dimensions_unittest.py deleted file mode 100755 index 9a29cd3960e..00000000000 --- a/chromium/testing/trigger_scripts/trigger_multiple_dimensions_unittest.py +++ /dev/null @@ -1,351 +0,0 @@ -#!/usr/bin/python -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Tests for trigger_multiple_dimensions.py.""" - -import unittest - -import trigger_multiple_dimensions - -class Args(object): - def __init__(self): - self.shards = 1 - self.shard_index = None - self.dump_json = '' - self.multiple_trigger_configs = [] - self.multiple_dimension_script_verbose = False - - -class FakeTriggerer(trigger_multiple_dimensions.MultiDimensionTestTriggerer): - def __init__(self, bot_configs, bot_statuses, - use_superclass_random_number_generator, first_random_number): - super(FakeTriggerer, self).__init__() - self._bot_configs = bot_configs - self._bot_statuses = bot_statuses - self._swarming_runs = [] - self._files = {} - self._temp_file_id = 0 - self._use_superclass_rng = use_superclass_random_number_generator - self._last_random_number = first_random_number - - def set_files(self, files): - self._files = files - - def choose_random_int(self, max_num): - if self._use_superclass_rng: - return super(FakeTriggerer, self).choose_random_int(max_num) - if self._last_random_number > max_num: - self._last_random_number = 1 - result = self._last_random_number - self._last_random_number += 1 - return result - - def make_temp_file(self, prefix=None, suffix=None): - result = prefix + str(self._temp_file_id) + suffix - self._temp_file_id += 1 - return result - - def delete_temp_file(self, temp_file): - pass - - def read_json_from_temp_file(self, temp_file): - return self._files[temp_file] - - def write_json_to_file(self, merged_json, output_file): - self._files[output_file] = merged_json - - def parse_bot_configs(self, args): - pass - - def query_swarming_for_bot_configs(self, verbose): - # Sum up the total count of all bots. - self._total_bots = sum(x['total'] for x in self._bot_statuses) - - def run_swarming(self, args, verbose): - self._swarming_runs.append(args) - - -WIN_NVIDIA_QUADRO_P400_STABLE_DRIVER = '10de:1cb3-23.21.13.8792' -WIN7 = 'Windows-2008ServerR2-SP1' -WIN10 = 'Windows-10' - -WIN7_NVIDIA = { - 'gpu': WIN_NVIDIA_QUADRO_P400_STABLE_DRIVER, - 'os': WIN7, - 'pool': 'Chrome-GPU', -} - -WIN10_NVIDIA = { - 'gpu': WIN_NVIDIA_QUADRO_P400_STABLE_DRIVER, - 'os': WIN10, - 'pool': 'Chrome-GPU', -} - -class UnitTest(unittest.TestCase): - def basic_win7_win10_setup(self, bot_statuses, - use_superclass_random_number_generator=False, - first_random_number=1, - shards=2): - triggerer = FakeTriggerer( - [ - WIN7_NVIDIA, - WIN10_NVIDIA - ], - bot_statuses, - use_superclass_random_number_generator, - first_random_number - ) - # Note: the contents of these JSON files don't accurately reflect - # that produced by "swarming.py trigger". The unit tests only - # verify that shard 0's JSON is preserved. - file_map = {} - for i in xrange(shards): - file_name = 'base_trigger_dimensions%d.json' % i - result = { - 'tasks': { - 'webgl_conformance_tests on NVIDIA GPU on Windows': { - 'task_id': hex(0xf000 + i)[2:] - }, - }, - } - if i == 0: - result['base_task_name'] = 'webgl_conformance_tests' - result['request'] = { - 'expiration_secs': 3600, - 'properties': { - 'execution_timeout_secs': 3600, - }, - } - file_map[file_name] = result - triggerer.set_files(file_map) - args = Args() - args.shards = shards - args.dump_json = 'output.json' - args.multiple_dimension_script_verbose = False - triggerer.trigger_tasks( - args, - [ - 'trigger', - '--dimension', - 'gpu', - WIN_NVIDIA_QUADRO_P400_STABLE_DRIVER, - '--dimension', - 'os', - WIN7, - '--dimension', - 'pool', - 'Chrome-GPU', - '--', - 'webgl_conformance', - ]) - return triggerer - - def list_contains_sublist(self, main_list, sub_list): - return any(sub_list == main_list[offset:offset + len(sub_list)] - for offset in xrange(len(main_list) - (len(sub_list) - 1))) - - def shard_runs_on_os(self, triggerer, shard_index, os): - return self.list_contains_sublist(triggerer._swarming_runs[shard_index], - ['--dimension', 'os', os]) - - def test_parse_bot_configs(self): - triggerer = trigger_multiple_dimensions.MultiDimensionTestTriggerer() - args = Args() - args.multiple_trigger_configs = "{ foo }" - self.assertRaisesRegexp(ValueError, "Error while parsing JSON.*", - triggerer.parse_bot_configs, args) - args.multiple_trigger_configs = "{ \"foo\": \"bar\" }" - self.assertRaisesRegexp(ValueError, "Bot configurations must be a list.*", - triggerer.parse_bot_configs, args) - args.multiple_trigger_configs = "[]" - self.assertRaisesRegexp(ValueError, - "Bot configuration list must have at least.*", - triggerer.parse_bot_configs, args) - args.multiple_trigger_configs = "[{}, \"\"]" - self.assertRaisesRegexp(ValueError, - "Bot configurations must all be.*", - triggerer.parse_bot_configs, args) - args.multiple_trigger_configs = "[{}]" - triggerer.parse_bot_configs(args) - self.assertEqual(triggerer._bot_configs, [{}]) - - def test_split_with_available_machines(self): - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 1, - 'available': 1, - }, - { - 'total': 1, - 'available': 1, - }, - ], - ) - # First shard should run on Win7. - self.assertTrue(self.shard_runs_on_os(triggerer, 0, WIN7)) - # Second shard should run on Win10. - self.assertTrue(self.shard_runs_on_os(triggerer, 1, WIN10)) - # And not vice versa. - self.assertFalse(self.shard_runs_on_os(triggerer, 0, WIN10)) - self.assertFalse(self.shard_runs_on_os(triggerer, 1, WIN7)) - - def test_shard_env_vars(self): - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 2, - 'available': 2, - }, - { - 'total': 2, - 'available': 0, - }, - ], - ) - self.assertTrue(self.list_contains_sublist( - triggerer._swarming_runs[0], ['--env', 'GTEST_SHARD_INDEX', '0'])) - self.assertTrue(self.list_contains_sublist( - triggerer._swarming_runs[1], ['--env', 'GTEST_SHARD_INDEX', '1'])) - self.assertTrue(self.list_contains_sublist( - triggerer._swarming_runs[0], ['--env', 'GTEST_TOTAL_SHARDS', '2'])) - self.assertTrue(self.list_contains_sublist( - triggerer._swarming_runs[1], ['--env', 'GTEST_TOTAL_SHARDS', '2'])) - - def test_json_merging(self): - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 1, - 'available': 1, - }, - { - 'total': 1, - 'available': 1, - }, - ], - ) - self.assertTrue('output.json' in triggerer._files) - output_json = triggerer._files['output.json'] - self.assertTrue('base_task_name' in output_json) - self.assertTrue('request' in output_json) - self.assertEqual(output_json['request']['expiration_secs'], 3600) - self.assertEqual( - output_json['request']['properties']['execution_timeout_secs'], 3600) - - def test_split_with_only_one_config_available(self): - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 2, - 'available': 2, - }, - { - 'total': 2, - 'available': 0, - }, - ], - ) - # Both shards should run on Win7. - self.assertTrue(self.shard_runs_on_os(triggerer, 0, WIN7)) - self.assertTrue(self.shard_runs_on_os(triggerer, 1, WIN7)) - # Redo with only Win10 bots available. - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 2, - 'available': 0, - }, - { - 'total': 2, - 'available': 2, - }, - ], - ) - self.assertTrue(self.shard_runs_on_os(triggerer, 0, WIN10)) - self.assertTrue(self.shard_runs_on_os(triggerer, 1, WIN10)) - - def test_split_with_no_bots_available(self): - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 1, - 'available': 0, - }, - { - 'total': 1, - 'available': 0, - }, - ], - ) - # Given the fake random number generator above, first shard should - # run on Win7. - self.assertTrue(self.shard_runs_on_os(triggerer, 0, WIN7)) - # Second shard should run on Win10. - self.assertTrue(self.shard_runs_on_os(triggerer, 1, WIN10)) - # Try again with different bot distribution and random numbers. - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 2, - 'available': 0, - }, - { - 'total': 2, - 'available': 0, - }, - ], - first_random_number=3, - ) - self.assertTrue(self.shard_runs_on_os(triggerer, 0, WIN10)) - self.assertTrue(self.shard_runs_on_os(triggerer, 1, WIN10)) - - def test_superclass_random_number_generator_works(self): - # Probe randomly a certain number of times. - num_runs = 0 - for _ in xrange(100): - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 2, - 'available': 0, - }, - { - 'total': 2, - 'available': 0, - }, - ], - use_superclass_random_number_generator=True - ) - for _ in xrange(2): - self.assertTrue(self.shard_runs_on_os(triggerer, 0, WIN7) or - self.shard_runs_on_os(triggerer, 0, WIN10)) - num_runs += 1 - self.assertEqual(num_runs, 200) - - def test_split_with_imbalanced_configs(self): - num_shards = 20 - triggerer = self.basic_win7_win10_setup( - [ - { - 'total': 15, - 'available': 1, - }, - { - 'total': 1, - 'available': 1, - }, - ], - shards=num_shards - ) - # Because the second configuration (Win10) has so few machines -- - # fewer than 10% of the total -- the triggerer should never - # schedule any shard on it. - for i in xrange(num_shards): - self.assertTrue(self.shard_runs_on_os(triggerer, i, WIN7)) - self.assertFalse(self.shard_runs_on_os(triggerer, i, WIN10)) - -if __name__ == '__main__': - unittest.main() diff --git a/chromium/testing/variations/PRESUBMIT.py b/chromium/testing/variations/PRESUBMIT.py index ed3b6c0b5f6..96c52d894ad 100644 --- a/chromium/testing/variations/PRESUBMIT.py +++ b/chromium/testing/variations/PRESUBMIT.py @@ -15,7 +15,8 @@ from collections import OrderedDict VALID_EXPERIMENT_KEYS = [ 'name', 'forcing_flag', 'params', 'enable_features', 'disable_features', - '//0', '//1', '//2', '//3', '//4', '//5', '//6', '//7', '//8', '//9' + 'min_os_version', '//0', '//1', '//2', '//3', '//4', '//5', '//6', '//7', + '//8', '//9' ] FIELDTRIAL_CONFIG_FILE_NAME = 'fieldtrial_testing_config.json' @@ -93,6 +94,8 @@ def PrettyPrint(contents): ordered_experiment['disable_features'] = \ sorted(experiment['disable_features']) ordered_experiment_config['experiments'].append(ordered_experiment) + if 'min_os_version' in experiment: + ordered_experiment['min_os_version'] = experiment['min_os_version'] ordered_study.append(ordered_experiment_config) ordered_config[key] = ordered_study return json.dumps( diff --git a/chromium/testing/variations/README.md b/chromium/testing/variations/README.md index 9568f20a6b3..dda05d59d01 100644 --- a/chromium/testing/variations/README.md +++ b/chromium/testing/variations/README.md @@ -92,13 +92,18 @@ the experiment group name. > client code, the `name` **must** exactly match the client code and the server > config. -The remaining keys -- `enable_features`, `disable_features`, and `params` -- are -optional. +The remaining keys -- `enable_features`, `disable_features`, `min_os_version`, +and `params` -- are optional. `enable_features` and `disable_features` indicate which features should be enabled and disabled, respectively, through the [Feature List API][FeatureListAPI]. +`min_os_version` indicates a minimum OS version level (e.g. "10.0.0") to apply +the experiment. This string is decoded as a `base::Version`. The same version is +applied to all platforms. If you need different versions for different +platforms, you will need to use different studies. + `params` is a dictionary mapping parameter name to parameter value. > Reminder: The variations framework does not actually fetch any field trial diff --git a/chromium/testing/variations/fieldtrial_testing_config.json b/chromium/testing/variations/fieldtrial_testing_config.json index f85fecad5ee..ca5a216cc38 100644 --- a/chromium/testing/variations/fieldtrial_testing_config.json +++ b/chromium/testing/variations/fieldtrial_testing_config.json @@ -26,6 +26,31 @@ ] } ], + "AlignFontDisplayAutoTimeoutWithLCPGoal": [ + { + "platforms": [ + "android", + "android_weblayer", + "android_webview", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "EnabledWithSwapModeAndTimeout1000ms", + "params": { + "intervention-mode": "swap", + "lcp-limit-in-ms": "1000" + }, + "enable_features": [ + "AlignFontDisplayAutoTimeoutWithLCPGoal" + ] + } + ] + } + ], "AllPassiveMixedContentAutoupgrade": [ { "platforms": [ @@ -64,22 +89,22 @@ ] } ], - "AndroidBlockExternalIntentsWithoutGesture": [ + "AndroidBlockIntentNonSafelistedHeaders": [ { "platforms": [ "android" ], "experiments": [ { - "name": "Enabled_20200323", + "name": "EnabledLaunch", "enable_features": [ - "IntentBlockExternalFormRedirectsNoGesture" + "AndroidBlockIntentNonSafelistedHeaders" ] } ] } ], - "AndroidBlockIntentNonSafelistedHeaders": [ + "AndroidDarkSearch": [ { "platforms": [ "android" @@ -88,13 +113,13 @@ { "name": "Enabled", "enable_features": [ - "AndroidBlockIntentNonSafelistedHeaders" + "AndroidDarkSearch" ] } ] } ], - "AndroidDarkSearch": [ + "AndroidDefaultBrowserPromo": [ { "platforms": [ "android" @@ -102,8 +127,29 @@ "experiments": [ { "name": "Enabled", + "params": { + "max_promo_count": "1", + "min_trigger_session_count": "3", + "promo_interval": "86400", + "skip_primer": "false" + }, "enable_features": [ - "AndroidDarkSearch" + "AndroidDefaultBrowserPromo" + ] + } + ] + } + ], + "AndroidDynamicWideColorGamut": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "DynamicColorGamut" ] } ] @@ -591,6 +637,28 @@ ] } ], + "AutofillCacheServerCardInfo": [ + { + "platforms": [ + "android", + "android_weblayer", + "android_webview", + "chromeos", + "ios", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "AutofillCacheServerCardInfo" + ] + } + ] + } + ], "AutofillCompany": [ { "platforms": [ @@ -630,6 +698,27 @@ ] } ], + "AutofillEnableSurfacingServerCardNickname": [ + { + "platforms": [ + "android", + "android_webview", + "chromeos", + "ios", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "AutofillEnableSurfacingServerCardNickname" + ] + } + ] + } + ], "AutofillFieldMetadata": [ { "platforms": [ @@ -651,8 +740,7 @@ "AutofillKeyboardAccessory": [ { "platforms": [ - "android", - "android_weblayer" + "android" ], "experiments": [ { @@ -858,27 +946,6 @@ ] } ], - "AutofillUpstreamEditableExpirationDate": [ - { - "platforms": [ - "android", - "android_weblayer", - "chromeos", - "ios", - "linux", - "mac", - "windows" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "AutofillUpstreamEditableExpirationDate" - ] - } - ] - } - ], "AutofillUseImprovedLabelDisambiguation": [ { "platforms": [ @@ -1161,6 +1228,27 @@ ] } ], + "CSSMatchedPropertiesCacheDependencies": [ + { + "platforms": [ + "android", + "android_weblayer", + "android_webview", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "CSSMatchedPropertiesCacheDependencies" + ] + } + ] + } + ], "CSSOMViewScrollCoordinates": [ { "platforms": [ @@ -1182,6 +1270,27 @@ ] } ], + "CSSReducedFontLoadingLayoutInvalidations": [ + { + "platforms": [ + "android", + "android_weblayer", + "android_webview", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "CSSReducedFontLoadingLayoutInvalidations" + ] + } + ] + } + ], "CacheStorageEagerReading": [ { "platforms": [ @@ -1202,21 +1311,48 @@ ] } ], - "CacheStorageSequence": [ + "CacheStorageSequenceChromeOS": [ + { + "platforms": [ + "chromeos" + ], + "experiments": [ + { + "name": "Enabled12_v1", + "params": { + "min": "12" + }, + "enable_features": [ + "BrowserThreadPoolAdjustment", + "CacheStorageSequence" + ] + } + ] + } + ], + "CastMediaRouteProviderAndGlobalMediaControlsForCast": [ { "platforms": [ - "windows", - "mac", "chromeos", "linux", - "android", - "android_weblayer" + "mac", + "windows" ], "experiments": [ { - "name": "Enabled1", + "name": "CMRP_and_GMCFC_enabled", "enable_features": [ - "CacheStorageSequence" + "CastMediaRouteProvider", + "GlobalMediaControlsForCast" + ] + }, + { + "name": "CMRP_enabled_GMCFC_disabled", + "enable_features": [ + "CastMediaRouteProvider" + ], + "disable_features": [ + "GlobalMediaControlsForCast" ] } ] @@ -1431,6 +1567,42 @@ ] } ], + "ClearSyncedData": [ + { + "platforms": [ + "ios" + ], + "experiments": [ + { + "name": "Enabled_20200407", + "enable_features": [ + "ClearSyncedData" + ] + } + ] + } + ], + "ClientSideDetectionModel": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "params": { + "ModelNum": "4" + }, + "enable_features": [ + "ClientSideDetectionModel" + ] + } + ] + } + ], "ClipboardMaximumAge": [ { "platforms": [ @@ -1449,12 +1621,10 @@ ] } ], - "CompositeCrossOriginIframes": [ + "CodeCacheDeletionWithoutFilter": [ { "platforms": [ "android", - "android_weblayer", - "android_webview", "chromeos", "linux", "mac", @@ -1464,7 +1634,7 @@ { "name": "Enabled", "enable_features": [ - "CompositeCrossOriginIframes" + "CodeCacheDeletionWithoutFilter" ] } ] @@ -1522,6 +1692,21 @@ ] } ], + "ContextMenuCopyImage": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "ContextMenuCopyImage" + ] + } + ] + } + ], "ContextMenuSearchWithGoogleLensMultiArm": [ { "platforms": [ @@ -1570,8 +1755,16 @@ "experiments": [ { "name": "ContextualSearchLongpressResolve", + "params": { + "availability": "any", + "event_longpress_opened": "name:contextual_search_panel_opened_after_longpress;comparator:==0;window:365;storage:365", + "event_trigger": "name:longpress_bubble_shown;comparator:<3;window:90;storage:90", + "event_used": "name:contextual_search_triggered_by_longpress;comparator:==0;window:90;storage:90", + "session_rate": "any" + }, "enable_features": [ - "ContextualSearchLongpressResolve" + "ContextualSearchLongpressResolve", + "IPH_ContextualSearchTappedButShouldLongpress" ] } ] @@ -1644,6 +1837,21 @@ ] } ], + "CrOSUserSpaceLowMemoryNotification": [ + { + "platforms": [ + "chromeos" + ], + "experiments": [ + { + "name": "Enabled_20200520", + "enable_features": [ + "CrOSUserSpaceLowMemoryNotification" + ] + } + ] + } + ], "CrostiniWebUIUpgrader": [ { "platforms": [ @@ -1822,7 +2030,8 @@ "DefaultPassthroughCommandDecoder": [ { "platforms": [ - "linux" + "linux", + "mac" ], "experiments": [ { @@ -1834,6 +2043,29 @@ ] } ], + "DelayAsyncScriptExecution": [ + { + "platforms": [ + "android", + "android_webview", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "params": { + "delay_type": "first_paint_or_finished_parsing" + }, + "enable_features": [ + "DelayAsyncScriptExecution" + ] + } + ] + } + ], "DeprecateFtp": [ { "platforms": [ @@ -1943,6 +2175,21 @@ ] } ], + "DirectCompositionSoftwareOverlays": [ + { + "platforms": [ + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "DirectCompositionSoftwareOverlays" + ] + } + ] + } + ], "DirectCompositionUseNV12DecodeSwapChain": [ { "platforms": [ @@ -1958,18 +2205,53 @@ ] } ], + "DisableMalwareExtensionsRemotely": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "DisableMalwareExtensionsRemotely" + ] + } + ] + } + ], "DnsOverHttps": [ { "platforms": [ "android", - "android_weblayer", + "android_weblayer" + ], + "experiments": [ + { + "name": "Enabled_crbug1006105", + "enable_features": [ + "DnsOverHttps" + ] + } + ] + } + ], + "DnsOverHttpsUi": [ + { + "platforms": [ "chromeos", "mac", "windows" ], "experiments": [ { - "name": "Enabled_crbug1006105", + "name": "EnabledLaunch", + "params": { + "ShowUi": "true" + }, "enable_features": [ "DnsOverHttps" ] @@ -2064,6 +2346,32 @@ ] } ], + "DownloadsAsMixedContent": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "params": { + "BlockExtensionList": "mix_dl_blocked_for_testing", + "SilentBlockExtensionList": "mix_dl_silently_blocked_for_testing", + "TreatBlockListAsAllowlist": "false", + "TreatSilentBlockListAsAllowlist": "false", + "TreatWarnListAsAllowlist": "false", + "WarnExtensionList": "exe,scr,msi,vb,dmg,pkg,crx" + }, + "enable_features": [ + "TreatUnsafeDownloadsAsActive" + ] + } + ] + } + ], "DownloadsAutoResumptionNative": [ { "platforms": [ @@ -2171,6 +2479,24 @@ ] } ], + "EnableNewBadgeOnMenuItems": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "EnableNewBadgeOnMenuItems" + ] + } + ] + } + ], "EnableSafetyTipUI": [ { "platforms": [ @@ -2196,29 +2522,22 @@ ] } ], - "EphemeralTab": [ + "EnterpriseReportingInChromeOS": [ { "platforms": [ - "android" + "chromeos" ], "experiments": [ { - "name": "Enabled", - "params": { - "availability": "any", - "event_trigger": "name:ephemeral_tab_triggered;comparator:<2;window:90;storage:90", - "event_used": "name:ephemeral_tab_used;comparator:==0;window:90;storage:90", - "session_rate": "any" - }, + "name": "Enabled_20200508", "enable_features": [ - "EphemeralTabUsingBottomSheet", - "IPH_EphemeralTab" + "EnterpriseReportingInChromeOS" ] } ] } ], - "ExpandablePaymentHandler": [ + "EphemeralTab": [ { "platforms": [ "android" @@ -2226,8 +2545,15 @@ "experiments": [ { "name": "Enabled", + "params": { + "availability": "any", + "event_trigger": "name:ephemeral_tab_triggered;comparator:<2;window:90;storage:90", + "event_used": "name:ephemeral_tab_used;comparator:==0;window:90;storage:90", + "session_rate": "any" + }, "enable_features": [ - "ScrollToExpandPaymentHandler" + "EphemeralTabUsingBottomSheet", + "IPH_EphemeralTab" ] } ] @@ -2317,6 +2643,28 @@ ] } ], + "FeedActionsUpload": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enaled", + "params": { + "availability": "any", + "event_trigger": "name:iph_feed_header_menu_triggered;comparator:==0;window:360;storage:360", + "event_used": "name:feed_header_menu_clicked;comparator:==0;window:360;storage:360", + "session_rate": "==0" + }, + "enable_features": [ + "IPH_FeedHeaderMenu", + "ReportFeedUserActions" + ] + } + ] + } + ], "FilterAdsOnAbusiveSites": [ { "platforms": [ @@ -2337,6 +2685,65 @@ ] } ], + "FindInPageBestEffortPriority": [ + { + "platforms": [ + "android", + "android_weblayer", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled_20200601", + "enable_features": [ + "BlinkSchedulerBestEffortPriorityForFindInPage" + ] + } + ] + } + ], + "FixedMaxBestEffortTasks": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "FixedMaxBestEffortTasks" + ] + } + ] + } + ], + "FlexNG": [ + { + "platforms": [ + "android", + "android_webview", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled_20200406", + "enable_features": [ + "FlexNG" + ] + } + ] + } + ], "FontPreloadingDelaysRendering": [ { "platforms": [ @@ -2381,6 +2788,28 @@ ] } ], + "ForegroundVideoCaptureService": [ + { + "platforms": [ + "android", + "android_weblayer", + "android_webview", + "chromeos", + "ios", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "ForegroundVideoCaptureService" + ] + } + ] + } + ], "FormControlsRefresh": [ { "platforms": [ @@ -2445,6 +2874,26 @@ ] } ], + "FriendlySettingsHats": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Launched_20200616", + "enable_features": [ + "PrivacySettingsRedesign", + "SafeBrowsingEnhancedProtection", + "SyncSetupFriendlySettings" + ] + } + ] + } + ], "FuzzyAppSearch": [ { "platforms": [ @@ -2494,7 +2943,7 @@ { "name": "Enabled", "params": { - "overscroll_history_navigation_bottom_sheet": "true" + "overscroll_history_navigation_bottom_sheet": "false" }, "enable_features": [ "OverscrollHistoryNavigation" @@ -2670,6 +3119,21 @@ ] } ], + "HashedLoggingForHindsight": [ + { + "platforms": [ + "chromeos" + ], + "experiments": [ + { + "name": "HashedLoggingForHindsight", + "enable_features": [ + "CrOSActionStructuredMetrics" + ] + } + ] + } + ], "HeapProfiling": [ { "platforms": [ @@ -2692,53 +3156,44 @@ ] } ], - "HistoryServiceAndProfileSyncServiceUseThreadPool": [ + "HideAutofillSuggestions": [ { "platforms": [ - "android", "chromeos", - "ios", "linux", "mac", "windows" ], "experiments": [ { - "name": "all_enabled_20200129", + "name": "Enabled", "enable_features": [ - "HistoryServiceUsesTaskScheduler", - "ProfileSyncServiceUsesThreadPool" - ] - }, - { - "name": "all_disabled_20200129", - "disable_features": [ - "HistoryServiceUsesTaskScheduler", - "ProfileSyncServiceUsesThreadPool" + "AutofillEnableHideSuggestionsUI" ] } ] } ], - "HomepagePromoCard": [ + "HomepagePromoAndroid": [ { "platforms": [ "android" ], "experiments": [ { - "name": "HomepagePromoCardForEveryone", + "name": "Enabled_Compact", "params": { - "availability": ">=30", - "event_homebutton_clicked": "name:homepage_button_clicked;comparator:==0;window:30;storage:360", - "event_ntp_shown": "name:ntp_shown;comparator:any;window:14;storage:360", + "availability": ">=1", + "event_homebutton_clicked": "name:homepage_button_clicked;comparator:==0;window:1;storage:360", + "event_ntp_shown": "name:ntp_shown;comparator:>1;window:14;storage:360", "event_promo_accepted": "name:homepage_promo_accepted;comparator:==0;window:360;storage:360", - "event_promo_seen": "name:homepage_promo_seen;comparator:any;window:30;storage:360", - "event_trigger": "name:homepage_promo_created;comparator:<=7;window:30;storage:360", + "event_promo_seen": "name:homepage_promo_seen;comparator:<7;window:30;storage:360", + "event_trigger": "name:homepage_promo_created;comparator:any;window:30;storage:360", "event_used": "name:ntp_homebutton_clicked;comparator:any;window:30;storage:360", "session_rate": "any" }, "enable_features": [ + "HomepagePromoCard", "IPH_HomepagePromoCard" ], "disable_features": [] @@ -2746,7 +3201,39 @@ ] } ], - "IOSEmbedderBlockRestoreUrl": [ + "HomepagePromoAndroidSeenEnabledCohort": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "HomepagePromoSyntheticPromoSeenEnabled" + ], + "disable_features": [] + } + ] + } + ], + "HomepagePromoAndroidSeenTrackingCohort": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "HomepagePromoSyntheticPromoSeenTracking" + ], + "disable_features": [] + } + ] + } + ], + "IOSChangeTabSwitcherPosition": [ { "platforms": [ "ios" @@ -2755,13 +3242,13 @@ { "name": "Enabled", "enable_features": [ - "EmbedderBlockRestoreUrl" + "ChangeTabSwitcherPosition" ] } ] } ], - "IOSEnablePersistentDownloads": [ + "IOSCrashReportBreadcrumbs": [ { "platforms": [ "ios" @@ -2770,14 +3257,13 @@ { "name": "Enabled", "enable_features": [ - "EnablePersistentDownloads", - "OpenDownloadsInFilesApp" + "LogBreadcrumbs" ] } ] } ], - "IOSReloadSadTab": [ + "IOSEmbedderBlockRestoreUrl": [ { "platforms": [ "ios" @@ -2786,13 +3272,13 @@ { "name": "Enabled", "enable_features": [ - "ReloadSadTab" + "EmbedderBlockRestoreUrl" ] } ] } ], - "IOSSSLCommittedInterstitials": [ + "IOSEnablePersistentDownloads": [ { "platforms": [ "ios" @@ -2801,13 +3287,14 @@ { "name": "Enabled", "enable_features": [ - "SSLCommittedInterstitials" + "EnablePersistentDownloads", + "OpenDownloadsInFilesApp" ] } ] } ], - "IOSWebPageTextAccessibility": [ + "IOSReloadSadTab": [ { "platforms": [ "ios" @@ -2816,7 +3303,22 @@ { "name": "Enabled", "enable_features": [ - "WebPageTextAccessibility" + "ReloadSadTab" + ] + } + ] + } + ], + "IOSSSLCommittedInterstitials": [ + { + "platforms": [ + "ios" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "SSLCommittedInterstitials" ] } ] @@ -2847,11 +3349,8 @@ "ImprovedCookieControls": [ { "platforms": [ - "chromeos", - "linux", - "mac", - "windows", - "android" + "android", + "android_weblayer" ], "experiments": [ { @@ -3029,6 +3528,21 @@ ] } ], + "IntentBlockExternalFormRedirectsNoGesture": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled_20200323", + "enable_features": [ + "IntentBlockExternalFormRedirectsNoGesture" + ] + } + ] + } + ], "InterestFeedContentSuggestions": [ { "platforms": [ @@ -3039,6 +3553,7 @@ "name": "EnabledUndoableActions", "params": { "card_menu_tooltip_eligible": "true", + "default_action_ttl_seconds": "90000", "feed_ui_enabled": "true", "init_feed_after_startup": "false", "manage_interests_enabled": "true", @@ -3086,6 +3601,43 @@ ] } ], + "IsolatedPrerender": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled_1Prefetch_ISPFiltering", + "params": { + "max_srp_prefetches": "1" + }, + "enable_features": [ + "IsolatePrerenders", + "IsolatePrerendersMustProbeOrigin" + ] + }, + { + "name": "NoPrefetches_CounterFactualControl", + "params": { + "max_srp_prefetches": "0" + }, + "enable_features": [ + "IsolatePrerenders" + ] + }, + { + "name": "Enabled_1Prefetch", + "params": { + "max_srp_prefetches": "1" + }, + "enable_features": [ + "IsolatePrerenders" + ] + } + ] + } + ], "KeepScriptResourceAlive": [ { "platforms": [ @@ -3270,6 +3822,25 @@ ] } ], + "LookalikeTargetEmbeddingHeuristic": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "TargetEmbeddingLookalikes" + ] + } + ] + } + ], "LowPriorityAdProcesses": [ { "platforms": [ @@ -3345,6 +3916,21 @@ ] } ], + "MacCustomPaperSize": [ + { + "platforms": [ + "mac" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "EnableCustomMacPaperSizes" + ] + } + ] + } + ], "MacSystemMediaPermissionsInfoUI": [ { "platforms": [ @@ -3375,10 +3961,12 @@ ] } ], - "MayBlockWithoutDelay": [ + "MaxOverlapBoundsForFixed": [ { "platforms": [ "android", + "android_weblayer", + "android_webview", "chromeos", "linux", "mac", @@ -3388,14 +3976,43 @@ { "name": "Enabled", "enable_features": [ - "FixedMaxBestEffortTasks", - "MayBlockWithoutDelay" + "MaxOverlapBoundsForFixed" ] } ] } ], - "MediaCapabilitiesWithParameters": [ + "MediaFoundationAsyncH264Encoding": [ + { + "platforms": [ + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "MediaFoundationAsyncH264Encoding" + ] + } + ] + } + ], + "MediaFoundationVP8Decoding": [ + { + "platforms": [ + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "MediaFoundationVP8Decoding" + ] + } + ] + } + ], + "MediaHistory": [ { "platforms": [ "android", @@ -3406,12 +4023,15 @@ ], "experiments": [ { - "name": "DBWindow5000Frames_20190225" + "name": "Enabled", + "enable_features": [ + "UseMediaHistoryStore" + ] } ] } ], - "MediaHistory": [ + "MediaLearningSmoothnessExperiment": [ { "platforms": [ "android", @@ -3424,7 +4044,8 @@ { "name": "Enabled", "enable_features": [ - "UseMediaHistoryStore" + "MediaLearningFramework", + "MediaLearningSmoothnessExperiment" ] } ] @@ -3483,6 +4104,25 @@ ] } ], + "MojoDedicatedThread": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "MojoDedicatedThread" + ] + } + ] + } + ], "MyChromeEverywhere": [ { "platforms": [ @@ -3661,6 +4301,21 @@ ] } ], + "NewUsbBackend": [ + { + "platforms": [ + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "NewUsbBackend" + ] + } + ] + } + ], "NotificationTimeouts": [ { "platforms": [ @@ -3676,6 +4331,25 @@ ] } ], + "NtpWebUIDesktop": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "DisableSearchSuggestChips", + "NtpWebUI" + ] + } + ] + } + ], "OfferUploadCreditCards": [ { "platforms": [ @@ -3696,6 +4370,25 @@ ] } ], + "OfflineCapabilityCheckForInstallablePrompt": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "CheckOfflineCapability" + ] + } + ] + } + ], "OfflineIndicator": [ { "platforms": [ @@ -3792,7 +4485,6 @@ }, "enable_features": [ "NtpRealbox", - "NtpRealboxMatchOmniboxTheme", "OmniboxDisplayTitleForCurrentUrl", "OmniboxDocumentProvider", "OmniboxHistoryQuickProviderAllowButDoNotScoreMidwordTerms", @@ -3803,7 +4495,8 @@ "OmniboxRichEntitySuggestions", "OmniboxUIExperimentMaxAutocompleteMatches", "OmniboxZeroSuggestionsOnNTP", - "QueryInOmnibox" + "QueryInOmnibox", + "SearchSuggestChips" ] } ] @@ -3822,7 +4515,6 @@ "rounded_edges": "true" }, "enable_features": [ - "OmniboxDisableInstantExtendedLimit", "OmniboxHistoryQuickProviderAllowButDoNotScoreMidwordTerms", "OmniboxHistoryQuickProviderAllowMidwordContinuations", "OmniboxOnFocusSuggestions" @@ -3843,6 +4535,7 @@ "ZeroSuggestVariant:8:*": "RemoteNoUrl" }, "enable_features": [ + "OmniboxDisableInstantExtendedLimit", "OmniboxHistoryQuickProviderAllowButDoNotScoreMidwordTerms", "OmniboxHistoryQuickProviderAllowMidwordContinuations", "OmniboxOnFocusSuggestions" @@ -3851,6 +4544,42 @@ ] } ], + "OmniboxContextMenuShowFullUrls": [ + { + "platforms": [ + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "OmniboxContextMenuShowFullUrls" + ] + } + ] + } + ], + "OmniboxEnableClipboardProviderImageSuggestions": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Enabled", + "params": { + "ImageSearchSuggestionThumbnail": "true" + }, + "enable_features": [ + "OmniboxEnableClipboardProviderImageSuggestions" + ] + } + ] + } + ], "OmniboxLocalEntities": [ { "platforms": [ @@ -3995,21 +4724,16 @@ ] } ], - "OptimizationTargetPrediction": [ + "OmniboxSuggestionsRecyclerView": [ { "platforms": [ - "android", - "android_weblayer" + "android" ], "experiments": [ { "name": "Enabled", - "params": { - "fetch_random_max_delay_secs": "60", - "max_effective_connection_type_for_navigation_hints_fetch": "4G" - }, "enable_features": [ - "OptimizationTargetPrediction" + "OmniboxSuggestionsRecyclerView" ] } ] @@ -4097,6 +4821,26 @@ ] } ], + "PaintHoldingCrossOrigin": [ + { + "platforms": [ + "android", + "android_weblayer", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Disabled", + "disable_features": [ + "PaintHoldingCrossOrigin" + ] + } + ] + } + ], "ParallelDownloadWithQUICAndHTTP2": [ { "platforms": [ @@ -4174,12 +4918,13 @@ ], "experiments": [ { - "name": "WithEnhancedExperiment", + "name": "WithEnhancedExperiment_20200616", + "params": { + "DefaultInIncognito": "true" + }, "enable_features": [ - "PasswordCheck", - "PrivacySettingsRedesign", - "SafeBrowsingEnhancedProtection", - "SyncSetupFriendlySettings" + "ImprovedCookieControls", + "ImprovedCookieControlsForThirdPartyCookieBlocking" ] } ] @@ -4335,6 +5080,25 @@ ] } ], + "PersistentHistogramsLocalMemory": [ + { + "platforms": [ + "android_webview" + ], + "experiments": [ + { + "name": "EnabledOnLocalMemory", + "params": { + "send_unreported_metrics": "yes", + "storage": "LocalMemory" + }, + "enable_features": [ + "PersistentHistograms" + ] + } + ] + } + ], "PlayStoreAppSearch": [ { "platforms": [ @@ -4468,20 +5232,16 @@ ] } ], - "PrerenderFallbackToPreconnect": [ + "PreventMessagePumpHangs": [ { "platforms": [ - "android", - "chromeos", - "linux", - "mac", "windows" ], "experiments": [ { - "name": "Enabled_20190410", + "name": "PreventMessagePumpHangs", "enable_features": [ - "PrerenderFallbackToPreconnect" + "PreventMessagePumpHangs" ] } ] @@ -4639,8 +5399,9 @@ { "name": "Enabled", "params": { - "connection_options": "5RTO,ACKD", - "enable_quic": "true" + "connection_options": "5RTO", + "enable_quic": "true", + "quic_version": "h3-Q050" }, "enable_features": [ "QuicDoesNotUseFeatures" @@ -4664,6 +5425,81 @@ ] } ], + "QueryTiles": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "EnabledStitch", + "params": { + "base_url": "https://chromeupboarding-pa.googleapis.com", + "experiment_tag": "", + "image_prefetch_mode": "top", + "most_visited_max_rows_normal_screen": "1", + "most_visited_max_rows_small_screen": "1" + }, + "enable_features": [ + "QueryTiles", + "QueryTilesInOmnibox" + ], + "disable_features": [ + "QueryTilesEnableQueryEditing" + ] + }, + { + "name": "EnabledStitchWithNoSearchParam", + "params": { + "base_url": "https://chromeupboarding-pa.googleapis.com", + "experiment_tag": "disableSearchParams : true", + "image_prefetch_mode": "top", + "most_visited_max_rows_normal_screen": "1", + "most_visited_max_rows_small_screen": "1" + }, + "enable_features": [ + "QueryTiles", + "QueryTilesInOmnibox" + ], + "disable_features": [ + "QueryTilesEnableQueryEditing" + ] + }, + { + "name": "EnabledExplore", + "params": { + "base_url": "https://chromeupboarding-pa.googleapis.com", + "experiment_tag": "maxLevels : 1", + "image_prefetch_mode": "top", + "most_visited_max_rows_normal_screen": "1", + "most_visited_max_rows_small_screen": "1" + }, + "enable_features": [ + "QueryTiles", + "QueryTilesInOmnibox" + ], + "disable_features": [ + "QueryTilesEnableQueryEditing" + ] + }, + { + "name": "EnabledStart", + "params": { + "base_url": "https://chromeupboarding-pa.googleapis.com", + "experiment_tag": "maxLevels : 1", + "image_prefetch_mode": "top", + "most_visited_max_rows_normal_screen": "1", + "most_visited_max_rows_small_screen": "1" + }, + "enable_features": [ + "QueryTiles", + "QueryTilesEnableQueryEditing", + "QueryTilesInOmnibox" + ] + } + ] + } + ], "QuietNotificationPrompts": [ { "platforms": [ @@ -4798,23 +5634,25 @@ ] } ], - "RendererSchedulerWakeUpThrottling": [ + "RenderDocument": [ { "platforms": [ "android", - "android_weblayer", + "android_webview", "chromeos", - "ios", "linux", "mac", "windows" ], "experiments": [ { - "name": "RendererSchedulerWakeUpThrottling", + "name": "EnableForCrashedFrame", "params": { - "wake_up_duration_ms": "5" - } + "level": "crashed-frame" + }, + "enable_features": [ + "RenderDocument" + ] } ] } @@ -4967,6 +5805,21 @@ ] } ], + "RespectMacLCDTextSetting": [ + { + "platforms": [ + "mac" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "RespectMacLCDTextSetting" + ] + } + ] + } + ], "RevampedContextMenu": [ { "platforms": [ @@ -5520,7 +6373,7 @@ ] } ], - "ServiceWorkerStartupOptimizations": [ + "ServiceWorkerOnUI": [ { "platforms": [ "android", @@ -5533,28 +6386,14 @@ ], "experiments": [ { - "name": "Enabled_Ui_Ps3", + "name": "Enabled1", "enable_features": [ - "ServiceWorkerOnUI", - "ServiceWorkerPrefersUnusedProcess" + "ServiceWorkerOnUI" ] } ] } ], - "SettingsEnforcement": [ - { - "platforms": [ - "mac", - "windows" - ], - "experiments": [ - { - "name": "enforce_always_with_extensions_and_dse" - } - ] - } - ], "SettingsResetPrompt": [ { "platforms": [ @@ -5605,9 +6444,10 @@ ], "experiments": [ { - "name": "Enabled", + "name": "Enabled_With_WebRTC_2020-05-21", "enable_features": [ - "SharedClipboardUI" + "SharedClipboardUI", + "SharingPeerConnectionReceiver" ] } ] @@ -5637,24 +6477,6 @@ ] } ], - "SharingWebRTC": [ - { - "platforms": [ - "chromeos", - "linux", - "mac", - "windows" - ], - "experiments": [ - { - "name": "Enabled", - "enable_features": [ - "SharingPeerConnectionReceiver" - ] - } - ] - } - ], "Show109ObsoleteInfobar": [ { "platforms": [ @@ -5704,6 +6526,21 @@ ] } ], + "SmartDimNewMlAgent": [ + { + "platforms": [ + "chromeos" + ], + "experiments": [ + { + "name": "Enabled_20200609", + "enable_features": [ + "SmartDimNewMlAgent" + ] + } + ] + } + ], "SplitCacheByNetworkIsolationKey": [ { "platforms": [ @@ -5788,65 +6625,60 @@ ] } ], - "SyncInstanceIDTokenTTL": [ + "SyncErrorInfoBarTesting": [ { "platforms": [ - "chromeos", - "ios", - "linux", - "mac", - "windows" + "android" ], "experiments": [ { "name": "Enabled", - "params": { - "time_to_live_seconds": "1209600" - }, "enable_features": [ - "SyncInstanceIDTokenTTL" + "SyncErrorInfoBarAndroid" ] } ] } ], - "SyncRefreshDeviceInfo": [ + "SyncInstanceIDTokenTTL": [ { "platforms": [ - "android", "chromeos", + "ios", "linux", "mac", "windows" ], "experiments": [ { - "name": "Enabled_1_hour", + "name": "Enabled", "params": { - "PulseIntervalMinutes": "60" + "time_to_live_seconds": "1209600" }, "enable_features": [ - "PulseInterval" + "SyncInstanceIDTokenTTL" ] } ] } ], - "SyncUSSNigori": [ + "SyncRefreshDeviceInfo": [ { "platforms": [ "android", "chromeos", - "ios", "linux", "mac", "windows" ], "experiments": [ { - "name": "Enabled", + "name": "Enabled_1_hour", + "params": { + "PulseIntervalMinutes": "60" + }, "enable_features": [ - "SyncUSSNigori" + "PulseInterval" ] } ] @@ -6110,6 +6942,7 @@ "android", "android_webview", "chromeos", + "ios", "linux", "mac", "windows" @@ -6164,6 +6997,28 @@ ] } ], + "ThrottleInstallingServiceWorker": [ + { + "platforms": [ + "windows", + "mac", + "chromeos", + "linux", + "android" + ], + "experiments": [ + { + "name": "Emabled5_v1", + "params": { + "limit": "5" + }, + "enable_features": [ + "ThrottleInstallingServiceWorker" + ] + } + ] + } + ], "TopCat": [ { "platforms": [ @@ -6283,6 +7138,22 @@ ] } ], + "TranslateSubFramesStudy": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "TranslateSubFrames" + } + ] + } + ], "TriggeredResetFieldTrial": [ { "platforms": [ @@ -6295,6 +7166,28 @@ ] } ], + "TrustTokenOriginTrial": [ + { + "platforms": [ + "android", + "chromeos", + "linux", + "mac", + "windows" + ], + "experiments": [ + { + "name": "Enabled", + "params": { + "TrustTokenOperationsRequiringOriginTrial": "only-issuance-requires-origin-trial" + }, + "enable_features": [ + "TrustTokens" + ] + } + ] + } + ], "TrustedTypes": [ { "platforms": [ @@ -6317,19 +7210,16 @@ ] } ], - "UnidoOnSignInV2": [ + "UnidoPreferSync": [ { "platforms": [ - "android", - "android_weblayer" + "android" ], "experiments": [ { - "name": "Enabled_2020-05-13", - "enable_features": [ - "SharingPreferVapid", - "SharingSendViaSync", - "SyncDeviceInfoInTransportMode" + "name": "Enabled", + "disable_features": [ + "SharingPreferVapid" ] } ] @@ -6416,11 +7306,17 @@ "UsePreferredIntervalForVideo": [ { "platforms": [ - "chromeos" + "chromeos", + "mac", + "windows", + "linux" ], "experiments": [ { "name": "Enabled", + "params": { + "NumOfFramesToToggleInterval": "6" + }, "enable_features": [ "UsePreferredIntervalForVideo" ] @@ -6552,66 +7448,6 @@ ] } ], - "V8WasmLiftoffMobile": [ - { - "platforms": [ - "android", - "android_weblayer", - "android_webview", - "ios" - ], - "experiments": [ - { - "name": "Tiering", - "enable_features": [ - "WebAssemblyBaseline", - "WebAssemblyTiering" - ], - "disable_features": [ - "WebAssemblyLazyCompilation" - ] - }, - { - "name": "LazyLiftoff", - "enable_features": [ - "WebAssemblyBaseline", - "WebAssemblyLazyCompilation" - ], - "disable_features": [ - "WebAssemblyTiering" - ] - }, - { - "name": "LazyTiering", - "enable_features": [ - "WebAssemblyBaseline", - "WebAssemblyLazyCompilation", - "WebAssemblyTiering" - ] - }, - { - "name": "Liftoff", - "enable_features": [ - "WebAssemblyBaseline" - ], - "disable_features": [ - "WebAssemblyLazyCompilation", - "WebAssemblyTiering" - ] - }, - { - "name": "TurboFan", - "enable_features": [ - "WebAssemblyTiering" - ], - "disable_features": [ - "WebAssemblyBaseline", - "WebAssemblyLazyCompilation" - ] - } - ] - } - ], "VaapiH264AMDEncoder": [ { "platforms": [ @@ -6713,6 +7549,22 @@ ] } ], + "Vulkan": [ + { + "platforms": [ + "android" + ], + "experiments": [ + { + "name": "Vulkan", + "enable_features": [ + "Vulkan" + ], + "min_os_version": "10.0.0" + } + ] + } + ], "WKHTTPSystemCookieStore": [ { "platforms": [ @@ -7159,6 +8011,21 @@ ] } ], + "WebViewCollectNonEmbeddedMetrics": [ + { + "platforms": [ + "android_webview" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "WebViewCollectNonEmbeddedMetrics" + ] + } + ] + } + ], "WebXrPermissionsApi": [ { "platforms": [ @@ -7184,8 +8051,7 @@ { "name": "Enabled", "enable_features": [ - "WinUseBrowserSpellChecker", - "WinUseHybridSpellChecker" + "WinUseBrowserSpellChecker" ] } ] @@ -7206,6 +8072,21 @@ ] } ], + "iOSQRCodeGenerator": [ + { + "platforms": [ + "ios" + ], + "experiments": [ + { + "name": "Enabled", + "enable_features": [ + "QRCodeGeneration" + ] + } + ] + } + ], "libvpx_for_vp8": [ { "platforms": [ diff --git a/chromium/testing/xvfb.py b/chromium/testing/xvfb.py index cabfd6985bf..d868072aa38 100755 --- a/chromium/testing/xvfb.py +++ b/chromium/testing/xvfb.py @@ -142,6 +142,7 @@ def _run_with_xvfb(cmd, env, stdoutfile, use_openbox, use_xcompmgr): openbox_proc = None xcompmgr_proc = None xvfb_proc = None + xwmstartupcheck_proc = None xvfb_ready = MutableBoolean() def set_xvfb_ready(*_): xvfb_ready.setvalue(True) @@ -181,8 +182,34 @@ def _run_with_xvfb(cmd, env, stdoutfile, use_openbox, use_xcompmgr): dbus_pid = launch_dbus(env) if use_openbox: + # This is not ideal, but x11_unittests require that (other X11 tests have + # a race with the openbox as well, but they take more time to initialize. + # And thus, they do no time out compate to the x11_unittests that are + # quick enough to start up before openbox is ready. + # TODO(dpranke): remove this nasty hack once the test() template is + # reworked. + wait_for_openbox = False + wait_openbox_program = './xwmstartupcheck' + if not os.path.isfile(wait_openbox_program): + wait_for_openbox = False + # Creates a dummy window that waits for a ReparentNotify event that is + # sent whenever Openbox WM starts. Must be started before the OpenBox WM + # so that it does not miss the event. This helper program is located in + # the current build directory. The program terminates automatically after + # 30 seconds of waiting for the event. + if wait_for_openbox: + xwmstartupcheck_proc = subprocess.Popen( + wait_openbox_program, stderr=subprocess.STDOUT, env=env) + openbox_proc = subprocess.Popen( - 'openbox', stderr=subprocess.STDOUT, env=env) + ['openbox', '--sm-disable'], stderr=subprocess.STDOUT, env=env) + + # Wait until execution is done. Does not block if the process has already + # been terminated. In that case, it's safe to read the return value. + if wait_for_openbox: + xwmstartupcheck_proc.wait() + if xwmstartupcheck_proc.returncode is not 0: + raise _XvfbProcessError('Failed to get OpenBox up.') if use_xcompmgr: xcompmgr_proc = subprocess.Popen( @@ -230,8 +257,12 @@ def _run_with_weston(cmd, env, stdoutfile): # to enter idle state. Otherwise, Weston stops to send frame callbacks, # and tests start to time out (this typically happens after 300 seconds - # the default time after which Weston enters the idle state). + # 3) --width && --height set size of a virtual display: we need to set + # an adequate size so that tests can have more room for managing size + # of windows. weston_proc = subprocess.Popen( - ('./weston', '--backend=headless-backend.so', '--idle-time=0'), + ('./weston', '--backend=headless-backend.so', '--idle-time=0', + '--width=1024', '--height=768'), stderr=subprocess.STDOUT, env=env) # Get the $WAYLAND_DISPLAY set by Weston and pass it to the test launcher. |