diff options
author | Allan Sandfeld Jensen <allan.jensen@theqtcompany.com> | 2016-05-09 14:22:11 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2016-05-09 15:11:45 +0000 |
commit | 2ddb2d3e14eef3de7dbd0cef553d669b9ac2361c (patch) | |
tree | e75f511546c5fd1a173e87c1f9fb11d7ac8d1af3 /chromium/components/domain_reliability | |
parent | a4f3d46271c57e8155ba912df46a05559d14726e (diff) | |
download | qtwebengine-chromium-2ddb2d3e14eef3de7dbd0cef553d669b9ac2361c.tar.gz |
BASELINE: Update Chromium to 51.0.2704.41
Also adds in all smaller components by reversing logic for exclusion.
Change-Id: Ibf90b506e7da088ea2f65dcf23f2b0992c504422
Reviewed-by: Joerg Bornemann <joerg.bornemann@theqtcompany.com>
Diffstat (limited to 'chromium/components/domain_reliability')
63 files changed, 7768 insertions, 1 deletions
diff --git a/chromium/components/domain_reliability/BUILD.gn b/chromium/components/domain_reliability/BUILD.gn new file mode 100644 index 00000000000..5834a824bcb --- /dev/null +++ b/chromium/components/domain_reliability/BUILD.gn @@ -0,0 +1,115 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Paths to the JSON files are kind of gross. They're stored in the gypi +# relative to //components, since that's the working directory gyp seems +# to use for all of the components. When we depend on them here, we need +# to remove the leading domain_reliability, since *our* working directory +# is one level deeper. When we call bake_in_configs.py, we need to give +# it a properly-rebased path to //components so it can properly join the +# paths relative to that and find the JSON files. + +baked_in_configs_gypi = exec_script("//build/gypi_to_gn.py", + [ rebase_path("baked_in_configs.gypi") ], + "scope", + [ "baked_in_configs.gypi" ]) + +# The config file names in the .gypi are relative to "//components". +baked_in_configs = + rebase_path(baked_in_configs_gypi.baked_in_configs, ".", "//components") + +action("bake_in_configs") { + visibility = [ ":*" ] + script = "bake_in_configs.py" + + inputs = baked_in_configs + [ "baked_in_configs.gypi" ] + output_file = "$target_gen_dir/baked_in_configs.cc" + outputs = [ + output_file, + ] + + # The JSON file list is too long for the command line on Windows, so put + # them in a response file. + response_file_contents = rebase_path(baked_in_configs, root_build_dir) + args = [ + "--file-list", + "{{response_file_name}}", + "--output", + rebase_path(output_file, root_build_dir), + ] +} + +component("domain_reliability") { + sources = [ + "baked_in_configs.h", + "beacon.cc", + "beacon.h", + "clear_mode.h", + "config.cc", + "config.h", + "context.cc", + "context.h", + "context_manager.cc", + "context_manager.h", + "dispatcher.cc", + "dispatcher.h", + "domain_reliability_export.h", + "google_configs.cc", + "google_configs.h", + "header.cc", + "header.h", + "monitor.cc", + "monitor.h", + "quic_error_mapping.cc", + "quic_error_mapping.h", + "scheduler.cc", + "scheduler.h", + "service.cc", + "service.h", + "uploader.cc", + "uploader.h", + "util.cc", + "util.h", + ] + sources += get_target_outputs(":bake_in_configs") + + defines = [ "DOMAIN_RELIABILITY_IMPLEMENTATION" ] + + deps = [ + ":bake_in_configs", + "//base", + "//components/data_use_measurement/core", + "//components/keyed_service/core", + "//content/public/common", + "//net", + "//url", + ] +} + +source_set("unit_tests") { + testonly = true + sources = [ + "config_unittest.cc", + "context_unittest.cc", + "dispatcher_unittest.cc", + "google_configs_unittest.cc", + "header_unittest.cc", + "monitor_unittest.cc", + "scheduler_unittest.cc", + "test_util.cc", + "test_util.h", + "uploader_unittest.cc", + "util_unittest.cc", + ] + + configs += [ "//build/config/compiler:no_size_t_to_int_warning" ] + + deps = [ + ":domain_reliability", + "//base", + "//base/test:test_support", + "//net:test_support", + "//testing/gtest", + ] +} diff --git a/chromium/components/domain_reliability/DEPS b/chromium/components/domain_reliability/DEPS new file mode 100644 index 00000000000..b5ddd354e6a --- /dev/null +++ b/chromium/components/domain_reliability/DEPS @@ -0,0 +1,11 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +include_rules = [ + "+net", + "+components/data_use_measurement/core", + "+components/keyed_service/core", + "+content/public/common", +] + diff --git a/chromium/components/domain_reliability/OWNERS b/chromium/components/domain_reliability/OWNERS new file mode 100644 index 00000000000..6abaef79054 --- /dev/null +++ b/chromium/components/domain_reliability/OWNERS @@ -0,0 +1,6 @@ +davidben@chromium.org +rdsmith@chromium.org +ttuttle@chromium.org + +per-file quic_error_mapping*=rch@chromium.org +per-file quic_error_mapping*=rtenneti@chromium.org diff --git a/chromium/components/domain_reliability/bake_in_configs.py b/chromium/components/domain_reliability/bake_in_configs.py new file mode 100755 index 00000000000..a6fa0ed6caf --- /dev/null +++ b/chromium/components/domain_reliability/bake_in_configs.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Takes the JSON files in components/domain_reliability/baked_in_configs and +encodes their contents as an array of C strings that gets compiled in to Chrome +and loaded at runtime.""" + + +import ast +import json +import optparse +import os +import shlex +import sys + + +# A whitelist of domains that the script will accept when baking configs in to +# Chrome, to ensure incorrect ones are not added accidentally. Subdomains of +# whitelist entries are also allowed (e.g. maps.google.com, ssl.gstatic.com). +DOMAIN_WHITELIST = ( + '2mdn.net', + 'admob.biz', + 'admob.co.in', + 'admob.co.kr', + 'admob.co.nz', + 'admob.co.uk', + 'admob.co.za', + 'admob.com', + 'admob.com.br', + 'admob.com.es', + 'admob.com.fr', + 'admob.com.mx', + 'admob.com.pt', + 'admob.de', + 'admob.dk', + 'admob.es', + 'admob.fi', + 'admob.fr', + 'admob.gr', + 'admob.hk', + 'admob.ie', + 'admob.in', + 'admob.it', + 'admob.jp', + 'admob.kr', + 'admob.mobi', + 'admob.no', + 'admob.ph', + 'admob.pt', + 'admob.sg', + 'admob.tw', + 'admob.us', + 'admob.vn', + 'dartmotif.com', + 'doubleclick.com', + 'doubleclick.ne.jp', + 'doubleclick.net', + 'doubleclickusercontent.com', + 'g.co', + 'ggpht.com', + 'gmodules.com', + 'goo.gl', + 'google-analytics.com', + 'google-syndication.com', + 'google.ac', + 'google.ad', + 'google.ae', + 'google.af', + 'google.ag', + 'google.al', + 'google.am', + 'google.as', + 'google.at', + 'google.az', + 'google.ba', + 'google.be', + 'google.bf', + 'google.bg', + 'google.bi', + 'google.bj', + 'google.bs', + 'google.bt', + 'google.by', + 'google.ca', + 'google.cat', + 'google.cc', + 'google.cd', + 'google.cf', + 'google.cg', + 'google.ch', + 'google.ci', + 'google.cl', + 'google.cm', + 'google.cn', + 'google.co.ao', + 'google.co.bw', + 'google.co.ck', + 'google.co.cr', + 'google.co.hu', + 'google.co.id', + 'google.co.il', + 'google.co.im', + 'google.co.in', + 'google.co.je', + 'google.co.jp', + 'google.co.ke', + 'google.co.kr', + 'google.co.ls', + 'google.co.ma', + 'google.co.mz', + 'google.co.nz', + 'google.co.th', + 'google.co.tz', + 'google.co.ug', + 'google.co.uk', + 'google.co.uz', + 'google.co.ve', + 'google.co.vi', + 'google.co.za', + 'google.co.zm', + 'google.co.zw', + 'google.com', + 'google.com.af', + 'google.com.ag', + 'google.com.ai', + 'google.com.ar', + 'google.com.au', + 'google.com.bd', + 'google.com.bh', + 'google.com.bn', + 'google.com.bo', + 'google.com.br', + 'google.com.by', + 'google.com.bz', + 'google.com.cn', + 'google.com.co', + 'google.com.cu', + 'google.com.cy', + 'google.com.do', + 'google.com.ec', + 'google.com.eg', + 'google.com.et', + 'google.com.fj', + 'google.com.ge', + 'google.com.gh', + 'google.com.gi', + 'google.com.gr', + 'google.com.gt', + 'google.com.hk', + 'google.com.iq', + 'google.com.jm', + 'google.com.jo', + 'google.com.kh', + 'google.com.kw', + 'google.com.lb', + 'google.com.ly', + 'google.com.mm', + 'google.com.mt', + 'google.com.mx', + 'google.com.my', + 'google.com.na', + 'google.com.nf', + 'google.com.ng', + 'google.com.ni', + 'google.com.np', + 'google.com.nr', + 'google.com.om', + 'google.com.pa', + 'google.com.pe', + 'google.com.pg', + 'google.com.ph', + 'google.com.pk', + 'google.com.pl', + 'google.com.pr', + 'google.com.py', + 'google.com.qa', + 'google.com.ru', + 'google.com.sa', + 'google.com.sb', + 'google.com.sg', + 'google.com.sl', + 'google.com.sv', + 'google.com.tj', + 'google.com.tn', + 'google.com.tr', + 'google.com.tw', + 'google.com.ua', + 'google.com.uy', + 'google.com.vc', + 'google.com.ve', + 'google.com.vn', + 'google.cv', + 'google.cz', + 'google.de', + 'google.dj', + 'google.dk', + 'google.dm', + 'google.dz', + 'google.ee', + 'google.es', + 'google.fi', + 'google.fm', + 'google.fr', + 'google.ga', + 'google.ge', + 'google.gg', + 'google.gl', + 'google.gm', + 'google.gp', + 'google.gr', + 'google.gy', + 'google.hk', + 'google.hn', + 'google.hr', + 'google.ht', + 'google.hu', + 'google.ie', + 'google.im', + 'google.info', + 'google.iq', + 'google.ir', + 'google.is', + 'google.it', + 'google.it.ao', + 'google.je', + 'google.jo', + 'google.jobs', + 'google.jp', + 'google.kg', + 'google.ki', + 'google.kz', + 'google.la', + 'google.li', + 'google.lk', + 'google.lt', + 'google.lu', + 'google.lv', + 'google.md', + 'google.me', + 'google.mg', + 'google.mk', + 'google.ml', + 'google.mn', + 'google.ms', + 'google.mu', + 'google.mv', + 'google.mw', + 'google.ne', + 'google.ne.jp', + 'google.net', + 'google.ng', + 'google.nl', + 'google.no', + 'google.nr', + 'google.nu', + 'google.off.ai', + 'google.org', + 'google.pk', + 'google.pl', + 'google.pn', + 'google.ps', + 'google.pt', + 'google.ro', + 'google.rs', + 'google.ru', + 'google.rw', + 'google.sc', + 'google.se', + 'google.sh', + 'google.si', + 'google.sk', + 'google.sm', + 'google.sn', + 'google.so', + 'google.sr', + 'google.st', + 'google.td', + 'google.tg', + 'google.tk', + 'google.tl', + 'google.tm', + 'google.tn', + 'google.to', + 'google.tt', + 'google.us', + 'google.uz', + 'google.vg', + 'google.vu', + 'google.ws', + 'googleadservices.com', + 'googleadsserving.cn', + 'googlealumni.com', + 'googleapis.com', + 'googleapps.com', + 'googlecbs.com', + 'googlecommerce.com', + 'googledrive.com', + 'googleenterprise.com', + 'googlegoro.com', + 'googlehosted.com', + 'googlepayments.com', + 'googlesource.com', + 'googlesyndication.com', + 'googletagmanager.com', + 'googletagservices.com', + 'googleusercontent.com', + 'googlevideo.com', + 'gstatic.com', + 'gvt1.com', + 'gvt2.com', + 'withgoogle.com', + 'youtu.be', + 'youtube-3rd-party.com', + 'youtube-nocookie.com', + 'youtube.ae', + 'youtube.al', + 'youtube.am', + 'youtube.at', + 'youtube.az', + 'youtube.ba', + 'youtube.be', + 'youtube.bg', + 'youtube.bh', + 'youtube.bo', + 'youtube.ca', + 'youtube.cat', + 'youtube.ch', + 'youtube.cl', + 'youtube.co', + 'youtube.co.ae', + 'youtube.co.at', + 'youtube.co.hu', + 'youtube.co.id', + 'youtube.co.il', + 'youtube.co.in', + 'youtube.co.jp', + 'youtube.co.ke', + 'youtube.co.kr', + 'youtube.co.ma', + 'youtube.co.nz', + 'youtube.co.th', + 'youtube.co.ug', + 'youtube.co.uk', + 'youtube.co.ve', + 'youtube.co.za', + 'youtube.com', + 'youtube.com.ar', + 'youtube.com.au', + 'youtube.com.az', + 'youtube.com.bh', + 'youtube.com.bo', + 'youtube.com.br', + 'youtube.com.by', + 'youtube.com.co', + 'youtube.com.do', + 'youtube.com.ee', + 'youtube.com.eg', + 'youtube.com.es', + 'youtube.com.gh', + 'youtube.com.gr', + 'youtube.com.gt', + 'youtube.com.hk', + 'youtube.com.hr', + 'youtube.com.jm', + 'youtube.com.jo', + 'youtube.com.kw', + 'youtube.com.lb', + 'youtube.com.lv', + 'youtube.com.mk', + 'youtube.com.mt', + 'youtube.com.mx', + 'youtube.com.my', + 'youtube.com.ng', + 'youtube.com.om', + 'youtube.com.pe', + 'youtube.com.ph', + 'youtube.com.pk', + 'youtube.com.pt', + 'youtube.com.qa', + 'youtube.com.ro', + 'youtube.com.sa', + 'youtube.com.sg', + 'youtube.com.tn', + 'youtube.com.tr', + 'youtube.com.tw', + 'youtube.com.ua', + 'youtube.com.uy', + 'youtube.com.ve', + 'youtube.cz', + 'youtube.de', + 'youtube.dk', + 'youtube.ee', + 'youtube.es', + 'youtube.fi', + 'youtube.fr', + 'youtube.ge', + 'youtube.gr', + 'youtube.gt', + 'youtube.hk', + 'youtube.hr', + 'youtube.hu', + 'youtube.ie', + 'youtube.in', + 'youtube.is', + 'youtube.it', + 'youtube.jo', + 'youtube.jp', + 'youtube.kr', + 'youtube.lk', + 'youtube.lt', + 'youtube.lv', + 'youtube.ma', + 'youtube.md', + 'youtube.me', + 'youtube.mk', + 'youtube.mx', + 'youtube.my', + 'youtube.ng', + 'youtube.nl', + 'youtube.no', + 'youtube.pe', + 'youtube.ph', + 'youtube.pk', + 'youtube.pl', + 'youtube.pr', + 'youtube.pt', + 'youtube.qa', + 'youtube.ro', + 'youtube.rs', + 'youtube.ru', + 'youtube.sa', + 'youtube.se', + 'youtube.sg', + 'youtube.si', + 'youtube.sk', + 'youtube.sn', + 'youtube.tn', + 'youtube.ua', + 'youtube.ug', + 'youtube.uy', + 'youtube.vn', + 'youtubeeducation.com', + 'youtubemobilesupport.com', + 'ytimg.com' +) + + +CC_HEADER = """// AUTOGENERATED FILE. DO NOT EDIT. +// +// (Update configs in components/domain_reliability/baked_in_configs and list +// configs in components/domain_reliability/baked_in_configs.gypi instead.) + +#include "components/domain_reliability/baked_in_configs.h" + +#include <stdlib.h> + +namespace domain_reliability { + +const char* const kBakedInJsonConfigs[] = { +""" + + +CC_FOOTER = """ nullptr +}; + +} // namespace domain_reliability +""" + + +def read_json_files_from_gypi(gypi_file): + with open(gypi_file, 'r') as f: + gypi_text = f.read() + json_files = ast.literal_eval(gypi_text)['variables']['baked_in_configs'] + return json_files + + +def read_json_files_from_file(list_file): + with open(list_file, 'r') as f: + list_text = f.read() + return shlex.split(list_text) + + +def origin_is_whitelisted(origin): + if origin.startswith('https://') and origin.endswith('/'): + domain = origin[8:-1] + else: + return False + return any(domain == e or domain.endswith('.' + e) for e in DOMAIN_WHITELIST) + + +def quote_and_wrap_text(text, width=79, prefix=' "', suffix='"'): + max_length = width - len(prefix) - len(suffix) + output = prefix + line_length = 0 + for c in text: + if c == "\"": + c = "\\\"" + elif c == "\n": + c = "\\n" + elif c == "\\": + c = "\\\\" + if line_length + len(c) > max_length: + output += suffix + "\n" + prefix + line_length = 0 + output += c + line_length += len(c) + output += suffix + return output + + +def main(): + parser = optparse.OptionParser(usage="bake_in_configs.py [options]") + parser.add_option("", "--output", metavar="FILE", + help="[Required] Name of the .cc file to write.") + + # For response file reading. + parser.add_option("", "--file-list", metavar="FILE", + help="File containing whitespace separated names of " + "the baked in configs files.") + + # For .gypi file reading. + parser.add_option("", "--gypi-file", metavar="FILE", + help=".gypi file containing baked_in_configs variable.") + parser.add_option("", "--gypi-relative-to", metavar="PATH", + help="Directory the baked_in_configs in the --gypi-file" + "are relative to.""") + + opts, args = parser.parse_args() + + if not opts.output: + print >> sys.stderr, "--output argument required" + return 1 + + if opts.gypi_file: + # .gypi-style input. + if not opts.gypi_relative_to: + print >> sys.stderr, "--gypi-relative-to is required with --gypi-file" + return 1 + json_files = read_json_files_from_gypi(opts.gypi_file) + json_files = [ os.path.join(opts.gypi_relative_to, f) for f in json_files ] + json_files = [ os.path.normpath(f) for f in json_files ] + elif opts.file_list: + # Regular file list input. + json_files = read_json_files_from_file(opts.file_list) + else: + print >> sys.stderr, "Either --file-list or --gypi-file is required." + return 1 + + cpp_code = CC_HEADER + found_invalid_config = False + + for json_file in json_files: + with open(json_file, 'r') as f: + json_text = f.read() + try: + config = json.loads(json_text) + except ValueError, e: + print >> sys.stderr, "%s: error parsing JSON: %s" % (json_file, e) + found_invalid_config = True + continue + if 'origin' not in config: + print >> sys.stderr, '%s: no origin found' % json_file + found_invalid_config = True + continue + origin = config['origin'] + if not origin_is_whitelisted(origin): + print >> sys.stderr, ('%s: origin "%s" not in whitelist' % + (json_file, origin)) + found_invalid_config = True + continue + + # Re-dump JSON to get a more compact representation. + dumped_json_text = json.dumps(config, separators=(',', ':')) + + cpp_code += " // " + json_file + ":\n" + cpp_code += quote_and_wrap_text(dumped_json_text) + ",\n" + cpp_code += "\n" + + cpp_code += CC_FOOTER + + if found_invalid_config: + return 1 + + with open(opts.output, 'wb') as f: + f.write(cpp_code) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/chromium/components/domain_reliability/baked_in_configs.gypi b/chromium/components/domain_reliability/baked_in_configs.gypi index eebb3acac63..a872ef8792e 100644 --- a/chromium/components/domain_reliability/baked_in_configs.gypi +++ b/chromium/components/domain_reliability/baked_in_configs.gypi @@ -20,7 +20,6 @@ 'domain_reliability/baked_in_configs/googlevideo_com.json', 'domain_reliability/baked_in_configs/gvt1_com.json', 'domain_reliability/baked_in_configs/gvt2_com.json', - 'domain_reliability/baked_in_configs/mail_google_com.json', 'domain_reliability/baked_in_configs/ssl_gstatic_com.json', 'domain_reliability/baked_in_configs/www_google_com.json', ], diff --git a/chromium/components/domain_reliability/baked_in_configs.h b/chromium/components/domain_reliability/baked_in_configs.h new file mode 100644 index 00000000000..31f1d5de54c --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs.h @@ -0,0 +1,18 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_BAKED_IN_CONFIGS_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_BAKED_IN_CONFIGS_H_ + +#include "components/domain_reliability/domain_reliability_export.h" + +namespace domain_reliability { + +// NULL-terminated array of pointers to JSON-encoded Domain Reliability +// configurations. Read by DomainReliabilityMonitor::AddBakedInConfigs. +DOMAIN_RELIABILITY_EXPORT extern const char* const kBakedInJsonConfigs[]; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_BAKED_IN_CONFIGS_H_ diff --git a/chromium/components/domain_reliability/baked_in_configs/c_2mdn_net.json b/chromium/components/domain_reliability/baked_in_configs/c_2mdn_net.json new file mode 100644 index 00000000000..30d052eec30 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_2mdn_net.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.2mdn.net/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_android_clients_google_com.json b/chromium/components/domain_reliability/baked_in_configs/c_android_clients_google_com.json new file mode 100644 index 00000000000..5a2c8925e9f --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_android_clients_google_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.android.clients.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_bigcache_googleapis_com.json b/chromium/components/domain_reliability/baked_in_configs/c_bigcache_googleapis_com.json new file mode 100644 index 00000000000..265d64c6a55 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_bigcache_googleapis_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.bigcache.googleapis.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_doc-0-0-sj_sj_googleusercontent_com.json b/chromium/components/domain_reliability/baked_in_configs/c_doc-0-0-sj_sj_googleusercontent_com.json new file mode 100644 index 00000000000..4598487559b --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_doc-0-0-sj_sj_googleusercontent_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.doc-0-0-sj.sj.googleusercontent.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_docs_google_com.json b/chromium/components/domain_reliability/baked_in_configs/c_docs_google_com.json new file mode 100644 index 00000000000..b94b23a5c8c --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_docs_google_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.docs.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_drive_google_com.json b/chromium/components/domain_reliability/baked_in_configs/c_drive_google_com.json new file mode 100644 index 00000000000..b8b9ed93667 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_drive_google_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.drive.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_googlesyndication_com.json b/chromium/components/domain_reliability/baked_in_configs/c_googlesyndication_com.json new file mode 100644 index 00000000000..6ef17739192 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_googlesyndication_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.googlesyndication.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_pack_google_com.json b/chromium/components/domain_reliability/baked_in_configs/c_pack_google_com.json new file mode 100644 index 00000000000..e054887ad30 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_pack_google_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.pack.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_play_google_com.json b/chromium/components/domain_reliability/baked_in_configs/c_play_google_com.json new file mode 100644 index 00000000000..1cd6a34f195 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_play_google_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.play.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/c_youtube_com.json b/chromium/components/domain_reliability/baked_in_configs/c_youtube_com.json new file mode 100644 index 00000000000..8f0f35e583e --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/c_youtube_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://c.youtube.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/clients2_google_com.json b/chromium/components/domain_reliability/baked_in_configs/clients2_google_com.json new file mode 100644 index 00000000000..a3ec5dbd48d --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/clients2_google_com.json @@ -0,0 +1,20 @@ +{ + "origin": "https://clients2.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": false, + "path_prefixes": [ + "/domainreliability/*", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/docs_google_com.json b/chromium/components/domain_reliability/baked_in_configs/docs_google_com.json new file mode 100644 index 00000000000..01ec7e79c4d --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/docs_google_com.json @@ -0,0 +1,22 @@ +{ + "origin": "https://docs.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": false, + "path_prefixes": [ + "/*/document", + "/*/presentation", + "/*/spreadsheets", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/google-analytics_com.json b/chromium/components/domain_reliability/baked_in_configs/google-analytics_com.json new file mode 100644 index 00000000000..6bd103b7c74 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/google-analytics_com.json @@ -0,0 +1,22 @@ +{ + "origin": "https://google-analytics.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/analytics.js", + "/ga.js", + "/__utm.gif", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/googlevideo_com.json b/chromium/components/domain_reliability/baked_in_configs/googlevideo_com.json new file mode 100644 index 00000000000..4f4f27427a1 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/googlevideo_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://googlevideo.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/gvt1_com.json b/chromium/components/domain_reliability/baked_in_configs/gvt1_com.json new file mode 100644 index 00000000000..509639b59c0 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/gvt1_com.json @@ -0,0 +1,34 @@ +{ + "origin": "https://gvt1.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/videoplayback", + "/videochunk", + "/chunk_present", + "/cm2_state", + "/videolookup", + "/videogoodput", + "/MotifFiles", + "/StudioFiles", + "/packdata", + "/edgedl", + "/market", + "/packages", + "/crx", + "/initplayback", + "/initsegment", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/gvt2_com.json b/chromium/components/domain_reliability/baked_in_configs/gvt2_com.json new file mode 100644 index 00000000000..cc696b9a583 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/gvt2_com.json @@ -0,0 +1,20 @@ +{ + "origin": "https://gvt2.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": true, + "path_prefixes": [ + "/domainreliability/*", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/ssl_gstatic_com.json b/chromium/components/domain_reliability/baked_in_configs/ssl_gstatic_com.json new file mode 100644 index 00000000000..dc9897769ab --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/ssl_gstatic_com.json @@ -0,0 +1,22 @@ +{ + "origin": "https://ssl.gstatic.com/", + "has_same_origin_collector": false, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": false, + "path_prefixes": [ + "/accounts", + "/analytics", + "/images", + "" + ] +} diff --git a/chromium/components/domain_reliability/baked_in_configs/www_google_com.json b/chromium/components/domain_reliability/baked_in_configs/www_google_com.json new file mode 100644 index 00000000000..d1c9fdd7950 --- /dev/null +++ b/chromium/components/domain_reliability/baked_in_configs/www_google_com.json @@ -0,0 +1,28 @@ +{ + "origin": "https://www.google.com/", + "has_same_origin_collector": true, + "success_sample_rate": 0.05, + "collectors": [ + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload" + ], + "failure_sample_rate": 1.0, + "include_subdomains": false, + "path_prefixes": [ + "/search", + "/maps", + "/calendar", + "/images", + "/adwords", + "/adsense", + "/ads", + "/business", + "/recaptcha", + "" + ] +} diff --git a/chromium/components/domain_reliability/beacon.cc b/chromium/components/domain_reliability/beacon.cc new file mode 100644 index 00000000000..3e176666d3e --- /dev/null +++ b/chromium/components/domain_reliability/beacon.cc @@ -0,0 +1,60 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/beacon.h" + +#include <utility> + +#include "base/values.h" +#include "components/domain_reliability/util.h" +#include "net/base/net_errors.h" + +namespace domain_reliability { + +using base::Value; +using base::DictionaryValue; + +DomainReliabilityBeacon::DomainReliabilityBeacon() {} +DomainReliabilityBeacon::DomainReliabilityBeacon( + const DomainReliabilityBeacon& other) = default; +DomainReliabilityBeacon::~DomainReliabilityBeacon() {} + +scoped_ptr<Value> DomainReliabilityBeacon::ToValue( + base::TimeTicks upload_time, + base::TimeTicks last_network_change_time, + const GURL& collector_url, + const ScopedVector<std::string>& path_prefixes) const { + scoped_ptr<DictionaryValue> beacon_value(new DictionaryValue()); + DCHECK(url.is_valid()); + GURL sanitized_url = SanitizeURLForReport(url, collector_url, path_prefixes); + beacon_value->SetString("url", sanitized_url.spec()); + beacon_value->SetString("status", status); + if (!quic_error.empty()) + beacon_value->SetString("quic_error", quic_error); + if (chrome_error != net::OK) { + DictionaryValue* failure_value = new DictionaryValue(); + failure_value->SetString("custom_error", + net::ErrorToString(chrome_error)); + beacon_value->Set("failure_data", failure_value); + } + beacon_value->SetString("server_ip", server_ip); + beacon_value->SetBoolean("was_proxied", was_proxied); + beacon_value->SetString("protocol", protocol); + if (details.quic_broken) + beacon_value->SetBoolean("quic_broken", details.quic_broken); + if (details.quic_port_migration_detected) + beacon_value->SetBoolean("quic_port_migration_detected", + details.quic_port_migration_detected); + if (http_response_code >= 0) + beacon_value->SetInteger("http_response_code", http_response_code); + beacon_value->SetInteger("request_elapsed_ms", elapsed.InMilliseconds()); + base::TimeDelta request_age = upload_time - start_time; + beacon_value->SetInteger("request_age_ms", request_age.InMilliseconds()); + bool network_changed = last_network_change_time > start_time; + beacon_value->SetBoolean("network_changed", network_changed); + beacon_value->SetDouble("sample_rate", sample_rate); + return std::move(beacon_value); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/beacon.h b/chromium/components/domain_reliability/beacon.h new file mode 100644 index 00000000000..43b1e4fedac --- /dev/null +++ b/chromium/components/domain_reliability/beacon.h @@ -0,0 +1,82 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_BEACON_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_BEACON_H_ + +#include <string> + +#include "base/memory/scoped_vector.h" +#include "base/time/time.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "net/base/net_error_details.h" +#include "url/gurl.h" + +namespace base { +class Value; +} // namespace base + +namespace domain_reliability { + +// The per-request data that is uploaded to the Domain Reliability collector. +struct DOMAIN_RELIABILITY_EXPORT DomainReliabilityBeacon { + public: + DomainReliabilityBeacon(); + DomainReliabilityBeacon(const DomainReliabilityBeacon& other); + ~DomainReliabilityBeacon(); + + // Converts the Beacon to JSON format for uploading. Calculates the age + // relative to an upload time of |upload_time|. + // + // |last_network_change_time| is used to determine which beacons are + // labeled as from a previous network connection. + // |collector_url| is compared to the URLs in the beacons to determine which + // are being uploaded to a same-origin collector. + // |path_prefixes| are used to include only a known-safe (not PII) prefix of + // URLs when uploading to a non-same-origin collector. + scoped_ptr<base::Value> ToValue( + base::TimeTicks upload_time, + base::TimeTicks last_network_change_time, + const GURL& collector_url, + const ScopedVector<std::string>& path_prefixes) const; + + // The URL that the beacon is reporting on, if included. + GURL url; + // The resource name that the beacon is reporting on, if included. + std::string resource; + // Status string (e.g. "ok", "dns.nxdomain", "http.403"). + std::string status; + // Granular QUIC error string (e.g. "quic.peer_going_away"). + std::string quic_error; + // Net error code. Encoded as a string in the final JSON. + int chrome_error; + // IP address of the server the request went to. + std::string server_ip; + // Whether the request went through a proxy. If true, |server_ip| will be + // empty. + bool was_proxied; + // Protocol used to make the request. + std::string protocol; + // Network error details for the request. + net::NetErrorDetails details; + // HTTP response code returned by the server, or -1 if none was received. + int http_response_code; + // Elapsed time between starting and completing the request. + base::TimeDelta elapsed; + // Start time of the request. Encoded as the request age in the final JSON. + base::TimeTicks start_time; + // Length of the chain of Domain Reliability uploads leading to this report. + // Zero if the request was not caused by an upload, one if the request was + // caused by an upload that itself contained no beacons caused by uploads, + // et cetera. + int upload_depth; + // The probability that this request had of being reported ("sample rate"). + double sample_rate; + + // Okay to copy and assign. +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_BEACON_H_ diff --git a/chromium/components/domain_reliability/clear_mode.h b/chromium/components/domain_reliability/clear_mode.h new file mode 100644 index 00000000000..c994524f97b --- /dev/null +++ b/chromium/components/domain_reliability/clear_mode.h @@ -0,0 +1,26 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_CLEAR_MODE_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_CLEAR_MODE_H_ + +#include "components/domain_reliability/domain_reliability_export.h" + +namespace domain_reliability { + +// Argument to DomainReliabilityMonitor::ClearBrowsingData. +enum DomainReliabilityClearMode { + // Clear accumulated beacons (which betray browsing history) but leave + // registered contexts intact. + CLEAR_BEACONS, + + // Clear registered contexts (which can act like cookies). + CLEAR_CONTEXTS, + + MAX_CLEAR_MODE +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_CLEAR_MODE_H_ diff --git a/chromium/components/domain_reliability/config.cc b/chromium/components/domain_reliability/config.cc new file mode 100644 index 00000000000..a5139d9e198 --- /dev/null +++ b/chromium/components/domain_reliability/config.cc @@ -0,0 +1,122 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Make sure stdint.h includes SIZE_MAX. (See C89, p259, footnote 221.) +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include "components/domain_reliability/config.h" + +#include <stdint.h> +#include <utility> + +#include "base/json/json_reader.h" +#include "base/json/json_value_converter.h" +#include "base/profiler/scoped_tracker.h" +#include "base/strings/pattern.h" +#include "base/strings/string_util.h" + +namespace { + +bool ConvertURL(const base::Value* value, GURL* url) { + std::string url_string; + if (!value->GetAsString(&url_string)) + return false; + *url = GURL(url_string); + return url->is_valid(); +} + +bool ConvertOrigin(const base::Value* value, GURL* url) { + return ConvertURL(value, url) && !url->has_username() && + !url->has_password() && url->SchemeIs("https") && + url->path_piece() == "/" && !url->has_query() && !url->has_ref(); +} + +bool IsValidSampleRate(double p) { + return p >= 0.0 && p <= 1.0; +} + +} // namespace + +namespace domain_reliability { + +DomainReliabilityConfig::DomainReliabilityConfig() + : include_subdomains(false), + success_sample_rate(-1.0), + failure_sample_rate(-1.0) { +} +DomainReliabilityConfig::~DomainReliabilityConfig() {} + +// static +scoped_ptr<const DomainReliabilityConfig> DomainReliabilityConfig::FromJSON( + const base::StringPiece& json) { + scoped_ptr<base::Value> value = base::JSONReader::Read(json); + base::JSONValueConverter<DomainReliabilityConfig> converter; + scoped_ptr<DomainReliabilityConfig> config(new DomainReliabilityConfig()); + + // If we can parse and convert the JSON into a valid config, return that. + if (value && converter.Convert(*value, config.get()) && config->IsValid()) + return std::move(config); + return scoped_ptr<const DomainReliabilityConfig>(); +} + +bool DomainReliabilityConfig::IsValid() const { + if (!origin.is_valid() || collectors.empty() || + !IsValidSampleRate(success_sample_rate) || + !IsValidSampleRate(failure_sample_rate)) { + return false; + } + + for (const auto& url : collectors) { + if (!url->is_valid()) + return false; + } + + return true; +} + +bool DomainReliabilityConfig::Equals(const DomainReliabilityConfig& other) + const { + if (include_subdomains != other.include_subdomains || + collectors.size() != other.collectors.size() || + success_sample_rate != other.success_sample_rate || + failure_sample_rate != other.failure_sample_rate || + path_prefixes.size() != other.path_prefixes.size()) { + return false; + } + + for (size_t i = 0; i < collectors.size(); ++i) + if (*collectors[i] != *other.collectors[i]) + return false; + + for (size_t i = 0; i < path_prefixes.size(); ++i) + if (*path_prefixes[i] != *other.path_prefixes[i]) + return false; + + return true; +} + +double DomainReliabilityConfig::GetSampleRate(bool request_successful) const { + return request_successful ? success_sample_rate : failure_sample_rate; +} + +// static +void DomainReliabilityConfig::RegisterJSONConverter( + base::JSONValueConverter<DomainReliabilityConfig>* converter) { + converter->RegisterCustomValueField<GURL>( + "origin", &DomainReliabilityConfig::origin, &ConvertOrigin); + converter->RegisterBoolField("include_subdomains", + &DomainReliabilityConfig::include_subdomains); + converter->RegisterRepeatedCustomValue( + "collectors", &DomainReliabilityConfig::collectors, &ConvertURL); + converter->RegisterRepeatedString("path_prefixes", + &DomainReliabilityConfig::path_prefixes); + converter->RegisterDoubleField("success_sample_rate", + &DomainReliabilityConfig::success_sample_rate); + converter->RegisterDoubleField("failure_sample_rate", + &DomainReliabilityConfig::failure_sample_rate); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/config.h b/chromium/components/domain_reliability/config.h new file mode 100644 index 00000000000..605bd583b82 --- /dev/null +++ b/chromium/components/domain_reliability/config.h @@ -0,0 +1,57 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_CONFIG_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_CONFIG_H_ + +#include <string> +#include <vector> + +#include "base/json/json_value_converter.h" +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" +#include "base/strings/string_piece.h" +#include "base/time/time.h" +#include "base/values.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "url/gurl.h" + +namespace domain_reliability { + +// The per-origin configuration that controls which requests are measured and +// reported, with what frequency, and where the beacons are uploaded. +struct DOMAIN_RELIABILITY_EXPORT DomainReliabilityConfig { + public: + DomainReliabilityConfig(); + ~DomainReliabilityConfig(); + + // Uses the JSONValueConverter to parse the JSON for a config into a struct. + static scoped_ptr<const DomainReliabilityConfig> FromJSON( + const base::StringPiece& json); + + bool IsValid() const; + bool Equals(const DomainReliabilityConfig& other) const; + + double GetSampleRate(bool request_successful) const; + + // Registers with the JSONValueConverter so it will know how to convert the + // JSON for a config into the struct. + static void RegisterJSONConverter( + base::JSONValueConverter<DomainReliabilityConfig>* converter); + + GURL origin; + bool include_subdomains; + ScopedVector<GURL> collectors; + + double success_sample_rate; + double failure_sample_rate; + ScopedVector<std::string> path_prefixes; + + private: + DISALLOW_COPY_AND_ASSIGN(DomainReliabilityConfig); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_CONFIG_H_ diff --git a/chromium/components/domain_reliability/config_unittest.cc b/chromium/components/domain_reliability/config_unittest.cc new file mode 100644 index 00000000000..8af32588302 --- /dev/null +++ b/chromium/components/domain_reliability/config_unittest.cc @@ -0,0 +1,94 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/config.h" + +#include <string> + +#include "base/memory/scoped_ptr.h" +#include "base/time/time.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +scoped_ptr<DomainReliabilityConfig> MakeBaseConfig() { + DomainReliabilityConfig* config = new DomainReliabilityConfig(); + config->origin = GURL("https://example/"); + config->include_subdomains = false; + config->collectors.push_back(new GURL("https://example/upload")); + config->failure_sample_rate = 1.0; + config->success_sample_rate = 0.0; + EXPECT_TRUE(config->IsValid()); + return scoped_ptr<DomainReliabilityConfig>(config); +} + +scoped_ptr<DomainReliabilityConfig> MakeSampleConfig() { + scoped_ptr<DomainReliabilityConfig> config(MakeBaseConfig()); + config->path_prefixes.push_back(new std::string("/css/")); + config->path_prefixes.push_back(new std::string("/js/")); + EXPECT_TRUE(config->IsValid()); + return config; +} + +class DomainReliabilityConfigTest : public testing::Test { }; + +TEST_F(DomainReliabilityConfigTest, IsValid) { + scoped_ptr<DomainReliabilityConfig> config; + + config = MakeSampleConfig(); + EXPECT_TRUE(config->IsValid()); + + config = MakeSampleConfig(); + config->origin = GURL(); + EXPECT_FALSE(config->IsValid()); + + config = MakeSampleConfig(); + config->collectors.clear(); + EXPECT_FALSE(config->IsValid()); + + config = MakeSampleConfig(); + delete config->collectors[0]; + config->collectors[0] = new GURL(); + EXPECT_FALSE(config->IsValid()); + + config = MakeSampleConfig(); + config->failure_sample_rate = 2.0; + EXPECT_FALSE(config->IsValid()); + + config = MakeSampleConfig(); + config->success_sample_rate = 2.0; + EXPECT_FALSE(config->IsValid()); +} + +TEST_F(DomainReliabilityConfigTest, FromJSON) { + std::string config_json = + "{ \"origin\": \"https://example/\"," + " \"include_subdomains\": false," + " \"collectors\": [ \"https://example/upload\" ]," + " \"path_prefixes\": [" + " \"/css/\"," + " \"/js/\"" + " ]," + " \"failure_sample_rate\": 0.10," + " \"success_sample_rate\": 0.01" + "}"; + + scoped_ptr<const DomainReliabilityConfig> config( + DomainReliabilityConfig::FromJSON(config_json)); + + EXPECT_TRUE(config); + EXPECT_EQ("https://example/", config->origin.spec()); + EXPECT_FALSE(config->include_subdomains); + EXPECT_EQ(1u, config->collectors.size()); + EXPECT_EQ(GURL("https://example/upload"), *config->collectors[0]); + EXPECT_EQ(2u, config->path_prefixes.size()); + EXPECT_EQ("/css/", *config->path_prefixes[0]); + EXPECT_EQ("/js/", *config->path_prefixes[1]); + EXPECT_EQ(0.10, config->failure_sample_rate); + EXPECT_EQ(0.01, config->success_sample_rate); +} + +} // namespace +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/context.cc b/chromium/components/domain_reliability/context.cc new file mode 100644 index 00000000000..4d7c83e2008 --- /dev/null +++ b/chromium/components/domain_reliability/context.cc @@ -0,0 +1,259 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/context.h" + +#include <algorithm> +#include <utility> + +#include "base/bind.h" +#include "base/json/json_writer.h" +#include "base/logging.h" +#include "base/metrics/histogram.h" +#include "base/metrics/sparse_histogram.h" +#include "base/rand_util.h" +#include "base/values.h" +#include "components/domain_reliability/dispatcher.h" +#include "components/domain_reliability/uploader.h" +#include "components/domain_reliability/util.h" +#include "net/base/net_errors.h" +#include "net/url_request/url_request_context_getter.h" + +using base::DictionaryValue; +using base::ListValue; +using base::Value; + +namespace domain_reliability { + +namespace { +void LogOnBeaconDidEvictHistogram(bool evicted) { + UMA_HISTOGRAM_BOOLEAN("DomainReliability.OnBeaconDidEvict", evicted); +} +} // namespace + +// static +const int DomainReliabilityContext::kMaxUploadDepthToSchedule = 1; + +DomainReliabilityContext::Factory::~Factory() { +} + +// static +const size_t DomainReliabilityContext::kMaxQueuedBeacons = 150; + +DomainReliabilityContext::DomainReliabilityContext( + MockableTime* time, + const DomainReliabilityScheduler::Params& scheduler_params, + const std::string& upload_reporter_string, + const base::TimeTicks* last_network_change_time, + DomainReliabilityDispatcher* dispatcher, + DomainReliabilityUploader* uploader, + scoped_ptr<const DomainReliabilityConfig> config) + : config_(std::move(config)), + time_(time), + upload_reporter_string_(upload_reporter_string), + scheduler_(time, + config_->collectors.size(), + scheduler_params, + base::Bind(&DomainReliabilityContext::ScheduleUpload, + base::Unretained(this))), + dispatcher_(dispatcher), + uploader_(uploader), + uploading_beacons_size_(0), + last_network_change_time_(last_network_change_time), + weak_factory_(this) {} + +DomainReliabilityContext::~DomainReliabilityContext() { + ClearBeacons(); +} + +void DomainReliabilityContext::OnBeacon( + scoped_ptr<DomainReliabilityBeacon> beacon) { + bool success = (beacon->status == "ok"); + double sample_rate = beacon->details.quic_port_migration_detected + ? 1.0 + : config().GetSampleRate(success); + bool should_report = base::RandDouble() < sample_rate; + UMA_HISTOGRAM_BOOLEAN("DomainReliability.BeaconReported", should_report); + if (!should_report) { + // If the beacon isn't queued to be reported, it definitely cannot evict + // an older beacon. (This histogram is also logged below based on whether + // an older beacon was actually evicted.) + LogOnBeaconDidEvictHistogram(false); + return; + } + beacon->sample_rate = sample_rate; + + UMA_HISTOGRAM_SPARSE_SLOWLY("DomainReliability.ReportedBeaconError", + -beacon->chrome_error); + if (!beacon->server_ip.empty()) { + UMA_HISTOGRAM_SPARSE_SLOWLY( + "DomainReliability.ReportedBeaconError_HasServerIP", + -beacon->chrome_error); + } + // TODO(ttuttle): Histogram HTTP response code? + + // Allow beacons about reports, but don't schedule an upload for more than + // one layer of recursion, to avoid infinite report loops. + if (beacon->upload_depth <= kMaxUploadDepthToSchedule) + scheduler_.OnBeaconAdded(); + beacons_.push_back(beacon.release()); + bool should_evict = beacons_.size() > kMaxQueuedBeacons; + if (should_evict) + RemoveOldestBeacon(); + + LogOnBeaconDidEvictHistogram(should_evict); +} + +void DomainReliabilityContext::ClearBeacons() { + STLDeleteElements(&beacons_); + beacons_.clear(); + uploading_beacons_size_ = 0; +} + +scoped_ptr<Value> DomainReliabilityContext::GetWebUIData() const { + DictionaryValue* context_value = new DictionaryValue(); + + context_value->SetString("origin", config().origin.spec()); + context_value->SetInteger("beacon_count", static_cast<int>(beacons_.size())); + context_value->SetInteger("uploading_beacon_count", + static_cast<int>(uploading_beacons_size_)); + context_value->Set("scheduler", scheduler_.GetWebUIData()); + + return scoped_ptr<Value>(context_value); +} + +void DomainReliabilityContext::GetQueuedBeaconsForTesting( + std::vector<const DomainReliabilityBeacon*>* beacons_out) const { + DCHECK(this); + DCHECK(beacons_out); + beacons_out->assign(beacons_.begin(), beacons_.end()); +} + +void DomainReliabilityContext::ScheduleUpload( + base::TimeDelta min_delay, + base::TimeDelta max_delay) { + dispatcher_->ScheduleTask( + base::Bind( + &DomainReliabilityContext::StartUpload, + weak_factory_.GetWeakPtr()), + min_delay, + max_delay); +} + +void DomainReliabilityContext::StartUpload() { + MarkUpload(); + + size_t collector_index = scheduler_.OnUploadStart(); + const GURL& collector_url = *config().collectors[collector_index]; + + DCHECK(upload_time_.is_null()); + upload_time_ = time_->NowTicks(); + std::string report_json = "{}"; + int max_upload_depth = -1; + bool wrote = base::JSONWriter::Write( + *CreateReport(upload_time_, + collector_url, + &max_upload_depth), + &report_json); + DCHECK(wrote); + DCHECK_NE(-1, max_upload_depth); + + uploader_->UploadReport( + report_json, + max_upload_depth, + collector_url, + base::Bind( + &DomainReliabilityContext::OnUploadComplete, + weak_factory_.GetWeakPtr())); + + UMA_HISTOGRAM_SPARSE_SLOWLY("DomainReliability.UploadCollectorIndex", + static_cast<int>(collector_index)); + if (!last_upload_time_.is_null()) { + UMA_HISTOGRAM_LONG_TIMES("DomainReliability.UploadInterval", + upload_time_ - last_upload_time_); + } +} + +void DomainReliabilityContext::OnUploadComplete( + const DomainReliabilityUploader::UploadResult& result) { + if (result.is_success()) + CommitUpload(); + else + RollbackUpload(); + base::TimeTicks first_beacon_time = scheduler_.first_beacon_time(); + scheduler_.OnUploadComplete(result); + UMA_HISTOGRAM_BOOLEAN("DomainReliability.UploadSuccess", + result.is_success()); + base::TimeTicks now = time_->NowTicks(); + UMA_HISTOGRAM_LONG_TIMES("DomainReliability.UploadLatency", + now - first_beacon_time); + DCHECK(!upload_time_.is_null()); + UMA_HISTOGRAM_MEDIUM_TIMES("DomainReliability.UploadDuration", + now - upload_time_); + UMA_HISTOGRAM_LONG_TIMES("DomainReliability.UploadCollectorRetryDelay", + scheduler_.last_collector_retry_delay()); + last_upload_time_ = upload_time_; + upload_time_ = base::TimeTicks(); +} + +scoped_ptr<const Value> DomainReliabilityContext::CreateReport( + base::TimeTicks upload_time, + const GURL& collector_url, + int* max_upload_depth_out) const { + int max_upload_depth = 0; + + scoped_ptr<ListValue> beacons_value(new ListValue()); + for (const auto& beacon : beacons_) { + beacons_value->Append(beacon->ToValue(upload_time, + *last_network_change_time_, + collector_url, + config().path_prefixes)); + if (beacon->upload_depth > max_upload_depth) + max_upload_depth = beacon->upload_depth; + } + + scoped_ptr<DictionaryValue> report_value(new DictionaryValue()); + report_value->SetString("reporter", upload_reporter_string_); + report_value->Set("entries", beacons_value.release()); + + *max_upload_depth_out = max_upload_depth; + return std::move(report_value); +} + +void DomainReliabilityContext::MarkUpload() { + DCHECK_EQ(0u, uploading_beacons_size_); + uploading_beacons_size_ = beacons_.size(); + DCHECK_NE(0u, uploading_beacons_size_); +} + +void DomainReliabilityContext::CommitUpload() { + auto begin = beacons_.begin(); + auto end = begin + uploading_beacons_size_; + STLDeleteContainerPointers(begin, end); + beacons_.erase(begin, end); + DCHECK_NE(0u, uploading_beacons_size_); + uploading_beacons_size_ = 0; +} + +void DomainReliabilityContext::RollbackUpload() { + DCHECK_NE(0u, uploading_beacons_size_); + uploading_beacons_size_ = 0; +} + +void DomainReliabilityContext::RemoveOldestBeacon() { + DCHECK(!beacons_.empty()); + + VLOG(1) << "Beacon queue for " << config().origin << " full; " + << "removing oldest beacon"; + + delete beacons_.front(); + beacons_.pop_front(); + + // If that just removed a beacon counted in uploading_beacons_size_, decrement + // that. + if (uploading_beacons_size_ > 0) + --uploading_beacons_size_; +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/context.h b/chromium/components/domain_reliability/context.h new file mode 100644 index 00000000000..fb216724d84 --- /dev/null +++ b/chromium/components/domain_reliability/context.h @@ -0,0 +1,134 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_CONTEXT_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_CONTEXT_H_ + +#include <stddef.h> + +#include <deque> +#include <vector> + +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/weak_ptr.h" +#include "base/time/time.h" +#include "components/domain_reliability/beacon.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "components/domain_reliability/scheduler.h" +#include "components/domain_reliability/uploader.h" + +class GURL; + +namespace base { +class Value; +} + +namespace domain_reliability { + +class DomainReliabilityDispatcher; +class DomainReliabilityUploader; +class MockableTime; + +// The per-domain context for the Domain Reliability client; includes the +// domain's config and beacon queue. +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityContext { + public: + // Maximum upload depth to schedule an upload. If a beacon is based on a more + // deeply nested upload, it will be reported eventually, but will not itself + // trigger a new upload. + static const int kMaxUploadDepthToSchedule; + + class DOMAIN_RELIABILITY_EXPORT Factory { + public: + virtual ~Factory(); + virtual scoped_ptr<DomainReliabilityContext> CreateContextForConfig( + scoped_ptr<const DomainReliabilityConfig> config) = 0; + }; + + DomainReliabilityContext( + MockableTime* time, + const DomainReliabilityScheduler::Params& scheduler_params, + const std::string& upload_reporter_string, + const base::TimeTicks* last_network_change_time, + DomainReliabilityDispatcher* dispatcher, + DomainReliabilityUploader* uploader, + scoped_ptr<const DomainReliabilityConfig> config); + ~DomainReliabilityContext(); + + // Notifies the context of a beacon on its domain(s); may or may not save the + // actual beacon to be uploaded, depending on the sample rates in the config, + // but will increment one of the request counters in any case. + void OnBeacon(scoped_ptr<DomainReliabilityBeacon> beacon); + + // Called to clear browsing data, since beacons are like browsing history. + void ClearBeacons(); + + // Gets a Value containing data that can be formatted into a web page for + // debugging purposes. + scoped_ptr<base::Value> GetWebUIData() const; + + // Gets the beacons queued for upload in this context. |*beacons_out| will be + // cleared and filled with pointers to the beacons; the pointers remain valid + // as long as no other requests are reported to the DomainReliabilityMonitor. + void GetQueuedBeaconsForTesting( + std::vector<const DomainReliabilityBeacon*>* beacons_out) const; + + const DomainReliabilityConfig& config() const { return *config_.get(); } + + // Maximum number of beacons queued per context; if more than this many are + // queued; the oldest beacons will be removed. + static const size_t kMaxQueuedBeacons; + + private: + // Deque of beacons owned by this context. (Deleted after uploading.) + typedef std::deque<DomainReliabilityBeacon*> BeaconDeque; + + void ScheduleUpload(base::TimeDelta min_delay, base::TimeDelta max_delay); + void StartUpload(); + void OnUploadComplete(const DomainReliabilityUploader::UploadResult& result); + + scoped_ptr<const base::Value> CreateReport(base::TimeTicks upload_time, + const GURL& collector_url, + int* max_beacon_depth_out) const; + + // Remembers the current state of the context when an upload starts. Can be + // called multiple times in a row (without |CommitUpload|) if uploads fail + // and are retried. + void MarkUpload(); + + // Uses the state remembered by |MarkUpload| to remove successfully uploaded + // data but keep beacons and request counts added after the upload started. + void CommitUpload(); + + void RollbackUpload(); + + // Finds and removes the oldest beacon. DCHECKs if there is none. (Called + // when there are too many beacons queued.) + void RemoveOldestBeacon(); + + scoped_ptr<const DomainReliabilityConfig> config_; + MockableTime* time_; + const std::string& upload_reporter_string_; + DomainReliabilityScheduler scheduler_; + DomainReliabilityDispatcher* dispatcher_; + DomainReliabilityUploader* uploader_; + + BeaconDeque beacons_; + size_t uploading_beacons_size_; + base::TimeTicks upload_time_; + base::TimeTicks last_upload_time_; + // The last network change time is not tracked per-context, so this is a + // pointer to that value in a wider (e.g. per-Monitor or unittest) scope. + const base::TimeTicks* last_network_change_time_; + + base::WeakPtrFactory<DomainReliabilityContext> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(DomainReliabilityContext); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_CONTEXT_H_ diff --git a/chromium/components/domain_reliability/context_manager.cc b/chromium/components/domain_reliability/context_manager.cc new file mode 100644 index 00000000000..b7036d43170 --- /dev/null +++ b/chromium/components/domain_reliability/context_manager.cc @@ -0,0 +1,129 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/context_manager.h" + +#include <utility> + +namespace domain_reliability { + +DomainReliabilityContextManager::DomainReliabilityContextManager( + DomainReliabilityContext::Factory* context_factory) + : context_factory_(context_factory) { +} + +DomainReliabilityContextManager::~DomainReliabilityContextManager() { + RemoveAllContexts(); +} + +void DomainReliabilityContextManager::RouteBeacon( + scoped_ptr<DomainReliabilityBeacon> beacon) { + DomainReliabilityContext* context = GetContextForHost(beacon->url.host()); + if (!context) + return; + + context->OnBeacon(std::move(beacon)); +} + +void DomainReliabilityContextManager::SetConfig( + const GURL& origin, + scoped_ptr<DomainReliabilityConfig> config, + base::TimeDelta max_age) { + std::string key = origin.host(); + + if (!contexts_.count(key) && !removed_contexts_.count(key)) { + LOG(WARNING) << "Ignoring NEL header for unknown origin " << origin.spec() + << "."; + return; + } + + if (contexts_.count(key)) { + // Currently, there is no easy way to change the config of a context, so + // updating the config requires recreating the context, which loses + // pending beacons and collector backoff state. Therefore, don't do so + // needlessly; make sure the config has actually changed before recreating + // the context. + if (contexts_[key]->config().Equals(*config)) { + DVLOG(1) << "Ignoring unchanged NEL header for existing origin " + << origin.spec() << "."; + return; + } + // TODO(ttuttle): Make Context accept Config changes. + } + + DVLOG(1) << "Adding/replacing context for existing origin " << origin.spec() + << "."; + removed_contexts_.erase(key); + config->origin = origin; + AddContextForConfig(std::move(config)); +} + +void DomainReliabilityContextManager::ClearConfig(const GURL& origin) { + std::string key = origin.host(); + + if (contexts_.count(key)) { + DVLOG(1) << "Removing context for existing origin " << origin.spec() << "."; + contexts_.erase(key); + removed_contexts_.insert(key); + } +} + +void DomainReliabilityContextManager::ClearBeaconsInAllContexts() { + for (auto& context_entry : contexts_) + context_entry.second->ClearBeacons(); +} + +DomainReliabilityContext* DomainReliabilityContextManager::AddContextForConfig( + scoped_ptr<const DomainReliabilityConfig> config) { + std::string key = config->origin.host(); + // TODO(ttuttle): Convert this to actual origin. + + scoped_ptr<DomainReliabilityContext> context = + context_factory_->CreateContextForConfig(std::move(config)); + DomainReliabilityContext** entry = &contexts_[key]; + if (*entry) + delete *entry; + + *entry = context.release(); + return *entry; +} + +void DomainReliabilityContextManager::RemoveAllContexts() { + STLDeleteContainerPairSecondPointers( + contexts_.begin(), contexts_.end()); + contexts_.clear(); +} + +scoped_ptr<base::Value> DomainReliabilityContextManager::GetWebUIData() const { + scoped_ptr<base::ListValue> contexts_value(new base::ListValue()); + for (const auto& context_entry : contexts_) + contexts_value->Append(context_entry.second->GetWebUIData().release()); + return std::move(contexts_value); +} + +DomainReliabilityContext* DomainReliabilityContextManager::GetContextForHost( + const std::string& host) { + ContextMap::const_iterator context_it; + + context_it = contexts_.find(host); + if (context_it != contexts_.end()) + return context_it->second; + + size_t dot_pos = host.find('.'); + if (dot_pos == std::string::npos) + return nullptr; + + // TODO(ttuttle): Make sure parent is not in PSL before using. + + std::string parent_host = host.substr(dot_pos + 1); + context_it = contexts_.find(parent_host); + if (context_it != contexts_.end() + && context_it->second->config().include_subdomains) { + return context_it->second; + } + + return nullptr; +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/context_manager.h b/chromium/components/domain_reliability/context_manager.h new file mode 100644 index 00000000000..302671e93d6 --- /dev/null +++ b/chromium/components/domain_reliability/context_manager.h @@ -0,0 +1,76 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_CONTEXT_MANAGER_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_CONTEXT_MANAGER_H_ + +#include <stddef.h> + +#include <map> +#include <string> +#include <unordered_set> + +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" +#include "base/values.h" +#include "components/domain_reliability/beacon.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/context.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "url/gurl.h" + +namespace domain_reliability { + +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityContextManager { + public: + DomainReliabilityContextManager( + DomainReliabilityContext::Factory* context_factory); + ~DomainReliabilityContextManager(); + + // If |url| maps to a context added to this manager, calls |OnBeacon| on + // that context with |beacon|. Otherwise, does nothing. + void RouteBeacon(scoped_ptr<DomainReliabilityBeacon> beacon); + + void SetConfig(const GURL& origin, + scoped_ptr<DomainReliabilityConfig> config, + base::TimeDelta max_age); + void ClearConfig(const GURL& origin); + + // Calls |ClearBeacons| on all contexts added to this manager, but leaves + // the contexts themselves intact. + void ClearBeaconsInAllContexts(); + + // TODO(ttuttle): Once unit tests test ContextManager directly, they can use + // a custom Context::Factory to get the created Context, and this can be void. + DomainReliabilityContext* AddContextForConfig( + scoped_ptr<const DomainReliabilityConfig> config); + + // Removes all contexts from this manager (discarding all queued beacons in + // the process). + void RemoveAllContexts(); + + scoped_ptr<base::Value> GetWebUIData() const; + + size_t contexts_size_for_testing() const { return contexts_.size(); } + + private: + typedef std::map<std::string, DomainReliabilityContext*> ContextMap; + + DomainReliabilityContext* GetContextForHost(const std::string& host); + + DomainReliabilityContext::Factory* context_factory_; + // Owns DomainReliabilityContexts. + ContextMap contexts_; + // Currently, Domain Reliability only allows header-based configuration by + // origins that already have baked-in configs. This is the set of origins + // that have removed their context (by sending "NEL: max-age=0"), so the + // context manager knows they are allowed to set a config again later. + std::unordered_set<std::string> removed_contexts_; + + DISALLOW_COPY_AND_ASSIGN(DomainReliabilityContextManager); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_CONTEXT_MANAGER_H_ diff --git a/chromium/components/domain_reliability/context_unittest.cc b/chromium/components/domain_reliability/context_unittest.cc new file mode 100644 index 00000000000..a54a1765f0d --- /dev/null +++ b/chromium/components/domain_reliability/context_unittest.cc @@ -0,0 +1,541 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/context.h" + +#include <stddef.h> +#include <map> +#include <string> +#include <utility> + +#include "base/bind.h" +#include "base/json/json_reader.h" +#include "base/memory/scoped_ptr.h" +#include "components/domain_reliability/beacon.h" +#include "components/domain_reliability/dispatcher.h" +#include "components/domain_reliability/scheduler.h" +#include "components/domain_reliability/test_util.h" +#include "components/domain_reliability/uploader.h" +#include "net/base/net_errors.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +using base::DictionaryValue; +using base::ListValue; +using base::Value; + +typedef std::vector<const DomainReliabilityBeacon*> BeaconVector; + +scoped_ptr<DomainReliabilityBeacon> MakeCustomizedBeacon( + MockableTime* time, + std::string status, + std::string quic_error, + bool quic_port_migration_detected) { + scoped_ptr<DomainReliabilityBeacon> beacon(new DomainReliabilityBeacon()); + beacon->url = GURL("https://localhost/"); + beacon->status = status; + beacon->quic_error = quic_error; + beacon->chrome_error = net::ERR_CONNECTION_RESET; + beacon->server_ip = "127.0.0.1"; + beacon->was_proxied = false; + beacon->protocol = "HTTP"; + beacon->details.quic_broken = true; + beacon->details.quic_port_migration_detected = quic_port_migration_detected; + beacon->http_response_code = -1; + beacon->elapsed = base::TimeDelta::FromMilliseconds(250); + beacon->start_time = time->NowTicks() - beacon->elapsed; + beacon->upload_depth = 0; + beacon->sample_rate = 1.0; + return beacon; +} + +scoped_ptr<DomainReliabilityBeacon> MakeBeacon(MockableTime* time) { + return MakeCustomizedBeacon(time, "tcp.connection_reset", "", false); +} + +template <typename ValueType, + bool (DictionaryValue::* GetValueType)(const std::string&, + ValueType*) const> +struct HasValue { + bool operator()(const DictionaryValue& dict, + const std::string& key, + ValueType expected_value) { + ValueType actual_value; + bool got_value = (dict.*GetValueType)(key, &actual_value); + if (got_value) + EXPECT_EQ(expected_value, actual_value); + return got_value && (expected_value == actual_value); + } +}; + +HasValue<bool, &DictionaryValue::GetBoolean> HasBooleanValue; +HasValue<double, &DictionaryValue::GetDouble> HasDoubleValue; +HasValue<int, &DictionaryValue::GetInteger> HasIntegerValue; +HasValue<std::string, &DictionaryValue::GetString> HasStringValue; + +bool GetEntryFromReport(const Value* report, + size_t index, + const DictionaryValue** entry_out) { + const DictionaryValue* report_dict; + const ListValue* entries; + + return report && + report->GetAsDictionary(&report_dict) && + report_dict->GetList("entries", &entries) && + entries->GetDictionary(index, entry_out); +} + +class DomainReliabilityContextTest : public testing::Test { + protected: + DomainReliabilityContextTest() + : last_network_change_time_(time_.NowTicks()), + dispatcher_(&time_), + params_(MakeTestSchedulerParams()), + uploader_(base::Bind(&DomainReliabilityContextTest::OnUploadRequest, + base::Unretained(this))), + upload_reporter_string_("test-reporter"), + upload_pending_(false) { + // Make sure that the last network change does not overlap requests + // made in test cases, which start 250ms in the past (see |MakeBeacon|). + last_network_change_time_ = time_.NowTicks(); + time_.Advance(base::TimeDelta::FromSeconds(1)); + } + + void InitContext(scoped_ptr<const DomainReliabilityConfig> config) { + context_.reset(new DomainReliabilityContext( + &time_, params_, upload_reporter_string_, &last_network_change_time_, + &dispatcher_, &uploader_, std::move(config))); + } + + TimeDelta min_delay() const { return params_.minimum_upload_delay; } + TimeDelta max_delay() const { return params_.maximum_upload_delay; } + TimeDelta retry_interval() const { return params_.upload_retry_interval; } + TimeDelta zero_delta() const { return TimeDelta::FromMicroseconds(0); } + + bool upload_pending() const { return upload_pending_; } + + const std::string& upload_report() const { + EXPECT_TRUE(upload_pending_); + return upload_report_; + } + + int upload_max_depth() const { + EXPECT_TRUE(upload_pending_); + return upload_max_depth_; + } + + const GURL& upload_url() const { + EXPECT_TRUE(upload_pending_); + return upload_url_; + } + + void CallUploadCallback(DomainReliabilityUploader::UploadResult result) { + ASSERT_TRUE(upload_pending_); + upload_callback_.Run(result); + upload_pending_ = false; + } + + bool CheckNoBeacons() { + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + return beacons.empty(); + } + + MockTime time_; + base::TimeTicks last_network_change_time_; + DomainReliabilityDispatcher dispatcher_; + DomainReliabilityScheduler::Params params_; + MockUploader uploader_; + std::string upload_reporter_string_; + scoped_ptr<DomainReliabilityContext> context_; + + private: + void OnUploadRequest( + const std::string& report_json, + int max_upload_depth, + const GURL& upload_url, + const DomainReliabilityUploader::UploadCallback& callback) { + ASSERT_FALSE(upload_pending_); + upload_report_ = report_json; + upload_max_depth_ = max_upload_depth; + upload_url_ = upload_url; + upload_callback_ = callback; + upload_pending_ = true; + } + + bool upload_pending_; + std::string upload_report_; + int upload_max_depth_; + GURL upload_url_; + DomainReliabilityUploader::UploadCallback upload_callback_; +}; + +TEST_F(DomainReliabilityContextTest, Create) { + InitContext(MakeTestConfig()); + EXPECT_TRUE(CheckNoBeacons()); +} + +TEST_F(DomainReliabilityContextTest, Report) { + InitContext(MakeTestConfig()); + context_->OnBeacon(MakeBeacon(&time_)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); +} + +TEST_F(DomainReliabilityContextTest, MaxNestedBeaconSchedules) { + InitContext(MakeTestConfig()); + GURL url("http://example/always_report"); + scoped_ptr<DomainReliabilityBeacon> beacon = MakeBeacon(&time_); + beacon->upload_depth = DomainReliabilityContext::kMaxUploadDepthToSchedule; + context_->OnBeacon(std::move(beacon)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); +} + +TEST_F(DomainReliabilityContextTest, OverlyNestedBeaconDoesNotSchedule) { + InitContext(MakeTestConfig()); + GURL url("http://example/always_report"); + scoped_ptr<DomainReliabilityBeacon> beacon = MakeBeacon(&time_); + beacon->upload_depth = + DomainReliabilityContext::kMaxUploadDepthToSchedule + 1; + context_->OnBeacon(std::move(beacon)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_FALSE(upload_pending()); +} + +TEST_F(DomainReliabilityContextTest, + MaxNestedBeaconAfterOverlyNestedBeaconSchedules) { + InitContext(MakeTestConfig()); + // Add a beacon for a report that's too nested to schedule a beacon. + scoped_ptr<DomainReliabilityBeacon> beacon = MakeBeacon(&time_); + beacon->upload_depth = + DomainReliabilityContext::kMaxUploadDepthToSchedule + 1; + context_->OnBeacon(std::move(beacon)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_FALSE(upload_pending()); + + // Add a beacon for a report that should schedule a beacon, and make sure it + // doesn't schedule until the deadline. + beacon = MakeBeacon(&time_); + beacon->upload_depth = DomainReliabilityContext::kMaxUploadDepthToSchedule; + context_->OnBeacon(std::move(beacon)); + + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(2u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + + // Check that both beacons were uploaded. + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +TEST_F(DomainReliabilityContextTest, ReportUpload) { + InitContext(MakeTestConfig()); + context_->OnBeacon( + MakeCustomizedBeacon(&time_, "tcp.connection_reset", "", true)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + EXPECT_EQ(0, upload_max_depth()); + EXPECT_EQ(GURL("https://exampleuploader/upload"), upload_url()); + + scoped_ptr<Value> value = base::JSONReader::Read(upload_report()); + const DictionaryValue* entry; + ASSERT_TRUE(GetEntryFromReport(value.get(), 0, &entry)); + EXPECT_TRUE(HasStringValue(*entry, "failure_data.custom_error", + "net::ERR_CONNECTION_RESET")); + EXPECT_TRUE(HasBooleanValue(*entry, "network_changed", false)); + EXPECT_TRUE(HasStringValue(*entry, "protocol", "HTTP")); + EXPECT_TRUE(HasBooleanValue(*entry, "quic_broken", true)); + EXPECT_TRUE(HasBooleanValue(*entry, "quic_port_migration_detected", true)); + // N.B.: Assumes max_delay is 5 minutes. + EXPECT_TRUE(HasIntegerValue(*entry, "request_age_ms", 300250)); + EXPECT_TRUE(HasIntegerValue(*entry, "request_elapsed_ms", 250)); + EXPECT_TRUE(HasDoubleValue(*entry, "sample_rate", 1.0)); + EXPECT_TRUE(HasStringValue(*entry, "server_ip", "127.0.0.1")); + EXPECT_TRUE(HasStringValue(*entry, "status", "tcp.connection_reset")); + EXPECT_TRUE(HasStringValue(*entry, "url", "https://localhost/")); + EXPECT_TRUE(HasBooleanValue(*entry, "was_proxied", false)); + + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +TEST_F(DomainReliabilityContextTest, NetworkChanged) { + InitContext(MakeTestConfig()); + context_->OnBeacon(MakeBeacon(&time_)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + // Simulate a network change after the request but before the upload. + last_network_change_time_ = time_.NowTicks(); + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + EXPECT_EQ(0, upload_max_depth()); + EXPECT_EQ(GURL("https://exampleuploader/upload"), upload_url()); + + scoped_ptr<Value> value = base::JSONReader::Read(upload_report()); + const DictionaryValue* entry; + ASSERT_TRUE(GetEntryFromReport(value.get(), 0, &entry)); + EXPECT_TRUE(HasBooleanValue(*entry, "network_changed", true)); + + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +// Always expecting granular QUIC errors if status is quic.protocol error. +TEST_F(DomainReliabilityContextTest, + ReportUploadWithQuicProtocolErrorAndQuicError) { + InitContext(MakeTestConfig()); + context_->OnBeacon(MakeCustomizedBeacon(&time_, "quic.protocol", + "quic.invalid.stream_data", true)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + EXPECT_EQ(0, upload_max_depth()); + EXPECT_EQ(GURL("https://exampleuploader/upload"), upload_url()); + + scoped_ptr<Value> value = base::JSONReader::Read(upload_report()); + const DictionaryValue* entry; + ASSERT_TRUE(GetEntryFromReport(value.get(), 0, &entry)); + + EXPECT_TRUE(HasBooleanValue(*entry, "quic_broken", true)); + EXPECT_TRUE(HasBooleanValue(*entry, "quic_port_migration_detected", true)); + EXPECT_TRUE(HasStringValue(*entry, "status", "quic.protocol")); + EXPECT_TRUE(HasStringValue(*entry, "quic_error", "quic.invalid.stream_data")); + + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +// If status is not quic.protocol, expect no granular QUIC error to be reported. +TEST_F(DomainReliabilityContextTest, + ReportUploadWithNonQuicProtocolErrorAndNoQuicError) { + InitContext(MakeTestConfig()); + context_->OnBeacon(MakeBeacon(&time_)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + EXPECT_EQ(0, upload_max_depth()); + EXPECT_EQ(GURL("https://exampleuploader/upload"), upload_url()); + + scoped_ptr<Value> value = base::JSONReader::Read(upload_report()); + const DictionaryValue* entry; + ASSERT_TRUE(GetEntryFromReport(value.get(), 0, &entry)); + + EXPECT_TRUE(HasStringValue(*entry, "status", "tcp.connection_reset")); + EXPECT_FALSE(HasStringValue(*entry, "quic_error", "")); + + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +// Edge cases that a non-QUIC protocol error with granular QUIC error reported, +// probably indicating state machine in http_network_transaction is working +// in a different way. +TEST_F(DomainReliabilityContextTest, + ReportUploadWithNonQuicProtocolErrorAndQuicError) { + InitContext(MakeTestConfig()); + context_->OnBeacon(MakeCustomizedBeacon(&time_, "tcp.connection_reset", + "quic.invalid.stream_data", false)); + + BeaconVector beacons; + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + EXPECT_EQ(0, upload_max_depth()); + EXPECT_EQ(GURL("https://exampleuploader/upload"), upload_url()); + + scoped_ptr<Value> value = base::JSONReader::Read(upload_report()); + const DictionaryValue* entry; + ASSERT_TRUE(GetEntryFromReport(value.get(), 0, &entry)); + EXPECT_TRUE(HasBooleanValue(*entry, "quic_broken", true)); + EXPECT_TRUE(HasStringValue(*entry, "status", "tcp.connection_reset")); + EXPECT_TRUE(HasStringValue(*entry, "quic_error", "quic.invalid.stream_data")); + + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +TEST_F(DomainReliabilityContextTest, ZeroSampleRate) { + scoped_ptr<DomainReliabilityConfig> config(MakeTestConfig()); + config->failure_sample_rate = 0.0; + InitContext(std::move(config)); + + BeaconVector beacons; + for (int i = 0; i < 100; i++) { + context_->OnBeacon(MakeBeacon(&time_)); + EXPECT_TRUE(CheckNoBeacons()); + } +} + +TEST_F(DomainReliabilityContextTest, FractionalSampleRate) { + scoped_ptr<DomainReliabilityConfig> config(MakeTestConfig()); + config->failure_sample_rate = 0.5; + InitContext(std::move(config)); + + BeaconVector beacons; + do { + context_->OnBeacon(MakeBeacon(&time_)); + context_->GetQueuedBeaconsForTesting(&beacons); + } while (beacons.empty()); + EXPECT_EQ(1u, beacons.size()); + + time_.Advance(max_delay()); + EXPECT_TRUE(upload_pending()); + EXPECT_EQ(0, upload_max_depth()); + EXPECT_EQ(GURL("https://exampleuploader/upload"), upload_url()); + + scoped_ptr<Value> value = base::JSONReader::Read(upload_report()); + const DictionaryValue* entry; + ASSERT_TRUE(GetEntryFromReport(value.get(), 0, &entry)); + EXPECT_TRUE(HasDoubleValue(*entry, "sample_rate", 0.5)); + + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + CallUploadCallback(result); + + EXPECT_TRUE(CheckNoBeacons()); +} + +TEST_F(DomainReliabilityContextTest, FailureSampleOnly) { + scoped_ptr<DomainReliabilityConfig> config(MakeTestConfig()); + config->success_sample_rate = 0.0; + config->failure_sample_rate = 1.0; + InitContext(std::move(config)); + + BeaconVector beacons; + + context_->OnBeacon(MakeBeacon(&time_)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + scoped_ptr<DomainReliabilityBeacon> beacon(MakeBeacon(&time_)); + beacon->status = "ok"; + beacon->chrome_error = net::OK; + context_->OnBeacon(std::move(beacon)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); +} + +TEST_F(DomainReliabilityContextTest, SuccessSampleOnly) { + scoped_ptr<DomainReliabilityConfig> config(MakeTestConfig()); + config->success_sample_rate = 1.0; + config->failure_sample_rate = 0.0; + InitContext(std::move(config)); + + BeaconVector beacons; + + context_->OnBeacon(MakeBeacon(&time_)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(0u, beacons.size()); + + scoped_ptr<DomainReliabilityBeacon> beacon(MakeBeacon(&time_)); + beacon->status = "ok"; + beacon->chrome_error = net::OK; + context_->OnBeacon(std::move(beacon)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); +} + +TEST_F(DomainReliabilityContextTest, SampleAllBeacons) { + scoped_ptr<DomainReliabilityConfig> config(MakeTestConfig()); + config->success_sample_rate = 1.0; + config->failure_sample_rate = 1.0; + InitContext(std::move(config)); + + BeaconVector beacons; + + context_->OnBeacon(MakeBeacon(&time_)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + + scoped_ptr<DomainReliabilityBeacon> beacon(MakeBeacon(&time_)); + beacon->status = "ok"; + beacon->chrome_error = net::OK; + context_->OnBeacon(std::move(beacon)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(2u, beacons.size()); +} + +TEST_F(DomainReliabilityContextTest, SampleNoBeacons) { + scoped_ptr<DomainReliabilityConfig> config(MakeTestConfig()); + config->success_sample_rate = 0.0; + config->failure_sample_rate = 0.0; + InitContext(std::move(config)); + + BeaconVector beacons; + + context_->OnBeacon(MakeBeacon(&time_)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(0u, beacons.size()); + + scoped_ptr<DomainReliabilityBeacon> beacon(MakeBeacon(&time_)); + beacon->status = "ok"; + beacon->chrome_error = net::OK; + context_->OnBeacon(std::move(beacon)); + context_->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(0u, beacons.size()); +} + +// TODO(ttuttle): Add beacon_unittest.cc to test serialization. + +} // namespace +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/dispatcher.cc b/chromium/components/domain_reliability/dispatcher.cc new file mode 100644 index 00000000000..5d0ed1ea9c9 --- /dev/null +++ b/chromium/components/domain_reliability/dispatcher.cc @@ -0,0 +1,119 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/dispatcher.h" + +#include <utility> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/message_loop/message_loop.h" +#include "base/stl_util.h" +#include "base/timer/timer.h" +#include "components/domain_reliability/util.h" + +namespace domain_reliability { + +struct DomainReliabilityDispatcher::Task { + Task(const base::Closure& closure, + scoped_ptr<MockableTime::Timer> timer, + base::TimeDelta min_delay, + base::TimeDelta max_delay); + ~Task(); + + base::Closure closure; + scoped_ptr<MockableTime::Timer> timer; + base::TimeDelta min_delay; + base::TimeDelta max_delay; + bool eligible; +}; + +DomainReliabilityDispatcher::Task::Task(const base::Closure& closure, + scoped_ptr<MockableTime::Timer> timer, + base::TimeDelta min_delay, + base::TimeDelta max_delay) + : closure(closure), + timer(std::move(timer)), + min_delay(min_delay), + max_delay(max_delay), + eligible(false) {} + +DomainReliabilityDispatcher::Task::~Task() {} + +DomainReliabilityDispatcher::DomainReliabilityDispatcher(MockableTime* time) + : time_(time) {} + +DomainReliabilityDispatcher::~DomainReliabilityDispatcher() { + // TODO(ttuttle): STLElementDeleter? + STLDeleteElements(&tasks_); +} + +void DomainReliabilityDispatcher::ScheduleTask( + const base::Closure& closure, + base::TimeDelta min_delay, + base::TimeDelta max_delay) { + DCHECK(!closure.is_null()); + // Would be DCHECK_LE, but you can't << a TimeDelta. + DCHECK(min_delay <= max_delay); + + Task* task = new Task(closure, time_->CreateTimer(), min_delay, max_delay); + tasks_.insert(task); + if (max_delay.InMicroseconds() < 0) + RunAndDeleteTask(task); + else if (min_delay.InMicroseconds() < 0) + MakeTaskEligible(task); + else + MakeTaskWaiting(task); +} + +void DomainReliabilityDispatcher::RunEligibleTasks() { + // Move all eligible tasks to a separate set so that eligible_tasks_.erase in + // RunAndDeleteTask won't erase elements out from under the iterator. (Also + // keeps RunEligibleTasks from running forever if a task adds a new, already- + // eligible task that does the same, and so on.) + std::set<Task*> tasks; + tasks.swap(eligible_tasks_); + + for (auto& task : tasks) { + DCHECK(task); + DCHECK(task->eligible); + RunAndDeleteTask(task); + } +} + +void DomainReliabilityDispatcher::MakeTaskWaiting(Task* task) { + DCHECK(task); + DCHECK(!task->eligible); + DCHECK(!task->timer->IsRunning()); + task->timer->Start(FROM_HERE, + task->min_delay, + base::Bind(&DomainReliabilityDispatcher::MakeTaskEligible, + base::Unretained(this), + task)); +} + +void +DomainReliabilityDispatcher::MakeTaskEligible(Task* task) { + DCHECK(task); + DCHECK(!task->eligible); + task->eligible = true; + eligible_tasks_.insert(task); + task->timer->Start(FROM_HERE, + task->max_delay - task->min_delay, + base::Bind(&DomainReliabilityDispatcher::RunAndDeleteTask, + base::Unretained(this), + task)); +} + +void DomainReliabilityDispatcher::RunAndDeleteTask(Task* task) { + DCHECK(task); + DCHECK(!task->closure.is_null()); + task->closure.Run(); + if (task->eligible) + eligible_tasks_.erase(task); + tasks_.erase(task); + delete task; +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/dispatcher.h b/chromium/components/domain_reliability/dispatcher.h new file mode 100644 index 00000000000..2c5d2d79dbf --- /dev/null +++ b/chromium/components/domain_reliability/dispatcher.h @@ -0,0 +1,63 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_DISPATCHER_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_DISPATCHER_H_ + +#include <set> + +#include "base/callback_forward.h" +#include "base/time/time.h" +#include "components/domain_reliability/domain_reliability_export.h" + +namespace tracked_objects { +class Location; +} // namespace tracked_objects + +namespace domain_reliability { + +class MockableTime; + +// Runs tasks during a specified interval. Calling |RunEligibleTasks| gives any +// task a chance to run early (if the minimum delay has already passed); tasks +// that aren't run early will be run once their maximum delay has passed. +// +// (See scheduler.h for an explanation of how the intervals are chosen.) +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityDispatcher { + public: + explicit DomainReliabilityDispatcher(MockableTime* time); + ~DomainReliabilityDispatcher(); + + // Schedules |task| to be executed between |min_delay| and |max_delay| from + // now. The task will be run at most |max_delay| from now; once |min_delay| + // has passed, any call to |RunEligibleTasks| will run the task earlier than + // that. + void ScheduleTask(const base::Closure& task, + base::TimeDelta min_delay, + base::TimeDelta max_delay); + + // Runs all tasks whose minimum delay has already passed. + void RunEligibleTasks(); + + private: + struct Task; + + // Adds |task| to the set of all tasks, but not the set of eligible tasks. + void MakeTaskWaiting(Task* task); + + // Adds |task| to the set of eligible tasks, and also the set of all tasks + // if not already there. + void MakeTaskEligible(Task* task); + + // Runs |task|'s callback, removes it from both sets, and deletes it. + void RunAndDeleteTask(Task* task); + + MockableTime* time_; + std::set<Task*> tasks_; + std::set<Task*> eligible_tasks_; +}; + +} // namespace domain_reliability + +#endif diff --git a/chromium/components/domain_reliability/dispatcher_unittest.cc b/chromium/components/domain_reliability/dispatcher_unittest.cc new file mode 100644 index 00000000000..0ffdbf059f3 --- /dev/null +++ b/chromium/components/domain_reliability/dispatcher_unittest.cc @@ -0,0 +1,63 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/dispatcher.h" + +#include "base/bind.h" +#include "components/domain_reliability/test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +using base::TimeDelta; +using base::TimeTicks; + +class DomainReliabilityDispatcherTest : public testing::Test { + public: + DomainReliabilityDispatcherTest() : dispatcher_(&time_) {} + + protected: + MockTime time_; + DomainReliabilityDispatcher dispatcher_; +}; + +TEST_F(DomainReliabilityDispatcherTest, Create) { +} + +TEST_F(DomainReliabilityDispatcherTest, TaskDoesntRunEarly) { + TimeDelta delay = TimeDelta::FromSeconds(1); + TestCallback callback; + + dispatcher_.ScheduleTask(callback.callback(), 2 * delay, 3 * delay); + time_.Advance(delay); + dispatcher_.RunEligibleTasks(); + EXPECT_FALSE(callback.called()); +} + +TEST_F(DomainReliabilityDispatcherTest, TaskRunsWhenEligible) { + TimeDelta delay = TimeDelta::FromSeconds(1); + TestCallback callback; + + dispatcher_.ScheduleTask(callback.callback(), 2 * delay, 3 * delay); + time_.Advance(2 * delay); + EXPECT_FALSE(callback.called()); + dispatcher_.RunEligibleTasks(); + EXPECT_TRUE(callback.called()); + time_.Advance(delay); +} + +TEST_F(DomainReliabilityDispatcherTest, TaskRunsAtDeadline) { + TimeDelta delay = TimeDelta::FromSeconds(1); + TestCallback callback; + + dispatcher_.ScheduleTask(callback.callback(), 2 * delay, 3 * delay); + time_.Advance(2 * delay); + EXPECT_FALSE(callback.called()); + time_.Advance(delay); + EXPECT_TRUE(callback.called()); +} + +} // namespace +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/domain_reliability_export.h b/chromium/components/domain_reliability/domain_reliability_export.h new file mode 100644 index 00000000000..e37571c12a8 --- /dev/null +++ b/chromium/components/domain_reliability/domain_reliability_export.h @@ -0,0 +1,32 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_DOMAIN_RELIABILITY_EXPORT_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_DOMAIN_RELIABILITY_EXPORT_H_ + +#if defined(COMPONENT_BUILD) +#if defined(WIN32) + +#if defined(DOMAIN_RELIABILITY_IMPLEMENTATION) +#define DOMAIN_RELIABILITY_EXPORT __declspec(dllexport) +#else +#define DOMAIN_RELIABILITY_EXPORT __declspec(dllimport) +#endif + +#else // defined(WIN32) + +#if defined(DOMAIN_RELIABILITY_IMPLEMENTATION) +#define DOMAIN_RELIABILITY_EXPORT __attribute__((visibility("default"))) +#else +#define DOMAIN_RELIABILITY_EXPORT +#endif + +#endif // defined(WIN32) +#else // defined(COMPONENT_BUILD) + +#define DOMAIN_RELIABILITY_EXPORT + +#endif + +#endif // COMPONENTS_DOMAIN_RELIABILITY_DOMAIN_RELIABILITY_EXPORT_H_ diff --git a/chromium/components/domain_reliability/google_configs.cc b/chromium/components/domain_reliability/google_configs.cc new file mode 100644 index 00000000000..0e0647cdd5b --- /dev/null +++ b/chromium/components/domain_reliability/google_configs.cc @@ -0,0 +1,550 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/google_configs.h" + +#include <stddef.h> + +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" +#include "components/domain_reliability/config.h" + +namespace domain_reliability { + +namespace { + +struct GoogleConfigParams { + const char* hostname; + bool include_subdomains; + + // If true, prepend a collector URL within https://|hostname|/. + bool include_origin_specific_collector; + + // If true, also add a config for www.|hostname|. + // + // |include_subdomains| will be false in the extra config, but + // |include_origin_specific_collector| will be respected, and will use the + // www subdomain as the origin for the collector so it matches the config. + bool duplicate_for_www; +}; + +const GoogleConfigParams kGoogleConfigs[] = { + // Origins with subdomains and same-origin collectors. + { "google.ac", true, true, true }, + { "google.ad", true, true, true }, + { "google.ae", true, true, true }, + { "google.af", true, true, true }, + { "google.ag", true, true, true }, + { "google.al", true, true, true }, + { "google.am", true, true, true }, + { "google.as", true, true, true }, + { "google.at", true, true, true }, + { "google.az", true, true, true }, + { "google.ba", true, true, true }, + { "google.be", true, true, true }, + { "google.bf", true, true, true }, + { "google.bg", true, true, true }, + { "google.bi", true, true, true }, + { "google.bj", true, true, true }, + { "google.bs", true, true, true }, + { "google.bt", true, true, true }, + { "google.by", true, true, true }, + { "google.ca", true, true, true }, + { "google.cc", true, true, true }, + { "google.cd", true, true, true }, + { "google.cf", true, true, true }, + { "google.cg", true, true, true }, + { "google.ch", true, true, true }, + { "google.ci", true, true, true }, + { "google.cl", true, true, true }, + { "google.cm", true, true, true }, + { "google.cn", true, true, true }, + { "google.co.ao", true, true, true }, + { "google.co.bw", true, true, true }, + { "google.co.ck", true, true, true }, + { "google.co.cr", true, true, true }, + { "google.co.hu", true, true, true }, + { "google.co.id", true, true, true }, + { "google.co.il", true, true, true }, + { "google.co.im", true, true, true }, + { "google.co.in", true, true, true }, + { "google.co.je", true, true, true }, + { "google.co.jp", true, true, true }, + { "google.co.ke", true, true, true }, + { "google.co.kr", true, true, true }, + { "google.co.ls", true, true, true }, + { "google.co.ma", true, true, true }, + { "google.co.mz", true, true, true }, + { "google.co.nz", true, true, true }, + { "google.co.th", true, true, true }, + { "google.co.tz", true, true, true }, + { "google.co.ug", true, true, true }, + { "google.co.uk", true, true, true }, + { "google.co.uz", true, true, true }, + { "google.co.ve", true, true, true }, + { "google.co.vi", true, true, true }, + { "google.co.za", true, true, true }, + { "google.co.zm", true, true, true }, + { "google.co.zw", true, true, true }, + { "google.com", true, true, false }, + { "google.com.af", true, true, true }, + { "google.com.ag", true, true, true }, + { "google.com.ai", true, true, true }, + { "google.com.ar", true, true, true }, + { "google.com.au", true, true, true }, + { "google.com.bd", true, true, true }, + { "google.com.bh", true, true, true }, + { "google.com.bn", true, true, true }, + { "google.com.bo", true, true, true }, + { "google.com.br", true, true, true }, + { "google.com.by", true, true, true }, + { "google.com.bz", true, true, true }, + { "google.com.cn", true, true, true }, + { "google.com.co", true, true, true }, + { "google.com.cu", true, true, true }, + { "google.com.cy", true, true, true }, + { "google.com.do", true, true, true }, + { "google.com.ec", true, true, true }, + { "google.com.eg", true, true, true }, + { "google.com.et", true, true, true }, + { "google.com.fj", true, true, true }, + { "google.com.ge", true, true, true }, + { "google.com.gh", true, true, true }, + { "google.com.gi", true, true, true }, + { "google.com.gr", true, true, true }, + { "google.com.gt", true, true, true }, + { "google.com.hk", true, true, true }, + { "google.com.iq", true, true, true }, + { "google.com.jm", true, true, true }, + { "google.com.jo", true, true, true }, + { "google.com.kh", true, true, true }, + { "google.com.kw", true, true, true }, + { "google.com.lb", true, true, true }, + { "google.com.ly", true, true, true }, + { "google.com.mm", true, true, true }, + { "google.com.mt", true, true, true }, + { "google.com.mx", true, true, true }, + { "google.com.my", true, true, true }, + { "google.com.na", true, true, true }, + { "google.com.nf", true, true, true }, + { "google.com.ng", true, true, true }, + { "google.com.ni", true, true, true }, + { "google.com.np", true, true, true }, + { "google.com.nr", true, true, true }, + { "google.com.om", true, true, true }, + { "google.com.pa", true, true, true }, + { "google.com.pe", true, true, true }, + { "google.com.pg", true, true, true }, + { "google.com.ph", true, true, true }, + { "google.com.pk", true, true, true }, + { "google.com.pl", true, true, true }, + { "google.com.pr", true, true, true }, + { "google.com.py", true, true, true }, + { "google.com.qa", true, true, true }, + { "google.com.ru", true, true, true }, + { "google.com.sa", true, true, true }, + { "google.com.sb", true, true, true }, + { "google.com.sg", true, true, true }, + { "google.com.sl", true, true, true }, + { "google.com.sv", true, true, true }, + { "google.com.tj", true, true, true }, + { "google.com.tn", true, true, true }, + { "google.com.tr", true, true, true }, + { "google.com.tw", true, true, true }, + { "google.com.ua", true, true, true }, + { "google.com.uy", true, true, true }, + { "google.com.vc", true, true, true }, + { "google.com.ve", true, true, true }, + { "google.com.vn", true, true, true }, + { "google.cv", true, true, true }, + { "google.cz", true, true, true }, + { "google.de", true, true, true }, + { "google.dj", true, true, true }, + { "google.dk", true, true, true }, + { "google.dm", true, true, true }, + { "google.dz", true, true, true }, + { "google.ee", true, true, true }, + { "google.es", true, true, true }, + { "google.fi", true, true, true }, + { "google.fm", true, true, true }, + { "google.fr", true, true, true }, + { "google.ga", true, true, true }, + { "google.ge", true, true, true }, + { "google.gg", true, true, true }, + { "google.gl", true, true, true }, + { "google.gm", true, true, true }, + { "google.gp", true, true, true }, + { "google.gr", true, true, true }, + { "google.gy", true, true, true }, + { "google.hk", true, true, true }, + { "google.hn", true, true, true }, + { "google.hr", true, true, true }, + { "google.ht", true, true, true }, + { "google.hu", true, true, true }, + { "google.ie", true, true, true }, + { "google.im", true, true, true }, + { "google.iq", true, true, true }, + { "google.ir", true, true, true }, + { "google.is", true, true, true }, + { "google.it", true, true, true }, + { "google.it.ao", true, true, true }, + { "google.je", true, true, true }, + { "google.jo", true, true, true }, + { "google.jp", true, true, true }, + { "google.kg", true, true, true }, + { "google.ki", true, true, true }, + { "google.kz", true, true, true }, + { "google.la", true, true, true }, + { "google.li", true, true, true }, + { "google.lk", true, true, true }, + { "google.lt", true, true, true }, + { "google.lu", true, true, true }, + { "google.lv", true, true, true }, + { "google.md", true, true, true }, + { "google.me", true, true, true }, + { "google.mg", true, true, true }, + { "google.mk", true, true, true }, + { "google.ml", true, true, true }, + { "google.mn", true, true, true }, + { "google.ms", true, true, true }, + { "google.mu", true, true, true }, + { "google.mv", true, true, true }, + { "google.mw", true, true, true }, + { "google.ne", true, true, true }, + { "google.ne.jp", true, true, true }, + { "google.ng", true, true, true }, + { "google.nl", true, true, true }, + { "google.no", true, true, true }, + { "google.nr", true, true, true }, + { "google.nu", true, true, true }, + { "google.off.ai", true, true, true }, + { "google.pk", true, true, true }, + { "google.pl", true, true, true }, + { "google.pn", true, true, true }, + { "google.ps", true, true, true }, + { "google.pt", true, true, true }, + { "google.ro", true, true, true }, + { "google.rs", true, true, true }, + { "google.ru", true, true, true }, + { "google.rw", true, true, true }, + { "google.sc", true, true, true }, + { "google.se", true, true, true }, + { "google.sh", true, true, true }, + { "google.si", true, true, true }, + { "google.sk", true, true, true }, + { "google.sm", true, true, true }, + { "google.sn", true, true, true }, + { "google.so", true, true, true }, + { "google.sr", true, true, true }, + { "google.st", true, true, true }, + { "google.td", true, true, true }, + { "google.tg", true, true, true }, + { "google.tk", true, true, true }, + { "google.tl", true, true, true }, + { "google.tm", true, true, true }, + { "google.tn", true, true, true }, + { "google.to", true, true, true }, + { "google.tt", true, true, true }, + { "google.us", true, true, true }, + { "google.uz", true, true, true }, + { "google.vg", true, true, true }, + { "google.vu", true, true, true }, + { "google.ws", true, true, true }, + { "l.google.com", true, true, true }, + + // Origins with subdomains and without same-origin collectors. + { "2mdn.net", true, false, false }, + { "adgoogle.net", true, false, false }, + { "admeld.com", true, false, false }, + { "admob.biz", true, false, false }, + { "admob.co.in", true, false, false }, + { "admob.co.kr", true, false, false }, + { "admob.co.nz", true, false, false }, + { "admob.co.uk", true, false, false }, + { "admob.co.za", true, false, false }, + { "admob.com", true, false, false }, + { "admob.com.br", true, false, false }, + { "admob.com.es", true, false, false }, + { "admob.com.fr", true, false, false }, + { "admob.com.mx", true, false, false }, + { "admob.com.pt", true, false, false }, + { "admob.de", true, false, false }, + { "admob.dk", true, false, false }, + { "admob.es", true, false, false }, + { "admob.fi", true, false, false }, + { "admob.fr", true, false, false }, + { "admob.gr", true, false, false }, + { "admob.hk", true, false, false }, + { "admob.ie", true, false, false }, + { "admob.in", true, false, false }, + { "admob.it", true, false, false }, + { "admob.jp", true, false, false }, + { "admob.kr", true, false, false }, + { "admob.mobi", true, false, false }, + { "admob.no", true, false, false }, + { "admob.ph", true, false, false }, + { "admob.pt", true, false, false }, + { "admob.sg", true, false, false }, + { "admob.tw", true, false, false }, + { "admob.us", true, false, false }, + { "admob.vn", true, false, false }, + { "adwhirl.com", true, false, false }, + { "android.com", true, false, false }, + { "chromecast.com", true, false, false }, + { "chromeexperiments.com", true, false, false }, + { "chromestatus.com", true, false, false }, + { "chromium.org", true, false, false }, + { "cloudendpointsapis.com", true, false, false }, + { "dartmotif.com", true, false, false }, + { "dartsearch.net", true, false, false }, + { "doubleclick.com", true, false, false }, + { "doubleclick.ne.jp", true, false, false }, + { "doubleclick.net", true, false, false }, + { "doubleclickusercontent.com", true, false, false }, + { "fls.doubleclick.net", true, false, false }, + { "g.co", true, false, false }, + { "g.doubleclick.net", true, false, false }, + { "ggpht.com", true, false, false }, + { "gmodules.com", true, false, false }, + { "goo.gl", true, false, false }, + { "google-syndication.com", true, false, false }, + { "google.cat", true, false, false }, + { "google.info", true, false, false }, + { "google.jobs", true, false, false }, + { "google.net", true, false, false }, + { "google.org", true, false, false }, + { "googleadapis.com", true, false, false }, + { "googleadservices.com", true, false, false }, + { "googleadsserving.cn", true, false, false }, + { "googlealumni.com", true, false, false }, + { "googleapis.cn", true, false, false }, + { "googleapis.com", true, false, false }, + { "googleapps.com", true, false, false }, + { "googlecbs.com", true, false, false }, + { "googlecode.com", true, false, false }, + { "googlecommerce.com", true, false, false }, + { "googledrive.com", true, false, false }, + { "googleenterprise.com", true, false, false }, + { "googlefiber.com", true, false, false }, + { "googlefiber.net", true, false, false }, + { "googlegoro.com", true, false, false }, + { "googlehosted.com", true, false, false }, + { "googlepayments.com", true, false, false }, + { "googlesource.com", true, false, false }, + { "googlesyndication.com", true, false, false }, + { "googletagmanager.com", true, false, false }, + { "googletagservices.com", true, false, false }, + { "googleusercontent.com", true, false, false }, + { "gstatic.cn", true, false, false }, + { "gstatic.com", true, false, false }, + { "picasa.com", true, false, false }, + { "recaptcha.net", true, false, false }, + { "waze.com", true, false, false }, + { "withgoogle.com", true, false, false }, + { "youtu.be", true, false, false }, + { "youtube-3rd-party.com", true, false, false }, + { "youtube-nocookie.com", true, false, false }, + { "youtube.ae", true, false, false }, + { "youtube.al", true, false, false }, + { "youtube.am", true, false, false }, + { "youtube.at", true, false, false }, + { "youtube.az", true, false, false }, + { "youtube.ba", true, false, false }, + { "youtube.be", true, false, false }, + { "youtube.bg", true, false, false }, + { "youtube.bh", true, false, false }, + { "youtube.bo", true, false, false }, + { "youtube.ca", true, false, false }, + { "youtube.cat", true, false, false }, + { "youtube.ch", true, false, false }, + { "youtube.cl", true, false, false }, + { "youtube.co", true, false, false }, + { "youtube.co.ae", true, false, false }, + { "youtube.co.at", true, false, false }, + { "youtube.co.hu", true, false, false }, + { "youtube.co.id", true, false, false }, + { "youtube.co.il", true, false, false }, + { "youtube.co.in", true, false, false }, + { "youtube.co.jp", true, false, false }, + { "youtube.co.ke", true, false, false }, + { "youtube.co.kr", true, false, false }, + { "youtube.co.ma", true, false, false }, + { "youtube.co.nz", true, false, false }, + { "youtube.co.th", true, false, false }, + { "youtube.co.ug", true, false, false }, + { "youtube.co.uk", true, false, false }, + { "youtube.co.ve", true, false, false }, + { "youtube.co.za", true, false, false }, + { "youtube.com", true, false, false }, + { "youtube.com.ar", true, false, false }, + { "youtube.com.au", true, false, false }, + { "youtube.com.az", true, false, false }, + { "youtube.com.bh", true, false, false }, + { "youtube.com.bo", true, false, false }, + { "youtube.com.br", true, false, false }, + { "youtube.com.by", true, false, false }, + { "youtube.com.co", true, false, false }, + { "youtube.com.do", true, false, false }, + { "youtube.com.ee", true, false, false }, + { "youtube.com.eg", true, false, false }, + { "youtube.com.es", true, false, false }, + { "youtube.com.gh", true, false, false }, + { "youtube.com.gr", true, false, false }, + { "youtube.com.gt", true, false, false }, + { "youtube.com.hk", true, false, false }, + { "youtube.com.hr", true, false, false }, + { "youtube.com.jm", true, false, false }, + { "youtube.com.jo", true, false, false }, + { "youtube.com.kw", true, false, false }, + { "youtube.com.lb", true, false, false }, + { "youtube.com.lv", true, false, false }, + { "youtube.com.mk", true, false, false }, + { "youtube.com.mt", true, false, false }, + { "youtube.com.mx", true, false, false }, + { "youtube.com.my", true, false, false }, + { "youtube.com.ng", true, false, false }, + { "youtube.com.om", true, false, false }, + { "youtube.com.pe", true, false, false }, + { "youtube.com.ph", true, false, false }, + { "youtube.com.pk", true, false, false }, + { "youtube.com.pt", true, false, false }, + { "youtube.com.qa", true, false, false }, + { "youtube.com.ro", true, false, false }, + { "youtube.com.sa", true, false, false }, + { "youtube.com.sg", true, false, false }, + { "youtube.com.tn", true, false, false }, + { "youtube.com.tr", true, false, false }, + { "youtube.com.tw", true, false, false }, + { "youtube.com.ua", true, false, false }, + { "youtube.com.uy", true, false, false }, + { "youtube.com.ve", true, false, false }, + { "youtube.cz", true, false, false }, + { "youtube.de", true, false, false }, + { "youtube.dk", true, false, false }, + { "youtube.ee", true, false, false }, + { "youtube.es", true, false, false }, + { "youtube.fi", true, false, false }, + { "youtube.fr", true, false, false }, + { "youtube.ge", true, false, false }, + { "youtube.gr", true, false, false }, + { "youtube.gt", true, false, false }, + { "youtube.hk", true, false, false }, + { "youtube.hr", true, false, false }, + { "youtube.hu", true, false, false }, + { "youtube.ie", true, false, false }, + { "youtube.in", true, false, false }, + { "youtube.is", true, false, false }, + { "youtube.it", true, false, false }, + { "youtube.jo", true, false, false }, + { "youtube.jp", true, false, false }, + { "youtube.kr", true, false, false }, + { "youtube.lk", true, false, false }, + { "youtube.lt", true, false, false }, + { "youtube.lv", true, false, false }, + { "youtube.ma", true, false, false }, + { "youtube.md", true, false, false }, + { "youtube.me", true, false, false }, + { "youtube.mk", true, false, false }, + { "youtube.mx", true, false, false }, + { "youtube.my", true, false, false }, + { "youtube.ng", true, false, false }, + { "youtube.nl", true, false, false }, + { "youtube.no", true, false, false }, + { "youtube.pe", true, false, false }, + { "youtube.ph", true, false, false }, + { "youtube.pk", true, false, false }, + { "youtube.pl", true, false, false }, + { "youtube.pr", true, false, false }, + { "youtube.pt", true, false, false }, + { "youtube.qa", true, false, false }, + { "youtube.ro", true, false, false }, + { "youtube.rs", true, false, false }, + { "youtube.ru", true, false, false }, + { "youtube.sa", true, false, false }, + { "youtube.se", true, false, false }, + { "youtube.sg", true, false, false }, + { "youtube.si", true, false, false }, + { "youtube.sk", true, false, false }, + { "youtube.sn", true, false, false }, + { "youtube.tn", true, false, false }, + { "youtube.ua", true, false, false }, + { "youtube.ug", true, false, false }, + { "youtube.uy", true, false, false }, + { "youtube.vn", true, false, false }, + { "youtubeeducation.com", true, false, false }, + { "youtubemobilesupport.com", true, false, false }, + { "ytimg.com", true, false, false }, + + // Origins without subdomains and with same-origin collectors. + { "accounts.google.com", false, true, false }, + { "apis.google.com", false, true, false }, + { "b.mail.google.com", false, true, false }, + { "chatenabled.mail.google.com", false, true, false }, + { "ddm.google.com", false, true, false }, + { "gmail.com", false, true, false }, + { "gmail.google.com", false, true, false }, + { "mail.google.com", false, true, false }, + { "mail-attachment.googleusercontent.com", false, true, false }, + { "www.gmail.com", false, true, false }, + + // Origins without subdomains or same-origin collectors. + { "ad.doubleclick.net", false, false, false }, + { "drive.google.com", false, false, false }, + { "redirector.googlevideo.com", false, false, false }, +}; + +const char* kGoogleStandardCollectors[] = { + "https://beacons.gvt2.com/domainreliability/upload", + "https://beacons2.gvt2.com/domainreliability/upload", + "https://beacons3.gvt2.com/domainreliability/upload", + "https://beacons4.gvt2.com/domainreliability/upload", + "https://beacons5.gvt2.com/domainreliability/upload", + "https://beacons5.gvt3.com/domainreliability/upload", + "https://clients2.google.com/domainreliability/upload", +}; + +const char* kGoogleOriginSpecificCollectorPathString = + "/domainreliability/upload"; + +static scoped_ptr<DomainReliabilityConfig> +CreateGoogleConfig(const GoogleConfigParams& params, bool is_www) { + if (is_www) + DCHECK(params.duplicate_for_www); + + std::string hostname = (is_www ? "www." : "") + std::string(params.hostname); + bool include_subdomains = params.include_subdomains && !is_www; + + scoped_ptr<DomainReliabilityConfig> config(new DomainReliabilityConfig()); + config->origin = GURL("https://" + hostname + "/"); + config->include_subdomains = include_subdomains; + config->collectors.clear(); + if (params.include_origin_specific_collector) { + GURL::Replacements replacements; + replacements.SetPathStr(kGoogleOriginSpecificCollectorPathString); + config->collectors.push_back( + new GURL(config->origin.ReplaceComponents(replacements))); + } + for (size_t i = 0; i < arraysize(kGoogleStandardCollectors); i++) + config->collectors.push_back(new GURL(kGoogleStandardCollectors[i])); + config->success_sample_rate = 0.05; + config->failure_sample_rate = 1.00; + config->path_prefixes.clear(); + return config; +} + +} // namespace + +// static +void GetAllGoogleConfigs( + std::vector<DomainReliabilityConfig*>* configs_out) { + configs_out->clear(); + + for (auto& params : kGoogleConfigs) { + configs_out->push_back(CreateGoogleConfig(params, false).release()); + if (params.duplicate_for_www) + configs_out->push_back(CreateGoogleConfig(params, true).release()); + } +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/google_configs.h b/chromium/components/domain_reliability/google_configs.h new file mode 100644 index 00000000000..51326c2abdf --- /dev/null +++ b/chromium/components/domain_reliability/google_configs.h @@ -0,0 +1,19 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_GOOGLE_CONFIGS_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_GOOGLE_CONFIGS_H_ + +#include "base/memory/scoped_ptr.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/domain_reliability_export.h" + +namespace domain_reliability { + +void DOMAIN_RELIABILITY_EXPORT GetAllGoogleConfigs( + std::vector<DomainReliabilityConfig*>* configs_out); + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_GOOGLE_CONFIGS_H_ diff --git a/chromium/components/domain_reliability/google_configs_unittest.cc b/chromium/components/domain_reliability/google_configs_unittest.cc new file mode 100644 index 00000000000..f7cf7c4d631 --- /dev/null +++ b/chromium/components/domain_reliability/google_configs_unittest.cc @@ -0,0 +1,34 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/google_configs.h" + +#include "base/stl_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { + +namespace { + +typedef std::vector<DomainReliabilityConfig*> ConfigPointerVector; + +TEST(DomainReliabilityGoogleConfigsTest, Enumerate) { + ConfigPointerVector configs; + STLElementDeleter<ConfigPointerVector> configs_deleter(&configs); + + GetAllGoogleConfigs(&configs); +} + +TEST(DomainReliabilityGoogleConfigsTest, ConfigsAreValid) { + ConfigPointerVector configs; + STLElementDeleter<ConfigPointerVector> configs_deleter(&configs); + + GetAllGoogleConfigs(&configs); + for (auto config : configs) + EXPECT_TRUE(config->IsValid()); +} + +} // namespace + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/header.cc b/chromium/components/domain_reliability/header.cc new file mode 100644 index 00000000000..09b09d3b6ec --- /dev/null +++ b/chromium/components/domain_reliability/header.cc @@ -0,0 +1,318 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/header.h" + +#include <stdint.h> +#include <string> + +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_tokenizer.h" +#include "components/domain_reliability/config.h" +#include "content/public/common/origin_util.h" + +namespace { + +// Parses directives in the format ("foo; bar=value for bar; baz; quux=123") +// used by NEL. +class DirectiveHeaderValueParser { + public: + enum State { + BEFORE_NAME, + AFTER_NAME, + BEFORE_VALUE, + AFTER_DIRECTIVE, + SYNTAX_ERROR + }; + + DirectiveHeaderValueParser(base::StringPiece value) + : value_(value.data()), + tokenizer_(value_.begin(), value_.end(), ";= "), + stopped_with_error_(false) { + tokenizer_.set_options(base::StringTokenizer::RETURN_DELIMS); + tokenizer_.set_quote_chars("\"'"); + } + + // Gets the next directive, if there is one. Returns whether there was one. + bool GetNext() { + if (stopped_with_error_) + return false; + + directive_name_ = base::StringPiece(); + directive_has_value_ = false; + directive_values_.clear(); + + State state = BEFORE_NAME; + while (state != AFTER_DIRECTIVE && state != SYNTAX_ERROR + && tokenizer_.GetNext()) { + if (*tokenizer_.token_begin() == ' ') + continue; + + switch (state) { + case BEFORE_NAME: + state = DoBeforeName(); + break; + case AFTER_NAME: + state = DoAfterName(); + break; + case BEFORE_VALUE: + state = DoBeforeValue(); + break; + case AFTER_DIRECTIVE: + case SYNTAX_ERROR: + NOTREACHED(); + break; + } + } + + switch (state) { + // If the parser just read the last directive, it may be in one of these + // states, so return true to yield that directive. + case AFTER_NAME: + case BEFORE_VALUE: + case AFTER_DIRECTIVE: + return true; + + // If the parser never found a name, return false, since it doesn't have + // a new directive for the caller. + case BEFORE_NAME: + return false; + + case SYNTAX_ERROR: + stopped_with_error_ = true; + return false; + + default: + NOTREACHED(); + return false; + } + } + + + base::StringPiece directive_name() const { return directive_name_; } + bool directive_has_value() const { return directive_has_value_; } + const std::vector<base::StringPiece>& directive_values() const { + return directive_values_; + } + bool stopped_with_error() const { return stopped_with_error_; } + + private: + State DoBeforeName() { + if (tokenizer_.token_is_delim()) + return SYNTAX_ERROR; + + directive_name_ = tokenizer_.token_piece(); + return AFTER_NAME; + } + + State DoAfterName() { + if (tokenizer_.token_is_delim()) { + char token_begin = *tokenizer_.token_begin(); + // Name can be followed by =value, ;, or just EOF. + if (token_begin == '=') { + directive_has_value_ = true; + return BEFORE_VALUE; + } + if (token_begin == ';') + return AFTER_DIRECTIVE; + } + return SYNTAX_ERROR; + } + + State DoBeforeValue() { + if (tokenizer_.token_is_delim()) { + char token_begin = *tokenizer_.token_begin(); + if (token_begin == ';') + return AFTER_DIRECTIVE; + return SYNTAX_ERROR; + } + + directive_values_.push_back(tokenizer_.token_piece()); + return BEFORE_VALUE; + } + + std::string value_; + base::StringTokenizer tokenizer_; + + base::StringPiece directive_name_; + bool directive_has_value_; + std::vector<base::StringPiece> directive_values_; + bool stopped_with_error_; +}; + +bool Unquote(const std::string& in, std::string* out) { + char first = in[0]; + char last = in[in.length() - 1]; + + if (((first == '"') ^ (last == '"')) || ((first == '<') ^ (last == '>'))) + return false; + + if ((first == '"') || (first == '<')) + *out = in.substr(1, in.length() - 2); + else + *out = in; + return true; +} + +bool ParseReportUri(const std::vector<base::StringPiece> in, + ScopedVector<GURL>* out) { + if (in.size() < 1u) + return false; + + out->clear(); + for (const auto& in_token : in) { + std::string unquoted; + if (!Unquote(in_token.as_string(), &unquoted)) + return false; + GURL url(unquoted); + if (!url.is_valid() || !content::IsOriginSecure(url)) + return false; + out->push_back(new GURL(url)); + } + + return true; +} + +bool ParseMaxAge(const std::vector<base::StringPiece> in, + base::TimeDelta* out) { + if (in.size() != 1u) + return false; + + int64_t seconds; + if (!base::StringToInt64(in[0], &seconds)) + return false; + + if (seconds < 0) + return false; + + *out = base::TimeDelta::FromSeconds(seconds); + return true; +} + +} // namespace + +namespace domain_reliability { + +DomainReliabilityHeader::~DomainReliabilityHeader() {} + +// static +scoped_ptr<DomainReliabilityHeader> +DomainReliabilityHeader::Parse(base::StringPiece value) { + ScopedVector<GURL> report_uri; + base::TimeDelta max_age; + bool include_subdomains = false; + + bool got_report_uri = false; + bool got_max_age = false; + bool got_include_subdomains = false; + + DirectiveHeaderValueParser parser(value); + while (parser.GetNext()) { + base::StringPiece name = parser.directive_name(); + if (name == "report-uri") { + if (got_report_uri + || !parser.directive_has_value() + || !ParseReportUri(parser.directive_values(), &report_uri)) { + return make_scoped_ptr(new DomainReliabilityHeader(PARSE_ERROR)); + } + got_report_uri = true; + } else if (name == "max-age") { + if (got_max_age + || !parser.directive_has_value() + || !ParseMaxAge(parser.directive_values(), &max_age)) { + return make_scoped_ptr(new DomainReliabilityHeader(PARSE_ERROR)); + } + got_max_age = true; + } else if (name == "includeSubdomains") { + if (got_include_subdomains || + parser.directive_has_value()) { + return make_scoped_ptr(new DomainReliabilityHeader(PARSE_ERROR)); + } + include_subdomains = true; + got_include_subdomains = true; + } else { + LOG(WARNING) << "Ignoring unknown NEL header directive " << name << "."; + } + } + + if (parser.stopped_with_error() || !got_max_age) + return make_scoped_ptr(new DomainReliabilityHeader(PARSE_ERROR)); + + if (max_age == base::TimeDelta::FromMicroseconds(0)) + return make_scoped_ptr(new DomainReliabilityHeader(PARSE_CLEAR_CONFIG)); + + if (!got_report_uri) + return make_scoped_ptr(new DomainReliabilityHeader(PARSE_ERROR)); + + scoped_ptr<DomainReliabilityConfig> config(new DomainReliabilityConfig()); + config->include_subdomains = include_subdomains; + config->collectors.clear(); + config->collectors.swap(report_uri); + config->success_sample_rate = 0.0; + config->failure_sample_rate = 1.0; + config->path_prefixes.clear(); + return make_scoped_ptr(new DomainReliabilityHeader( + PARSE_SET_CONFIG, std::move(config), max_age)); +} + +const DomainReliabilityConfig& DomainReliabilityHeader::config() const { + DCHECK_EQ(PARSE_SET_CONFIG, status_); + return *config_; +} + +base::TimeDelta DomainReliabilityHeader::max_age() const { + DCHECK_EQ(PARSE_SET_CONFIG, status_); + return max_age_; +} + +scoped_ptr<DomainReliabilityConfig> DomainReliabilityHeader::ReleaseConfig() { + DCHECK_EQ(PARSE_SET_CONFIG, status_); + status_ = PARSE_ERROR; + return std::move(config_); +} + +std::string DomainReliabilityHeader::ToString() const { + std::string string = ""; + int64_t max_age_s = max_age_.InSeconds(); + + if (config_->collectors.empty()) { + DCHECK_EQ(0, max_age_s); + } else { + string += "report-uri="; + for (const auto& uri : config_->collectors) + string += uri->spec() + " "; + // Remove trailing space. + string.erase(string.length() - 1, 1); + string += "; "; + } + + string += "max-age=" + base::Int64ToString(max_age_s) + "; "; + + if (config_->include_subdomains) + string += "includeSubdomains; "; + + // Remove trailing "; ". + string.erase(string.length() - 2, 2); + + return string; +} + +DomainReliabilityHeader::DomainReliabilityHeader(ParseStatus status) + : status_(status) { + DCHECK_NE(PARSE_SET_CONFIG, status_); +} + +DomainReliabilityHeader::DomainReliabilityHeader( + ParseStatus status, + scoped_ptr<DomainReliabilityConfig> config, + base::TimeDelta max_age) + : status_(status), + config_(std::move(config)), + max_age_(max_age) { + DCHECK_EQ(PARSE_SET_CONFIG, status_); + DCHECK(config_.get()); + DCHECK_NE(0, max_age_.InMicroseconds()); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/header.h b/chromium/components/domain_reliability/header.h new file mode 100644 index 00000000000..104968fffc0 --- /dev/null +++ b/chromium/components/domain_reliability/header.h @@ -0,0 +1,70 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_HEADER_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_HEADER_H_ + +#include <vector> + +#include "base/memory/scoped_ptr.h" +#include "base/strings/string_piece.h" +#include "base/time/time.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "url/gurl.h" + +namespace domain_reliability { + +struct DomainReliabilityConfig; + +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityHeader { + public: + // The outcome of the parse: PARSE_SET_CONFIG if the header specified a + // valid config to set, PARSE_CLEAR_CONFIG if the header requested that an + // existing config (if any) be cleared, or PARSE_ERROR if the heder did not + // parse correctly. + enum ParseStatus { + PARSE_SET_CONFIG, + PARSE_CLEAR_CONFIG, + PARSE_ERROR + }; + + ~DomainReliabilityHeader(); + + static scoped_ptr<DomainReliabilityHeader> Parse(base::StringPiece value); + + bool IsSetConfig() const { return status_ == PARSE_SET_CONFIG; } + bool IsClearConfig() const { return status_ == PARSE_CLEAR_CONFIG; } + bool IsParseError() const { return status_ == PARSE_ERROR; } + + ParseStatus status() const { return status_; } + const DomainReliabilityConfig& config() const; + base::TimeDelta max_age() const; + + scoped_ptr<DomainReliabilityConfig> ReleaseConfig(); + + std::string ToString() const; + + private: + // Constructor for PARSE_SET_CONFIG status. + DomainReliabilityHeader(ParseStatus status, + scoped_ptr<DomainReliabilityConfig> config, + base::TimeDelta max_age); + // Constructor for PARSE_CLEAR_CONFIG and PARSE_ERROR statuses. + DomainReliabilityHeader(ParseStatus status); + + ParseStatus status_; + + // The configuration specified by the header, if the status is + // PARSE_SET_CONFIG. + scoped_ptr<DomainReliabilityConfig> config_; + + // The max-age specified by the header, if the status is PARSE_SET_CONFIG. + base::TimeDelta max_age_; + + DISALLOW_COPY_AND_ASSIGN(DomainReliabilityHeader); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_HEADER_H_ diff --git a/chromium/components/domain_reliability/header_unittest.cc b/chromium/components/domain_reliability/header_unittest.cc new file mode 100644 index 00000000000..d0aba162eaa --- /dev/null +++ b/chromium/components/domain_reliability/header_unittest.cc @@ -0,0 +1,157 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/header.h" + +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" +#include "base/strings/stringprintf.h" +#include "components/domain_reliability/config.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +class DomainReliabilityHeaderTest : public testing::Test { + protected: + DomainReliabilityHeaderTest() {} + ~DomainReliabilityHeaderTest() override {} + + void Parse(std::string value) { + parsed_ = DomainReliabilityHeader::Parse(value); + } + + const DomainReliabilityHeader* parsed() const { return parsed_.get(); } + + private: + scoped_ptr<DomainReliabilityHeader> parsed_; +}; + +bool CheckReportUris(const char* pipe_separated_expected_report_uris, + const ScopedVector<GURL>& actual_report_uris) { + if (!pipe_separated_expected_report_uris) + return actual_report_uris.empty(); + + std::vector<std::string> expected_report_uri_strings = + SplitString(pipe_separated_expected_report_uris, + "|", + base::KEEP_WHITESPACE, + base::SPLIT_WANT_ALL); + if (expected_report_uri_strings.size() != actual_report_uris.size()) + return false; + + for (size_t i = 0; i < actual_report_uris.size(); ++i) { + if (actual_report_uris[i]->spec() != expected_report_uri_strings[i]) + return false; + } + + return true; +} + +TEST_F(DomainReliabilityHeaderTest, SetConfig) { + static const struct { + const char* header; + const char* pipe_separated_report_uris; + int64_t max_age_in_seconds; + bool include_subdomains; + const char* description; + } test_cases[] = { + { "report-uri=https://a/; max-age=5", + "https://a/", 5, false, + "register" }, + { "report-uri=\"https://a/\"; max-age=5", + "https://a/", 5, false, + "register with quoted report-uri" }, + { "report-uri=<https://a/>; max-age=5", + "https://a/", 5, false, + "register with bracketed report-uri" }, + { "report-uri=https://a/ https://b/; max-age=5", + "https://a/|https://b/", 5, false, + "register with two report-uris" }, + { "report-uri=https://a/; max-age=5; includeSubdomains", + "https://a/", 5, true, + "register with includeSubdomains" }, + }; + + for (const auto& test_case : test_cases) { + std::string assert_prefix = base::StringPrintf( + "Valid set-config NEL header \"%s\" (%s) incorrectly parsed as ", + test_case.header, + test_case.description); + Parse(test_case.header); + EXPECT_FALSE(parsed()->IsParseError()) << assert_prefix << "invalid."; + EXPECT_FALSE(parsed()->IsClearConfig()) << assert_prefix << "clear-config."; + if (parsed()->IsParseError() || parsed()->IsClearConfig()) + continue; + EXPECT_TRUE(parsed()->IsSetConfig()); + + std::string assert_message = + assert_prefix + "\"" + parsed()->ToString() + "\""; + EXPECT_TRUE(CheckReportUris(test_case.pipe_separated_report_uris, + parsed()->config().collectors)) + << assert_message; + EXPECT_EQ(test_case.max_age_in_seconds, + parsed()->max_age().InSeconds()) + << assert_message; + EXPECT_EQ(test_case.include_subdomains, + parsed()->config().include_subdomains) + << assert_message; + } +} + +TEST_F(DomainReliabilityHeaderTest, ClearConfig) { + static const struct { + const char* header; + const char* description; + } test_cases[] = { + { "max-age=0", "unregister" }, + { "report-uri=https://a/; max-age=0", "unregister with report-uri" }, + { "max-age=0; includeSubdomains", "unregister with includeSubdomains" }, + }; + + for (const auto& test_case : test_cases) { + std::string assert_prefix = base::StringPrintf( + "Valid clear-config NEL header \"%s\" (%s) incorrectly parsed as ", + test_case.header, + test_case.description); + Parse(test_case.header); + EXPECT_FALSE(parsed()->IsParseError()) << assert_prefix << "invalid."; + EXPECT_FALSE(parsed()->IsSetConfig()) << assert_prefix << "set-config."; + } +} + +TEST_F(DomainReliabilityHeaderTest, Error) { + static const struct { + const char* header; + const char* description; + } test_cases[] = { + { "", "empty" }, + { "max-age=5", "report-uri missing with non-zero max-age" }, + { "report-uri; max-age=5", "report-uri has no value" }, + { "report-uri=; max-age=5", "report-uri value is empty" }, + { "report-uri=http://a/; max-age=5", "report-uri is insecure" }, + { "report-uri=https://a/ http://b/; max-age=5", + "one report-uri of two is insecure" }, + { "report-uri=\"https://a/; max-age=5", "report-uri is unbalanced" }, + { "report-uri=https://a/\"; max-age=5", "report-uri is unbalanced" }, + { "report-uri=<https://a/; max-age=5", "report-uri is unbalanced" }, + { "report-uri=https://a/>; max-age=5", "report-uri is unbalanced" }, + { "report-uri=https://a/", "max-age is missing" }, + { "report-uri=https://a/; max-age", "max-age has no value" }, + { "report-uri=https://a/; max-age=", "max-age value is empty" }, + { "report-uri=https://a/; max-age=a", "max-age is entirely non-numeric" }, + { "report-uri=https://a/; max-age=5a", "max-age is partly non-numeric" }, + { "report-uri=https://a/; max-age=5 5", "max-age has multiple values" }, + }; + + for (const auto& test_case : test_cases) { + Parse(test_case.header); + EXPECT_TRUE(parsed()->IsParseError()) + << "Invalid NEL header \"" << test_case.header << "\" (" + << test_case.description << ") incorrectly parsed as valid."; + } +} + +} // namespace +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/monitor.cc b/chromium/components/domain_reliability/monitor.cc new file mode 100644 index 00000000000..2fdafadcbd1 --- /dev/null +++ b/chromium/components/domain_reliability/monitor.cc @@ -0,0 +1,421 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/monitor.h" + +#include <utility> + +#include "base/command_line.h" +#include "base/logging.h" +#include "base/profiler/scoped_tracker.h" +#include "base/single_thread_task_runner.h" +#include "base/task_runner.h" +#include "components/domain_reliability/baked_in_configs.h" +#include "components/domain_reliability/google_configs.h" +#include "components/domain_reliability/header.h" +#include "components/domain_reliability/quic_error_mapping.h" +#include "net/base/ip_endpoint.h" +#include "net/base/load_flags.h" +#include "net/base/net_errors.h" +#include "net/http/http_response_headers.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context.h" +#include "net/url_request/url_request_context_getter.h" + +namespace domain_reliability { + +namespace { + +int URLRequestStatusToNetError(const net::URLRequestStatus& status) { + switch (status.status()) { + case net::URLRequestStatus::SUCCESS: + return net::OK; + case net::URLRequestStatus::IO_PENDING: + return net::ERR_IO_PENDING; + case net::URLRequestStatus::CANCELED: + return net::ERR_ABORTED; + case net::URLRequestStatus::FAILED: + return status.error(); + default: + NOTREACHED(); + return net::ERR_UNEXPECTED; + } +} + +// Creates a new beacon based on |beacon_template| but fills in the status, +// chrome_error, and server_ip fields based on the endpoint and result of +// |attempt|. +// +// If there is no matching status for the result, returns false (which +// means the attempt should not result in a beacon being reported). +scoped_ptr<DomainReliabilityBeacon> CreateBeaconFromAttempt( + const DomainReliabilityBeacon& beacon_template, + const net::ConnectionAttempt& attempt) { + std::string status; + if (!GetDomainReliabilityBeaconStatus( + attempt.result, beacon_template.http_response_code, &status)) { + return scoped_ptr<DomainReliabilityBeacon>(); + } + + scoped_ptr<DomainReliabilityBeacon> beacon( + new DomainReliabilityBeacon(beacon_template)); + beacon->status = status; + beacon->chrome_error = attempt.result; + if (!attempt.endpoint.address().empty()) + beacon->server_ip = attempt.endpoint.ToString(); + else + beacon->server_ip = ""; + return beacon; +} + +const char* kDomainReliabilityHeaderName = "NEL"; + +} // namespace + +DomainReliabilityMonitor::DomainReliabilityMonitor( + const std::string& upload_reporter_string, + const scoped_refptr<base::SingleThreadTaskRunner>& pref_thread, + const scoped_refptr<base::SingleThreadTaskRunner>& network_thread) + : time_(new ActualTime()), + upload_reporter_string_(upload_reporter_string), + scheduler_params_( + DomainReliabilityScheduler::Params::GetFromFieldTrialsOrDefaults()), + dispatcher_(time_.get()), + context_manager_(this), + pref_task_runner_(pref_thread), + network_task_runner_(network_thread), + moved_to_network_thread_(false), + discard_uploads_set_(false), + weak_factory_(this) { + DCHECK(OnPrefThread()); + net::NetworkChangeNotifier::AddNetworkChangeObserver(this); +} + +DomainReliabilityMonitor::DomainReliabilityMonitor( + const std::string& upload_reporter_string, + const scoped_refptr<base::SingleThreadTaskRunner>& pref_thread, + const scoped_refptr<base::SingleThreadTaskRunner>& network_thread, + scoped_ptr<MockableTime> time) + : time_(std::move(time)), + upload_reporter_string_(upload_reporter_string), + scheduler_params_( + DomainReliabilityScheduler::Params::GetFromFieldTrialsOrDefaults()), + dispatcher_(time_.get()), + context_manager_(this), + pref_task_runner_(pref_thread), + network_task_runner_(network_thread), + moved_to_network_thread_(false), + discard_uploads_set_(false), + weak_factory_(this) { + DCHECK(OnPrefThread()); + net::NetworkChangeNotifier::AddNetworkChangeObserver(this); +} + +DomainReliabilityMonitor::~DomainReliabilityMonitor() { + if (moved_to_network_thread_) + DCHECK(OnNetworkThread()); + else + DCHECK(OnPrefThread()); + + net::NetworkChangeNotifier::RemoveNetworkChangeObserver(this); +} + +void DomainReliabilityMonitor::MoveToNetworkThread() { + DCHECK(OnPrefThread()); + DCHECK(!moved_to_network_thread_); + + moved_to_network_thread_ = true; +} + +void DomainReliabilityMonitor::InitURLRequestContext( + net::URLRequestContext* url_request_context) { + DCHECK(OnNetworkThread()); + DCHECK(moved_to_network_thread_); + + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter = + new net::TrivialURLRequestContextGetter(url_request_context, + network_task_runner_); + InitURLRequestContext(url_request_context_getter); +} + +void DomainReliabilityMonitor::InitURLRequestContext( + const scoped_refptr<net::URLRequestContextGetter>& + url_request_context_getter) { + DCHECK(OnNetworkThread()); + DCHECK(moved_to_network_thread_); + + // Make sure the URLRequestContext actually lives on what was declared to be + // the network thread. + DCHECK(url_request_context_getter->GetNetworkTaskRunner()-> + RunsTasksOnCurrentThread()); + + uploader_ = DomainReliabilityUploader::Create(time_.get(), + url_request_context_getter); +} + +void DomainReliabilityMonitor::AddBakedInConfigs() { + DCHECK(OnNetworkThread()); + DCHECK(moved_to_network_thread_); + + for (size_t i = 0; kBakedInJsonConfigs[i]; ++i) { + base::StringPiece json(kBakedInJsonConfigs[i]); + scoped_ptr<const DomainReliabilityConfig> config = + DomainReliabilityConfig::FromJSON(json); + if (!config) { + DLOG(WARNING) << "Baked-in Domain Reliability config failed to parse: " + << json; + continue; + } + context_manager_.AddContextForConfig(std::move(config)); + } + + std::vector<DomainReliabilityConfig*> google_configs; + GetAllGoogleConfigs(&google_configs); + for (auto google_config : google_configs) + context_manager_.AddContextForConfig(make_scoped_ptr(google_config)); +} + +void DomainReliabilityMonitor::SetDiscardUploads(bool discard_uploads) { + DCHECK(OnNetworkThread()); + DCHECK(moved_to_network_thread_); + DCHECK(uploader_); + + uploader_->set_discard_uploads(discard_uploads); + discard_uploads_set_ = true; +} + +void DomainReliabilityMonitor::OnBeforeRedirect(net::URLRequest* request) { + DCHECK(OnNetworkThread()); + DCHECK(discard_uploads_set_); + + // Record the redirect itself in addition to the final request. + OnRequestLegComplete(RequestInfo(*request)); +} + +void DomainReliabilityMonitor::OnCompleted(net::URLRequest* request, + bool started) { + DCHECK(OnNetworkThread()); + DCHECK(discard_uploads_set_); + + if (!started) + return; + RequestInfo request_info(*request); + OnRequestLegComplete(request_info); + + if (request_info.response_info.network_accessed) { + // A request was just using the network, so now is a good time to run any + // pending and eligible uploads. + dispatcher_.RunEligibleTasks(); + } +} + +void DomainReliabilityMonitor::OnNetworkChanged( + net::NetworkChangeNotifier::ConnectionType type) { + last_network_change_time_ = time_->NowTicks(); +} + +void DomainReliabilityMonitor::ClearBrowsingData( + DomainReliabilityClearMode mode) { + DCHECK(OnNetworkThread()); + + switch (mode) { + case CLEAR_BEACONS: + context_manager_.ClearBeaconsInAllContexts(); + break; + case CLEAR_CONTEXTS: + context_manager_.RemoveAllContexts(); + break; + case MAX_CLEAR_MODE: + NOTREACHED(); + } +} + +scoped_ptr<base::Value> DomainReliabilityMonitor::GetWebUIData() const { + DCHECK(OnNetworkThread()); + + scoped_ptr<base::DictionaryValue> data_value(new base::DictionaryValue()); + data_value->Set("contexts", context_manager_.GetWebUIData()); + return std::move(data_value); +} + +DomainReliabilityContext* DomainReliabilityMonitor::AddContextForTesting( + scoped_ptr<const DomainReliabilityConfig> config) { + DCHECK(OnNetworkThread()); + + return context_manager_.AddContextForConfig(std::move(config)); +} + +scoped_ptr<DomainReliabilityContext> +DomainReliabilityMonitor::CreateContextForConfig( + scoped_ptr<const DomainReliabilityConfig> config) { + DCHECK(OnNetworkThread()); + DCHECK(config); + DCHECK(config->IsValid()); + + return make_scoped_ptr(new DomainReliabilityContext( + time_.get(), scheduler_params_, upload_reporter_string_, + &last_network_change_time_, &dispatcher_, uploader_.get(), + std::move(config))); +} + +DomainReliabilityMonitor::RequestInfo::RequestInfo() {} + +DomainReliabilityMonitor::RequestInfo::RequestInfo( + const net::URLRequest& request) + : url(request.url()), + status(request.status()), + response_info(request.response_info()), + load_flags(request.load_flags()), + upload_depth( + DomainReliabilityUploader::GetURLRequestUploadDepth(request)) { + request.GetLoadTimingInfo(&load_timing_info); + request.GetConnectionAttempts(&connection_attempts); + request.PopulateNetErrorDetails(&details); + if (!request.GetRemoteEndpoint(&remote_endpoint)) + remote_endpoint = net::IPEndPoint(); +} + +DomainReliabilityMonitor::RequestInfo::RequestInfo(const RequestInfo& other) = + default; + +DomainReliabilityMonitor::RequestInfo::~RequestInfo() {} + +// static +bool DomainReliabilityMonitor::RequestInfo::ShouldReportRequest( + const DomainReliabilityMonitor::RequestInfo& request) { + // Don't report requests that weren't supposed to send cookies. + if (request.load_flags & net::LOAD_DO_NOT_SEND_COOKIES) + return false; + + // Report requests that accessed the network or failed with an error code + // that Domain Reliability is interested in. + if (request.response_info.network_accessed) + return true; + if (URLRequestStatusToNetError(request.status) != net::OK) + return true; + if (request.details.quic_port_migration_detected) + return true; + + return false; +} + +void DomainReliabilityMonitor::OnRequestLegComplete( + const RequestInfo& request) { + // Check these again because unit tests call this directly. + DCHECK(OnNetworkThread()); + DCHECK(discard_uploads_set_); + + MaybeHandleHeader(request); + + if (!RequestInfo::ShouldReportRequest(request)) + return; + + int response_code; + if (request.response_info.headers.get()) + response_code = request.response_info.headers->response_code(); + else + response_code = -1; + + net::ConnectionAttempt url_request_attempt( + request.remote_endpoint, URLRequestStatusToNetError(request.status)); + + DomainReliabilityBeacon beacon_template; + if (request.response_info.connection_info != + net::HttpResponseInfo::CONNECTION_INFO_UNKNOWN) { + beacon_template.protocol = + GetDomainReliabilityProtocol(request.response_info.connection_info, + request.response_info.ssl_info.is_valid()); + } else { + // Use the connection info from the network error details if the response + // is unavailable. + beacon_template.protocol = + GetDomainReliabilityProtocol(request.details.connection_info, + request.response_info.ssl_info.is_valid()); + } + GetDomainReliabilityBeaconQuicError(request.details.quic_connection_error, + &beacon_template.quic_error); + beacon_template.http_response_code = response_code; + beacon_template.start_time = request.load_timing_info.request_start; + beacon_template.elapsed = time_->NowTicks() - beacon_template.start_time; + beacon_template.was_proxied = request.response_info.was_fetched_via_proxy; + beacon_template.url = request.url; + beacon_template.upload_depth = request.upload_depth; + beacon_template.details = request.details; + + // This is not foolproof -- it's possible that we'll see the same error twice + // (e.g. an SSL error during connection on one attempt, and then an error + // that maps to the same code during a read). + // TODO(ttuttle): Find a way for this code to reliably tell whether we + // eventually established a connection or not. + bool url_request_attempt_is_duplicate = false; + for (const auto& attempt : request.connection_attempts) { + if (attempt.result == url_request_attempt.result) + url_request_attempt_is_duplicate = true; + + scoped_ptr<DomainReliabilityBeacon> beacon = + CreateBeaconFromAttempt(beacon_template, attempt); + if (beacon) + context_manager_.RouteBeacon(std::move(beacon)); + } + + if (url_request_attempt_is_duplicate) + return; + + scoped_ptr<DomainReliabilityBeacon> beacon = + CreateBeaconFromAttempt(beacon_template, url_request_attempt); + if (beacon) + context_manager_.RouteBeacon(std::move(beacon)); +} + +void DomainReliabilityMonitor::MaybeHandleHeader( + const RequestInfo& request) { + if (!request.response_info.headers.get()) + return; + + size_t iter = 0; + std::string kHeaderNameString(kDomainReliabilityHeaderName); + + std::string header_value; + if (!request.response_info.headers->EnumerateHeader( + &iter, kHeaderNameString, &header_value)) { + // No header found. + return; + } + + std::string ignored_header_value; + if (request.response_info.headers->EnumerateHeader( + &iter, kHeaderNameString, &ignored_header_value)) { + LOG(WARNING) << "Request to " << request.url << " had (at least) two " + << kHeaderNameString << " headers: \"" << header_value + << "\" and \"" << ignored_header_value << "\"."; + return; + } + + scoped_ptr<DomainReliabilityHeader> parsed = + DomainReliabilityHeader::Parse(header_value); + GURL origin = request.url.GetOrigin(); + switch (parsed->status()) { + case DomainReliabilityHeader::PARSE_SET_CONFIG: + { + base::TimeDelta max_age = parsed->max_age(); + context_manager_.SetConfig(origin, parsed->ReleaseConfig(), max_age); + } + break; + case DomainReliabilityHeader::PARSE_CLEAR_CONFIG: + context_manager_.ClearConfig(origin); + break; + case DomainReliabilityHeader::PARSE_ERROR: + LOG(WARNING) << "Request to " << request.url << " had invalid " + << kHeaderNameString << " header \"" << header_value + << "\"."; + break; + } +} + +base::WeakPtr<DomainReliabilityMonitor> +DomainReliabilityMonitor::MakeWeakPtr() { + return weak_factory_.GetWeakPtr(); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/monitor.h b/chromium/components/domain_reliability/monitor.h new file mode 100644 index 00000000000..fd1f32f4f4d --- /dev/null +++ b/chromium/components/domain_reliability/monitor.h @@ -0,0 +1,194 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_MONITOR_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_MONITOR_H_ + +#include <stddef.h> + +#include <map> + +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/weak_ptr.h" +#include "base/single_thread_task_runner.h" +#include "base/time/time.h" +#include "components/domain_reliability/beacon.h" +#include "components/domain_reliability/clear_mode.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/context.h" +#include "components/domain_reliability/context_manager.h" +#include "components/domain_reliability/dispatcher.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "components/domain_reliability/scheduler.h" +#include "components/domain_reliability/uploader.h" +#include "components/domain_reliability/util.h" +#include "net/base/ip_endpoint.h" +#include "net/base/load_timing_info.h" +#include "net/base/net_error_details.h" +#include "net/base/network_change_notifier.h" +#include "net/http/http_response_info.h" +#include "net/socket/connection_attempts.h" +#include "net/url_request/url_request_status.h" + +namespace base { +class ThreadChecker; +class Value; +} // namespace base + +namespace net { +class URLRequest; +class URLRequestContext; +class URLRequestContextGetter; +} // namespace net + +namespace domain_reliability { + +// The top-level object that measures requests and hands off the measurements +// to the proper |DomainReliabilityContext|. +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityMonitor + : public net::NetworkChangeNotifier::NetworkChangeObserver, + DomainReliabilityContext::Factory { + public: + // Creates a Monitor. |local_state_pref_service| must live on |pref_thread| + // (which should be the current thread); |network_thread| is the thread + // on which requests will actually be monitored and reported. + DomainReliabilityMonitor( + const std::string& upload_reporter_string, + const scoped_refptr<base::SingleThreadTaskRunner>& pref_thread, + const scoped_refptr<base::SingleThreadTaskRunner>& network_thread); + + // Same, but specifies a mock interface for time functions for testing. + DomainReliabilityMonitor( + const std::string& upload_reporter_string, + const scoped_refptr<base::SingleThreadTaskRunner>& pref_thread, + const scoped_refptr<base::SingleThreadTaskRunner>& network_thread, + scoped_ptr<MockableTime> time); + + // Must be called from the pref thread if |MoveToNetworkThread| was not + // called, or from the network thread if it was called. + ~DomainReliabilityMonitor() override; + + // Must be called before |InitURLRequestContext| on the same thread on which + // the Monitor was constructed. Moves (most of) the Monitor to the network + // thread passed in the constructor. + void MoveToNetworkThread(); + + // All public methods below this point must be called on the network thread + // after |MoveToNetworkThread| is called on the pref thread. + + // Initializes the Monitor's URLRequestContextGetter. + // + // Must be called on the network thread, after |MoveToNetworkThread|. + void InitURLRequestContext(net::URLRequestContext* url_request_context); + + // Same, but for unittests where the Getter is readily available. + void InitURLRequestContext( + const scoped_refptr<net::URLRequestContextGetter>& + url_request_context_getter); + + // Populates the monitor with contexts that were configured at compile time. + void AddBakedInConfigs(); + + // Sets whether the uploader will discard uploads. Must be called after + // |InitURLRequestContext|. + void SetDiscardUploads(bool discard_uploads); + + // Should be called when |request| is about to follow a redirect. Will + // examine and possibly log the redirect request. Must be called after + // |SetDiscardUploads|. + void OnBeforeRedirect(net::URLRequest* request); + + // Should be called when |request| is complete. Will examine and possibly + // log the (final) request. |started| should be true if the request was + // actually started before it was terminated. Must be called after + // |SetDiscardUploads|. + void OnCompleted(net::URLRequest* request, bool started); + + // net::NetworkChangeNotifier::NetworkChangeObserver implementation: + void OnNetworkChanged( + net::NetworkChangeNotifier::ConnectionType type) override; + + // Called to remove browsing data. With CLEAR_BEACONS, leaves contexts in + // place but clears beacons (which betray browsing history); with + // CLEAR_CONTEXTS, removes all contexts (which can behave as cookies). + void ClearBrowsingData(DomainReliabilityClearMode mode); + + // Gets a Value containing data that can be formatted into a web page for + // debugging purposes. + scoped_ptr<base::Value> GetWebUIData() const; + + DomainReliabilityContext* AddContextForTesting( + scoped_ptr<const DomainReliabilityConfig> config); + + size_t contexts_size_for_testing() const { + return context_manager_.contexts_size_for_testing(); + } + + // DomainReliabilityContext::Factory implementation: + scoped_ptr<DomainReliabilityContext> CreateContextForConfig( + scoped_ptr<const DomainReliabilityConfig> config) override; + + private: + friend class DomainReliabilityMonitorTest; + // Allow the Service to call |MakeWeakPtr|. + friend class DomainReliabilityServiceImpl; + + typedef std::map<std::string, DomainReliabilityContext*> ContextMap; + + struct DOMAIN_RELIABILITY_EXPORT RequestInfo { + RequestInfo(); + explicit RequestInfo(const net::URLRequest& request); + RequestInfo(const RequestInfo& other); + ~RequestInfo(); + + static bool ShouldReportRequest(const RequestInfo& request); + + GURL url; + net::URLRequestStatus status; + net::HttpResponseInfo response_info; + int load_flags; + net::LoadTimingInfo load_timing_info; + net::ConnectionAttempts connection_attempts; + net::IPEndPoint remote_endpoint; + int upload_depth; + net::NetErrorDetails details; + }; + + void OnRequestLegComplete(const RequestInfo& info); + + void MaybeHandleHeader(const RequestInfo& info); + + bool OnPrefThread() const { + return pref_task_runner_->BelongsToCurrentThread(); + } + bool OnNetworkThread() const { + return network_task_runner_->BelongsToCurrentThread(); + } + + base::WeakPtr<DomainReliabilityMonitor> MakeWeakPtr(); + + scoped_ptr<MockableTime> time_; + base::TimeTicks last_network_change_time_; + const std::string upload_reporter_string_; + DomainReliabilityScheduler::Params scheduler_params_; + DomainReliabilityDispatcher dispatcher_; + scoped_ptr<DomainReliabilityUploader> uploader_; + DomainReliabilityContextManager context_manager_; + + scoped_refptr<base::SingleThreadTaskRunner> pref_task_runner_; + scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_; + + bool moved_to_network_thread_; + bool discard_uploads_set_; + + base::WeakPtrFactory<DomainReliabilityMonitor> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(DomainReliabilityMonitor); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_MONITOR_H_ diff --git a/chromium/components/domain_reliability/monitor_unittest.cc b/chromium/components/domain_reliability/monitor_unittest.cc new file mode 100644 index 00000000000..ac9c7f2e6e9 --- /dev/null +++ b/chromium/components/domain_reliability/monitor_unittest.cc @@ -0,0 +1,385 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/monitor.h" + +#include <stddef.h> +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "base/bind.h" +#include "base/memory/scoped_ptr.h" +#include "base/test/test_simple_task_runner.h" +#include "components/domain_reliability/baked_in_configs.h" +#include "components/domain_reliability/beacon.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/google_configs.h" +#include "components/domain_reliability/test_util.h" +#include "net/base/host_port_pair.h" +#include "net/base/load_flags.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_util.h" +#include "net/url_request/url_request_context_getter.h" +#include "net/url_request/url_request_status.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { + +namespace { + +typedef std::vector<const DomainReliabilityBeacon*> BeaconVector; + +scoped_refptr<net::HttpResponseHeaders> MakeHttpResponseHeaders( + const std::string& headers) { + return scoped_refptr<net::HttpResponseHeaders>( + new net::HttpResponseHeaders(net::HttpUtil::AssembleRawHeaders( + headers.c_str(), headers.length()))); +} + +size_t CountQueuedBeacons(DomainReliabilityContext* context) { + BeaconVector beacons; + context->GetQueuedBeaconsForTesting(&beacons); + return beacons.size(); +} + +} // namespace + +class DomainReliabilityMonitorTest : public testing::Test { + protected: + typedef DomainReliabilityMonitor::RequestInfo RequestInfo; + + DomainReliabilityMonitorTest() + : pref_task_runner_(new base::TestSimpleTaskRunner()), + network_task_runner_(new base::TestSimpleTaskRunner()), + url_request_context_getter_( + new net::TestURLRequestContextGetter(network_task_runner_)), + time_(new MockTime()), + monitor_("test-reporter", + pref_task_runner_, + network_task_runner_, + scoped_ptr<MockableTime>(time_)) { + monitor_.MoveToNetworkThread(); + monitor_.InitURLRequestContext(url_request_context_getter_); + monitor_.SetDiscardUploads(false); + } + + static RequestInfo MakeRequestInfo() { + RequestInfo request; + request.status = net::URLRequestStatus(); + request.response_info.socket_address = + net::HostPortPair::FromString("12.34.56.78:80"); + request.response_info.headers = MakeHttpResponseHeaders( + "HTTP/1.1 200 OK\n\n"); + request.response_info.was_cached = false; + request.response_info.network_accessed = true; + request.response_info.was_fetched_via_proxy = false; + request.load_flags = 0; + request.upload_depth = 0; + return request; + } + + void OnRequestLegComplete(const RequestInfo& info) { + monitor_.OnRequestLegComplete(info); + } + + DomainReliabilityContext* CreateAndAddContext() { + return monitor_.AddContextForTesting(MakeTestConfig()); + } + + DomainReliabilityContext* CreateAndAddContextForOrigin(const GURL& origin, + bool wildcard) { + scoped_ptr<DomainReliabilityConfig> config( + MakeTestConfigWithOrigin(origin)); + config->include_subdomains = wildcard; + return monitor_.AddContextForTesting(std::move(config)); + } + + scoped_refptr<base::TestSimpleTaskRunner> pref_task_runner_; + scoped_refptr<base::TestSimpleTaskRunner> network_task_runner_; + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + MockTime* time_; + DomainReliabilityMonitor monitor_; + DomainReliabilityMonitor::RequestInfo request_; +}; + +namespace { + +TEST_F(DomainReliabilityMonitorTest, Create) { +} + +TEST_F(DomainReliabilityMonitorTest, NoContext) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://no-context/"); + OnRequestLegComplete(request); + + EXPECT_EQ(0u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, NetworkFailure) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + request.response_info.headers = nullptr; + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, GoAwayWithPortMigrationDetected) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.details.quic_port_migration_detected = true; + request.response_info.headers = nullptr; + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, ServerFailure) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.response_info.headers = + MakeHttpResponseHeaders("HTTP/1.1 500 :(\n\n"); + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context)); +} + +// Make sure the monitor does not log requests that did not access the network. +TEST_F(DomainReliabilityMonitorTest, DidNotAccessNetwork) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.response_info.network_accessed = false; + OnRequestLegComplete(request); + + EXPECT_EQ(0u, CountQueuedBeacons(context)); +} + +// Make sure the monitor does not log requests that don't send cookies. +TEST_F(DomainReliabilityMonitorTest, DoNotSendCookies) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.load_flags = net::LOAD_DO_NOT_SEND_COOKIES; + OnRequestLegComplete(request); + + EXPECT_EQ(0u, CountQueuedBeacons(context)); +} + +// Make sure the monitor does not log a network-local error. +TEST_F(DomainReliabilityMonitorTest, LocalError) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.status = + net::URLRequestStatus::FromError(net::ERR_PROXY_CONNECTION_FAILED); + OnRequestLegComplete(request); + + EXPECT_EQ(0u, CountQueuedBeacons(context)); +} + +// Make sure the monitor does not log the proxy's IP if one was used. +TEST_F(DomainReliabilityMonitorTest, WasFetchedViaProxy) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + request.response_info.socket_address = + net::HostPortPair::FromString("127.0.0.1:3128"); + request.response_info.was_fetched_via_proxy = true; + OnRequestLegComplete(request); + + BeaconVector beacons; + context->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + EXPECT_TRUE(beacons[0]->server_ip.empty()); +} + +// Make sure the monitor does not log the cached IP returned after a successful +// cache revalidation request. +TEST_F(DomainReliabilityMonitorTest, + NoCachedIPFromSuccessfulRevalidationRequest) { + scoped_ptr<DomainReliabilityConfig> config = MakeTestConfig(); + config->success_sample_rate = 1.0; + DomainReliabilityContext* context = + monitor_.AddContextForTesting(std::move(config)); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.response_info.was_cached = true; + OnRequestLegComplete(request); + + BeaconVector beacons; + context->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + EXPECT_TRUE(beacons[0]->server_ip.empty()); +} + +// Make sure the monitor does not log the cached IP returned with a failed +// cache revalidation request. +TEST_F(DomainReliabilityMonitorTest, NoCachedIPFromFailedRevalidationRequest) { + DomainReliabilityContext* context = CreateAndAddContext(); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.response_info.was_cached = true; + request.status = + net::URLRequestStatus::FromError(net::ERR_NAME_RESOLUTION_FAILED); + OnRequestLegComplete(request); + + BeaconVector beacons; + context->GetQueuedBeaconsForTesting(&beacons); + EXPECT_EQ(1u, beacons.size()); + EXPECT_TRUE(beacons[0]->server_ip.empty()); +} + +TEST_F(DomainReliabilityMonitorTest, AtLeastOneBakedInConfig) { + DCHECK(kBakedInJsonConfigs[0] != nullptr); +} + +// Will fail when baked-in configs expire, as a reminder to update them. +// (Contact ttuttle@chromium.org if this starts failing.) +TEST_F(DomainReliabilityMonitorTest, AddBakedInConfigs) { + // AddBakedInConfigs DCHECKs that the baked-in configs parse correctly, so + // this unittest will fail if someone tries to add an invalid config to the + // source tree. + monitor_.AddBakedInConfigs(); + + // Count the number of baked-in configs. + size_t num_baked_in_configs = 0; + for (const char* const* p = kBakedInJsonConfigs; *p; ++p) + ++num_baked_in_configs; + + // Also count the Google configs stored in abbreviated form. + std::vector<DomainReliabilityConfig*> google_configs; + GetAllGoogleConfigs(&google_configs); + size_t num_google_configs = google_configs.size(); + STLDeleteElements(&google_configs); + + // The monitor should have contexts for all of the baked-in configs. + EXPECT_EQ(num_baked_in_configs + num_google_configs, + monitor_.contexts_size_for_testing()); +} + +TEST_F(DomainReliabilityMonitorTest, ClearBeacons) { + DomainReliabilityContext* context = CreateAndAddContext(); + + // Initially the monitor should have just the test context, with no beacons. + EXPECT_EQ(1u, monitor_.contexts_size_for_testing()); + EXPECT_EQ(0u, CountQueuedBeacons(context)); + + // Add a beacon. + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://example/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + OnRequestLegComplete(request); + + // Make sure it was added. + EXPECT_EQ(1u, CountQueuedBeacons(context)); + + monitor_.ClearBrowsingData(CLEAR_BEACONS); + + // Make sure the beacon was cleared, but not the contexts. + EXPECT_EQ(1u, monitor_.contexts_size_for_testing()); + EXPECT_EQ(0u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, ClearContexts) { + CreateAndAddContext(); + + // Initially the monitor should have just the test context. + EXPECT_EQ(1u, monitor_.contexts_size_for_testing()); + + monitor_.ClearBrowsingData(CLEAR_CONTEXTS); + + // Clearing contexts should leave the monitor with none. + EXPECT_EQ(0u, monitor_.contexts_size_for_testing()); +} + +TEST_F(DomainReliabilityMonitorTest, WildcardMatchesSelf) { + DomainReliabilityContext* context = + CreateAndAddContextForOrigin(GURL("https://wildcard/"), true); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://wildcard/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, WildcardMatchesSubdomain) { + DomainReliabilityContext* context = + CreateAndAddContextForOrigin(GURL("https://wildcard/"), true); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://test.wildcard/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, WildcardDoesntMatchSubsubdomain) { + DomainReliabilityContext* context = + CreateAndAddContextForOrigin(GURL("https://wildcard/"), true); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://test.test.wildcard/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + OnRequestLegComplete(request); + + EXPECT_EQ(0u, CountQueuedBeacons(context)); +} + +TEST_F(DomainReliabilityMonitorTest, WildcardPrefersSelfToParentWildcard) { + DomainReliabilityContext* context1 = + CreateAndAddContextForOrigin(GURL("https://test.wildcard/"), false); + DomainReliabilityContext* context2 = + CreateAndAddContextForOrigin(GURL("https://wildcard/"), true); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://test.wildcard/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context1)); + EXPECT_EQ(0u, CountQueuedBeacons(context2)); +} + +TEST_F(DomainReliabilityMonitorTest, + WildcardPrefersSelfWildcardToParentWildcard) { + DomainReliabilityContext* context1 = + CreateAndAddContextForOrigin(GURL("https://test.wildcard/"), true); + DomainReliabilityContext* context2 = + CreateAndAddContextForOrigin(GURL("https://wildcard/"), true); + + RequestInfo request = MakeRequestInfo(); + request.url = GURL("http://test.wildcard/"); + request.status = net::URLRequestStatus::FromError(net::ERR_CONNECTION_RESET); + OnRequestLegComplete(request); + + EXPECT_EQ(1u, CountQueuedBeacons(context1)); + EXPECT_EQ(0u, CountQueuedBeacons(context2)); +} + +} // namespace + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/quic_error_mapping.cc b/chromium/components/domain_reliability/quic_error_mapping.cc new file mode 100644 index 00000000000..7331d503ae0 --- /dev/null +++ b/chromium/components/domain_reliability/quic_error_mapping.cc @@ -0,0 +1,255 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/quic_error_mapping.h" + +namespace domain_reliability { + +namespace { + +const struct QuicErrorMapping { + net::QuicErrorCode quic_error; + const char* beacon_quic_error; +} kQuicErrorMap[] = { + // Connection has reached an invalid state. + { net::QUIC_INTERNAL_ERROR, "quic.internal_error" }, + // There were data frames after the a fin or reset. + { net::QUIC_STREAM_DATA_AFTER_TERMINATION, + "quic.stream_data.after_termination" }, + // Control frame is malformed. + { net::QUIC_INVALID_PACKET_HEADER, "quic.invalid.packet_header" }, + // Frame data is malformed. + { net::QUIC_INVALID_FRAME_DATA, "quic.invalid_frame_data" }, + // The packet contained no payload. + { net::QUIC_MISSING_PAYLOAD, "quic.missing.payload" }, + // FEC data is malformed. + { net::QUIC_INVALID_FEC_DATA, "quic.invalid.fec_data" }, + // STREAM frame data is malformed. + { net::QUIC_INVALID_STREAM_DATA, "quic.invalid.stream_data" }, + // STREAM frame data is not encrypted. + { net::QUIC_UNENCRYPTED_STREAM_DATA, "quic.unencrypted.stream_data" }, + // Attempt to send unencrypted STREAM frame. + { net::QUIC_ATTEMPT_TO_SEND_UNENCRYPTED_STREAM_DATA, + "quic.attempt.to.unencrypted.stream.data" }, + // FEC frame data is not encrypted. + { net::QUIC_UNENCRYPTED_FEC_DATA, "quic.unencrypted.fec.data" }, + // RST_STREAM frame data is malformed. + { net::QUIC_INVALID_RST_STREAM_DATA, "quic.invalid.rst_stream_data" }, + // CONNECTION_CLOSE frame data is malformed. + { net::QUIC_INVALID_CONNECTION_CLOSE_DATA, + "quic.invalid.connection_close_data" }, + // GOAWAY frame data is malformed. + { net::QUIC_INVALID_GOAWAY_DATA, "quic.invalid.goaway_data" }, + // WINDOW_UPDATE frame data is malformed. + { net::QUIC_INVALID_WINDOW_UPDATE_DATA, "quic.invalid.window_update_data" }, + // BLOCKED frame data is malformed. + { net::QUIC_INVALID_BLOCKED_DATA, "quic.invalid.blocked_data" }, + // STOP_WAITING frame data is malformed. + { net::QUIC_INVALID_STOP_WAITING_DATA, "quic.invalid.stop_waiting_data" }, + // PATH_CLOSE frame data is malformed. + { net::QUIC_INVALID_PATH_CLOSE_DATA, "quic.invalid_path_close_data" }, + // ACK frame data is malformed. + { net::QUIC_INVALID_ACK_DATA, "quic.invalid.ack_data" }, + + // Version negotiation packet is malformed. + { net::QUIC_INVALID_VERSION_NEGOTIATION_PACKET, + "quic_invalid_version_negotiation_packet" }, + // Public RST packet is malformed. + { net::QUIC_INVALID_PUBLIC_RST_PACKET, "quic.invalid.public_rst_packet" }, + + // There was an error decrypting. + { net::QUIC_DECRYPTION_FAILURE, "quic.decryption.failure" }, + // There was an error encrypting. + { net::QUIC_ENCRYPTION_FAILURE, "quic.encryption.failure" }, + // The packet exceeded kMaxPacketSize. + { net::QUIC_PACKET_TOO_LARGE, "quic.packet.too_large" }, + // The peer is going away. May be a client or server. + { net::QUIC_PEER_GOING_AWAY, "quic.peer_going_away" }, + // A stream ID was invalid. + { net::QUIC_INVALID_STREAM_ID, "quic.invalid_stream_id" }, + // A priority was invalid. + { net::QUIC_INVALID_PRIORITY, "quic.invalid_priority" }, + // Too many streams already open. + { net::QUIC_TOO_MANY_OPEN_STREAMS, "quic.too_many_open_streams" }, + // The peer created too many available streams. + { net::QUIC_TOO_MANY_AVAILABLE_STREAMS, "quic.too_many_available_streams" }, + // Received public reset for this connection. + { net::QUIC_PUBLIC_RESET, "quic.public_reset" }, + // Invalid protocol version. + { net::QUIC_INVALID_VERSION, "quic.invalid_version" }, + + // The Header ID for a stream was too far from the previous. + { net::QUIC_INVALID_HEADER_ID, "quic.invalid_header_id" }, + // Negotiable parameter received during handshake had invalid value. + { net::QUIC_INVALID_NEGOTIATED_VALUE, "quic.invalid_negotiated_value" }, + // There was an error decompressing data. + { net::QUIC_DECOMPRESSION_FAILURE, "quic.decompression_failure" }, + // We hit our prenegotiated (or default) timeout + { net::QUIC_NETWORK_IDLE_TIMEOUT, "quic.connection.idle_time_out" }, + // We hit our overall connection timeout + { net::QUIC_HANDSHAKE_TIMEOUT, + "quic.connection.handshake_timed_out" }, + // There was an error encountered migrating addresses. + { net::QUIC_ERROR_MIGRATING_ADDRESS, "quic.error_migrating_address" }, + // There was an error encountered migrating port only. + { net::QUIC_ERROR_MIGRATING_PORT, "quic.error_migrating_port" }, + // There was an error while writing to the socket. + { net::QUIC_PACKET_WRITE_ERROR, "quic.packet.write_error" }, + // There was an error while reading from the socket. + { net::QUIC_PACKET_READ_ERROR, "quic.packet.read_error" }, + // We received a STREAM_FRAME with no data and no fin flag set. + { net::QUIC_EMPTY_STREAM_FRAME_NO_FIN, "quic.empty_stream_frame_no_fin" }, + // We received invalid data on the headers stream. + { net::QUIC_INVALID_HEADERS_STREAM_DATA, "quic.invalid_headers_stream_data" }, + // The peer received too much data, violating flow control. + { net::QUIC_FLOW_CONTROL_RECEIVED_TOO_MUCH_DATA, + "quic.flow_control.received_too_much_data" }, + // The peer sent too much data, violating flow control. + { net::QUIC_FLOW_CONTROL_SENT_TOO_MUCH_DATA, + "quic.flow_control.sent_too_much_data" }, + // The peer received an invalid flow control window. + { net::QUIC_FLOW_CONTROL_INVALID_WINDOW, "quic.flow_control.invalid_window" }, + // The connection has been IP pooled into an existing connection. + { net::QUIC_CONNECTION_IP_POOLED, "quic.connection.ip_pooled" }, + // The connection has too many outstanding sent packets. + { net::QUIC_TOO_MANY_OUTSTANDING_SENT_PACKETS, + "quic.too_many_outstanding_sent_packets" }, + // The connection has too many outstanding received packets. + { net::QUIC_TOO_MANY_OUTSTANDING_RECEIVED_PACKETS, + "quic.too_many_outstanding_received_packets" }, + // The quic connection job to load server config is cancelled. + { net::QUIC_CONNECTION_CANCELLED, "quic.connection.cancelled" }, + // Disabled QUIC because of high packet loss rate. + { net::QUIC_BAD_PACKET_LOSS_RATE, "quic.bad_packet_loss_rate" }, + // Disabled QUIC because of too many PUBLIC_RESETs post handshake. + { net::QUIC_PUBLIC_RESETS_POST_HANDSHAKE, + "quic.public_resets_post_handshake" }, + // Disabled QUIC because of too many timeouts with streams open. + { net::QUIC_TIMEOUTS_WITH_OPEN_STREAMS, "quic.timeouts_with_open_streams" }, + // Closed because we failed to serialize a packet. + { net::QUIC_FAILED_TO_SERIALIZE_PACKET, "quic.failed_to_serialize_packet" }, + // QUIC timed out after too many RTOs. + { net::QUIC_TOO_MANY_RTOS, "quic.too_many_rtos" }, + // Crypto errors. + + // Hanshake failed. + { net::QUIC_HANDSHAKE_FAILED, "quic.handshake_failed" }, + // Handshake message contained out of order tags. + { net::QUIC_CRYPTO_TAGS_OUT_OF_ORDER, "quic.crypto.tags_out_of_order" }, + // Handshake message contained too many entries. + { net::QUIC_CRYPTO_TOO_MANY_ENTRIES, "quic.crypto.too_many_entries" }, + // Handshake message contained an invalid value length. + { net::QUIC_CRYPTO_INVALID_VALUE_LENGTH, "quic.crypto.invalid_value_length" }, + // A crypto message was received after the handshake was complete. + { net::QUIC_CRYPTO_MESSAGE_AFTER_HANDSHAKE_COMPLETE, + "quic.crypto_message_after_handshake_complete" }, + // A crypto message was received with an illegal message tag. + { net::QUIC_INVALID_CRYPTO_MESSAGE_TYPE, "quic.invalid_crypto_message_type" }, + // A crypto message was received with an illegal parameter. + { net::QUIC_INVALID_CRYPTO_MESSAGE_PARAMETER, + "quic.invalid_crypto_message_parameter" }, + // An invalid channel id signature was supplied. + { net::QUIC_INVALID_CHANNEL_ID_SIGNATURE, + "quic.invalid_channel_id_signature" }, + // A crypto message was received with a mandatory parameter missing. + { net::QUIC_CRYPTO_MESSAGE_PARAMETER_NOT_FOUND, + "quic.crypto_message.parameter_not_found" }, + // A crypto message was received with a parameter that has no overlap + // with the local parameter. + { net::QUIC_CRYPTO_MESSAGE_PARAMETER_NO_OVERLAP, + "quic.crypto_message.parameter_no_overlap" }, + // A crypto message was received that contained a parameter with too few + // values. + { net::QUIC_CRYPTO_MESSAGE_INDEX_NOT_FOUND, + "quic_crypto_message_index_not_found" }, + // An internal error occured in crypto processing. + { net::QUIC_CRYPTO_INTERNAL_ERROR, "quic.crypto.internal_error" }, + // A crypto handshake message specified an unsupported version. + { net::QUIC_CRYPTO_VERSION_NOT_SUPPORTED, + "quic.crypto.version_not_supported" }, + // A crypto handshake message resulted in a stateless reject. + { net::QUIC_CRYPTO_HANDSHAKE_STATELESS_REJECT, + "quic.crypto.handshake_stateless_reject" }, + // There was no intersection between the crypto primitives supported by the + // peer and ourselves. + { net::QUIC_CRYPTO_NO_SUPPORT, "quic.crypto.no_support" }, + // The server rejected our client hello messages too many times. + { net::QUIC_CRYPTO_TOO_MANY_REJECTS, "quic.crypto.too_many_rejects" }, + // The client rejected the server's certificate chain or signature. + { net::QUIC_PROOF_INVALID, "quic.proof_invalid" }, + // A crypto message was received with a duplicate tag. + { net::QUIC_CRYPTO_DUPLICATE_TAG, "quic.crypto.duplicate_tag" }, + // A crypto message was received with the wrong encryption level (i.e. it + // should have been encrypted but was not.) + { net::QUIC_CRYPTO_ENCRYPTION_LEVEL_INCORRECT, + "quic.crypto.encryption_level_incorrect" }, + // The server config for a server has expired. + { net::QUIC_CRYPTO_SERVER_CONFIG_EXPIRED, + "quic.crypto.server_config_expired" }, + // We failed to setup the symmetric keys for a connection. + { net::QUIC_CRYPTO_SYMMETRIC_KEY_SETUP_FAILED, + "quic.crypto.symmetric_key_setup_failed" }, + // A handshake message arrived, but we are still validating the + // previous handshake message. + { net::QUIC_CRYPTO_MESSAGE_WHILE_VALIDATING_CLIENT_HELLO, + "quic.crypto_message_while_validating_client_hello" }, + // A server config update arrived before the handshake is complete. + { net::QUIC_CRYPTO_UPDATE_BEFORE_HANDSHAKE_COMPLETE, + "quic.crypto.update_before_handshake_complete" }, + // This connection involved a version negotiation which appears to have been + // tampered with. + { net::QUIC_VERSION_NEGOTIATION_MISMATCH, + "quic.version_negotiation_mismatch" }, + + // Multipath is not enabled, but a packet with multipath flag on is received. + { net::QUIC_BAD_MULTIPATH_FLAG, "quic.bad_multipath_flag" }, + + // Network change and connection migration errors. + + // IP address changed causing connection close. + { net::QUIC_IP_ADDRESS_CHANGED, "quic.ip_address_changed" }, + // Network changed, but connection had no migratable streams. + { net::QUIC_CONNECTION_MIGRATION_NO_MIGRATABLE_STREAMS, + "quic.connection_migration_no_migratable_streams" }, + // Connection changed networks too many times. + { net::QUIC_CONNECTION_MIGRATION_TOO_MANY_CHANGES, + "quic.connection_migration_too_many_changes" }, + // Connection migration was attempted, but there was no new network to + // migrate to. + { net::QUIC_CONNECTION_MIGRATION_NO_NEW_NETWORK, + "quic.connection_migration_no_new_network" }, + // Network changed, but connection had one or more non-migratable streams. + { net::QUIC_CONNECTION_MIGRATION_NON_MIGRATABLE_STREAM, + "quic.connection_migration_non_migratable_stream" }, + // Stream frame overlaps with buffered data. + { net::QUIC_OVERLAPPING_STREAM_DATA, + "quic.overlapping_stream_data" }, + + // No error. Used as bound while iterating. + { net::QUIC_LAST_ERROR, "quic.last_error"} +}; + +static_assert(arraysize(kQuicErrorMap) == net::kActiveQuicErrorCount, + "quic_error_map is not in sync with quic protocol!"); + +} // namespace + +// static +bool GetDomainReliabilityBeaconQuicError(net::QuicErrorCode quic_error, + std::string* beacon_quic_error_out) { + if (quic_error != net::QUIC_NO_ERROR) { + // Convert a QUIC error. + // TODO(ttuttle): Consider sorting and using binary search? + for (size_t i = 0; i < arraysize(kQuicErrorMap); i++) { + if (kQuicErrorMap[i].quic_error == quic_error) { + *beacon_quic_error_out = kQuicErrorMap[i].beacon_quic_error; + return true; + } + } + } + beacon_quic_error_out->clear(); + return false; +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/quic_error_mapping.h b/chromium/components/domain_reliability/quic_error_mapping.h new file mode 100644 index 00000000000..7d936eabc33 --- /dev/null +++ b/chromium/components/domain_reliability/quic_error_mapping.h @@ -0,0 +1,26 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_QUIC_ERROR_MAPPING_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_QUIC_ERROR_MAPPING_H_ + +#include <string> + +#include "net/quic/quic_protocol.h" + +// N.B. This file and the .cc are separate from util.h/.cc so that they can be +// independently updated by folks working on QUIC when new errors are added. + +namespace domain_reliability { + +// Attempts to convert a QUIC error into the quic_error string +// that should be recorded in a beacon. Returns true and parse the QUIC error +// code in |beacon_quic_error_out| if it could. +// Returns false and clear |beacon_quic_error_out| otherwise. +bool GetDomainReliabilityBeaconQuicError(net::QuicErrorCode quic_error, + std::string* beacon_quic_error_out); + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_QUIC_ERROR_MAPPING_H_ diff --git a/chromium/components/domain_reliability/scheduler.cc b/chromium/components/domain_reliability/scheduler.cc new file mode 100644 index 00000000000..79ed191a4fb --- /dev/null +++ b/chromium/components/domain_reliability/scheduler.cc @@ -0,0 +1,270 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/scheduler.h" + +#include <stdint.h> +#include <algorithm> +#include <utility> + +#include "base/metrics/field_trial.h" +#include "base/strings/string_number_conversions.h" +#include "base/values.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/util.h" +#include "net/base/backoff_entry.h" + +namespace { + +const unsigned kInvalidCollectorIndex = static_cast<unsigned>(-1); + +const unsigned kDefaultMinimumUploadDelaySec = 60; +const unsigned kDefaultMaximumUploadDelaySec = 300; +const unsigned kDefaultUploadRetryIntervalSec = 60; + +const char* kMinimumUploadDelayFieldTrialName = "DomRel-MinimumUploadDelay"; +const char* kMaximumUploadDelayFieldTrialName = "DomRel-MaximumUploadDelay"; +const char* kUploadRetryIntervalFieldTrialName = "DomRel-UploadRetryInterval"; + +// Fixed elements of backoff policy +const double kMultiplyFactor = 2.0; +const double kJitterFactor = 0.1; +const int64_t kMaximumBackoffMs = 60 * 1000 * 1000; + +unsigned GetUnsignedFieldTrialValueOrDefault(std::string field_trial_name, + unsigned default_value) { + if (!base::FieldTrialList::TrialExists(field_trial_name)) + return default_value; + + std::string group_name = base::FieldTrialList::FindFullName(field_trial_name); + unsigned value; + if (!base::StringToUint(group_name, &value)) { + LOG(ERROR) << "Expected unsigned integer for field trial " + << field_trial_name << " group name, but got \"" << group_name + << "\"."; + return default_value; + } + + return value; +} + +} // namespace + +namespace domain_reliability { + +// static +DomainReliabilityScheduler::Params +DomainReliabilityScheduler::Params::GetFromFieldTrialsOrDefaults() { + DomainReliabilityScheduler::Params params; + + params.minimum_upload_delay = + base::TimeDelta::FromSeconds(GetUnsignedFieldTrialValueOrDefault( + kMinimumUploadDelayFieldTrialName, kDefaultMinimumUploadDelaySec)); + params.maximum_upload_delay = + base::TimeDelta::FromSeconds(GetUnsignedFieldTrialValueOrDefault( + kMaximumUploadDelayFieldTrialName, kDefaultMaximumUploadDelaySec)); + params.upload_retry_interval = + base::TimeDelta::FromSeconds(GetUnsignedFieldTrialValueOrDefault( + kUploadRetryIntervalFieldTrialName, kDefaultUploadRetryIntervalSec)); + + return params; +} + +DomainReliabilityScheduler::DomainReliabilityScheduler( + MockableTime* time, + size_t num_collectors, + const Params& params, + const ScheduleUploadCallback& callback) + : time_(time), + params_(params), + callback_(callback), + upload_pending_(false), + upload_scheduled_(false), + upload_running_(false), + collector_index_(kInvalidCollectorIndex), + last_upload_finished_(false) { + backoff_policy_.num_errors_to_ignore = 0; + backoff_policy_.initial_delay_ms = + params.upload_retry_interval.InMilliseconds(); + backoff_policy_.multiply_factor = kMultiplyFactor; + backoff_policy_.jitter_factor = kJitterFactor; + backoff_policy_.maximum_backoff_ms = kMaximumBackoffMs; + backoff_policy_.entry_lifetime_ms = 0; + backoff_policy_.always_use_initial_delay = false; + + for (size_t i = 0; i < num_collectors; ++i) { + collectors_.push_back( + new net::BackoffEntry(&backoff_policy_, time_)); + } +} + +DomainReliabilityScheduler::~DomainReliabilityScheduler() {} + +void DomainReliabilityScheduler::OnBeaconAdded() { + if (!upload_pending_) + first_beacon_time_ = time_->NowTicks(); + upload_pending_ = true; + MaybeScheduleUpload(); +} + +size_t DomainReliabilityScheduler::OnUploadStart() { + DCHECK(upload_scheduled_); + DCHECK_EQ(kInvalidCollectorIndex, collector_index_); + upload_pending_ = false; + upload_scheduled_ = false; + upload_running_ = true; + + base::TimeTicks now = time_->NowTicks(); + base::TimeTicks min_upload_time; + GetNextUploadTimeAndCollector(now, &min_upload_time, &collector_index_); + DCHECK(min_upload_time <= now); + + VLOG(1) << "Starting upload to collector " << collector_index_ << "."; + + last_upload_start_time_ = now; + last_upload_collector_index_ = collector_index_; + + return collector_index_; +} + +void DomainReliabilityScheduler::OnUploadComplete( + const DomainReliabilityUploader::UploadResult& result) { + DCHECK(upload_running_); + DCHECK_NE(kInvalidCollectorIndex, collector_index_); + upload_running_ = false; + + VLOG(1) << "Upload to collector " << collector_index_ + << (result.is_success() ? " succeeded." : " failed."); + + net::BackoffEntry* backoff = collectors_[collector_index_]; + collector_index_ = kInvalidCollectorIndex; + + backoff->InformOfRequest(result.is_success()); + if (result.is_retry_after()) + backoff->SetCustomReleaseTime(time_->NowTicks() + result.retry_after); + last_collector_retry_delay_ = backoff->GetTimeUntilRelease(); + + if (!result.is_success()) { + // Restore upload_pending_ and first_beacon_time_ to pre-upload state, + // since upload failed. + upload_pending_ = true; + first_beacon_time_ = old_first_beacon_time_; + } + + last_upload_end_time_ = time_->NowTicks(); + last_upload_success_ = result.is_success(); + last_upload_finished_ = true; + + MaybeScheduleUpload(); +} + +scoped_ptr<base::Value> DomainReliabilityScheduler::GetWebUIData() const { + base::TimeTicks now = time_->NowTicks(); + + scoped_ptr<base::DictionaryValue> data(new base::DictionaryValue()); + + data->SetBoolean("upload_pending", upload_pending_); + data->SetBoolean("upload_scheduled", upload_scheduled_); + data->SetBoolean("upload_running", upload_running_); + + data->SetInteger("scheduled_min", (scheduled_min_time_ - now).InSeconds()); + data->SetInteger("scheduled_max", (scheduled_max_time_ - now).InSeconds()); + + data->SetInteger("collector_index", static_cast<int>(collector_index_)); + + if (last_upload_finished_) { + scoped_ptr<base::DictionaryValue> last(new base::DictionaryValue()); + last->SetInteger("start_time", (now - last_upload_start_time_).InSeconds()); + last->SetInteger("end_time", (now - last_upload_end_time_).InSeconds()); + last->SetInteger("collector_index", + static_cast<int>(last_upload_collector_index_)); + last->SetBoolean("success", last_upload_success_); + data->Set("last_upload", std::move(last)); + } + + scoped_ptr<base::ListValue> collectors_value(new base::ListValue()); + for (const auto& collector : collectors_) { + scoped_ptr<base::DictionaryValue> value(new base::DictionaryValue()); + value->SetInteger("failures", collector->failure_count()); + value->SetInteger("next_upload", + (collector->GetReleaseTime() - now).InSeconds()); + // Using release instead of Pass because Pass can't implicitly upcast. + collectors_value->Append(value.release()); + } + data->Set("collectors", std::move(collectors_value)); + + return std::move(data); +} + +void DomainReliabilityScheduler::MakeDeterministicForTesting() { + backoff_policy_.jitter_factor = 0.0; +} + +void DomainReliabilityScheduler::MaybeScheduleUpload() { + if (!upload_pending_ || upload_scheduled_ || upload_running_) + return; + + upload_scheduled_ = true; + old_first_beacon_time_ = first_beacon_time_; + + base::TimeTicks now = time_->NowTicks(); + + base::TimeTicks min_by_deadline, max_by_deadline; + min_by_deadline = first_beacon_time_ + params_.minimum_upload_delay; + max_by_deadline = first_beacon_time_ + params_.maximum_upload_delay; + DCHECK(min_by_deadline <= max_by_deadline); + + base::TimeTicks min_by_backoff; + size_t collector_index; + GetNextUploadTimeAndCollector(now, &min_by_backoff, &collector_index); + + scheduled_min_time_ = std::max(min_by_deadline, min_by_backoff); + scheduled_max_time_ = std::max(max_by_deadline, min_by_backoff); + + base::TimeDelta min_delay = scheduled_min_time_ - now; + base::TimeDelta max_delay = scheduled_max_time_ - now; + + VLOG(1) << "Scheduling upload for between " << min_delay.InSeconds() + << " and " << max_delay.InSeconds() << " seconds from now."; + + callback_.Run(min_delay, max_delay); +} + +// TODO(ttuttle): Add min and max interval to config, use that instead. + +// TODO(ttuttle): Cap min and max intervals received from config. + +void DomainReliabilityScheduler::GetNextUploadTimeAndCollector( + base::TimeTicks now, + base::TimeTicks* upload_time_out, + size_t* collector_index_out) { + DCHECK(upload_time_out); + DCHECK(collector_index_out); + + base::TimeTicks min_time; + size_t min_index = kInvalidCollectorIndex; + + for (size_t i = 0; i < collectors_.size(); ++i) { + net::BackoffEntry* backoff = collectors_[i]; + // If a collector is usable, use the first one in the list. + if (!backoff->ShouldRejectRequest()) { + min_time = now; + min_index = i; + break; + } + + // If not, keep track of which will be usable soonest: + base::TimeTicks time = backoff->GetReleaseTime(); + if (min_index == kInvalidCollectorIndex || time < min_time) { + min_time = time; + min_index = i; + } + } + + DCHECK_NE(kInvalidCollectorIndex, min_index); + *upload_time_out = min_time; + *collector_index_out = min_index; +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/scheduler.h b/chromium/components/domain_reliability/scheduler.h new file mode 100644 index 00000000000..3810e3a7bb4 --- /dev/null +++ b/chromium/components/domain_reliability/scheduler.h @@ -0,0 +1,149 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_SCHEDULER_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_SCHEDULER_H_ + +#include <stddef.h> + +#include <vector> + +#include "base/callback.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" +#include "base/time/time.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "components/domain_reliability/uploader.h" +#include "net/base/backoff_entry.h" + +namespace base { +class Value; +} // namespace base + +namespace domain_reliability { + +struct DomainReliabilityConfig; +class MockableTime; + +// Determines when an upload should be scheduled. A domain's config will +// specify minimum and maximum upload delays; the minimum upload delay ensures +// that Chrome will not send too many upload requests to a site by waiting at +// least that long after the first beacon, while the maximum upload delay makes +// sure the server receives the reports while they are still fresh. +// +// When everything is working fine, the scheduler will return precisely that +// interval. If all uploaders have failed, then the beginning or ending points +// of the interval may be pushed later to accomodate the retry with exponential +// backoff. +// +// See dispatcher.h for an explanation of what happens with the scheduled +// interval. +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityScheduler { + public: + typedef base::Callback<void(base::TimeDelta, base::TimeDelta)> + ScheduleUploadCallback; + + struct Params { + public: + base::TimeDelta minimum_upload_delay; + base::TimeDelta maximum_upload_delay; + base::TimeDelta upload_retry_interval; + + static Params GetFromFieldTrialsOrDefaults(); + }; + + DomainReliabilityScheduler(MockableTime* time, + size_t num_collectors, + const Params& params, + const ScheduleUploadCallback& callback); + ~DomainReliabilityScheduler(); + + // If there is no upload pending, schedules an upload based on the provided + // parameters (some time between the minimum and maximum delay from now). + // May call the ScheduleUploadCallback. + void OnBeaconAdded(); + + // Returns which collector to use for an upload that is about to start. Must + // be called exactly once during or after the ScheduleUploadCallback but + // before OnUploadComplete is called. (Also records the upload start time for + // future retries, if the upload ends up failing.) + size_t OnUploadStart(); + + // Updates the scheduler state based on the result of an upload. Must be + // called exactly once after |OnUploadStart|. |result| should be the result + // passed to the upload callback by the Uploader. + void OnUploadComplete(const DomainReliabilityUploader::UploadResult& result); + + scoped_ptr<base::Value> GetWebUIData() const; + + // Disables jitter in BackoffEntries to make scheduling deterministic for + // unit tests. + void MakeDeterministicForTesting(); + + // Gets the time of the first beacon that has not yet been successfully + // uploaded. + base::TimeTicks first_beacon_time() const { return first_beacon_time_; } + + // Gets the time until the next upload attempt on the last collector used. + // This will be 0 if the upload was a success; it does not take into account + // minimum_upload_delay and maximum_upload_delay. + base::TimeDelta last_collector_retry_delay() const { + return last_collector_retry_delay_; + } + + private: + void MaybeScheduleUpload(); + + void GetNextUploadTimeAndCollector(base::TimeTicks now, + base::TimeTicks* upload_time_out, + size_t* collector_index_out); + + MockableTime* time_; + Params params_; + ScheduleUploadCallback callback_; + net::BackoffEntry::Policy backoff_policy_; + ScopedVector<net::BackoffEntry> collectors_; + + // Whether there are beacons that have not yet been uploaded. Set when a + // beacon arrives or an upload fails, and cleared when an upload starts. + bool upload_pending_; + + // Whether the scheduler has called the ScheduleUploadCallback to schedule + // the next upload. Set when an upload is scheduled and cleared when the + // upload starts. + bool upload_scheduled_; + + // Whether the last scheduled upload is in progress. Set when the upload + // starts and cleared when the upload completes (successfully or not). + bool upload_running_; + + // Index of the collector selected for the next upload. (Set in + // |OnUploadStart| and cleared in |OnUploadComplete|.) + size_t collector_index_; + + // Time of the first beacon that was not included in the last successful + // upload. + base::TimeTicks first_beacon_time_; + + // first_beacon_time_ saved during uploads. Restored if upload fails. + base::TimeTicks old_first_beacon_time_; + + // Time until the next upload attempt on the last collector used. (Saved for + // histograms in Context.) + base::TimeDelta last_collector_retry_delay_; + + // Extra bits to return in GetWebUIData. + base::TimeTicks scheduled_min_time_; + base::TimeTicks scheduled_max_time_; + // Whether the other last_upload_* fields are populated. + bool last_upload_finished_; + base::TimeTicks last_upload_start_time_; + base::TimeTicks last_upload_end_time_; + size_t last_upload_collector_index_; + bool last_upload_success_; +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_SCHEDULER_H_ diff --git a/chromium/components/domain_reliability/scheduler_unittest.cc b/chromium/components/domain_reliability/scheduler_unittest.cc new file mode 100644 index 00000000000..c86adf45d6a --- /dev/null +++ b/chromium/components/domain_reliability/scheduler_unittest.cc @@ -0,0 +1,285 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/scheduler.h" + +#include <stddef.h> + +#include "base/bind.h" +#include "base/time/time.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/test_util.h" +#include "components/domain_reliability/uploader.h" +#include "components/domain_reliability/util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +using base::TimeDelta; +using base::TimeTicks; + +class DomainReliabilitySchedulerTest : public testing::Test { + public: + DomainReliabilitySchedulerTest() + : num_collectors_(0), + params_(MakeTestSchedulerParams()), + callback_called_(false) {} + + void CreateScheduler(int num_collectors) { + DCHECK_LT(0, num_collectors); + DCHECK(!scheduler_); + + num_collectors_ = num_collectors; + scheduler_.reset(new DomainReliabilityScheduler( + &time_, + num_collectors_, + params_, + base::Bind(&DomainReliabilitySchedulerTest::ScheduleUploadCallback, + base::Unretained(this)))); + scheduler_->MakeDeterministicForTesting(); + } + + void NotifySuccessfulUpload() { + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::SUCCESS; + scheduler_->OnUploadComplete(result); + } + + void NotifyFailedUpload() { + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::FAILURE; + scheduler_->OnUploadComplete(result); + } + + void NotifyRetryAfterUpload(base::TimeDelta retry_after) { + DomainReliabilityUploader::UploadResult result; + result.status = DomainReliabilityUploader::UploadResult::RETRY_AFTER; + result.retry_after = retry_after; + scheduler_->OnUploadComplete(result); + } + + ::testing::AssertionResult CheckNoPendingUpload() { + DCHECK(scheduler_); + + if (!callback_called_) + return ::testing::AssertionSuccess(); + + return ::testing::AssertionFailure() + << "expected no upload, got upload between " + << callback_min_.InSeconds() << " and " + << callback_max_.InSeconds() << " seconds from now"; + } + + ::testing::AssertionResult CheckPendingUpload(TimeDelta expected_min, + TimeDelta expected_max) { + DCHECK(scheduler_); + DCHECK_LE(expected_min.InMicroseconds(), expected_max.InMicroseconds()); + + if (callback_called_ && expected_min == callback_min_ + && expected_max == callback_max_) { + callback_called_ = false; + return ::testing::AssertionSuccess(); + } + + if (callback_called_) { + return ::testing::AssertionFailure() + << "expected upload between " << expected_min.InSeconds() + << " and " << expected_max.InSeconds() << " seconds from now, " + << "got upload between " << callback_min_.InSeconds() + << " and " << callback_max_.InSeconds() << " seconds from now"; + } else { + return ::testing::AssertionFailure() + << "expected upload between " << expected_min.InSeconds() + << " and " << expected_max.InSeconds() << " seconds from now, " + << "got no upload"; + } + } + + ::testing::AssertionResult CheckStartingUpload(size_t expected_collector) { + DCHECK(scheduler_); + DCHECK_GT(num_collectors_, expected_collector); + + size_t collector = scheduler_->OnUploadStart(); + if (collector == expected_collector) + return ::testing::AssertionSuccess(); + + return ::testing::AssertionFailure() + << "expected upload to collector " << expected_collector + << ", got upload to collector " << collector; + } + + TimeDelta min_delay() const { return params_.minimum_upload_delay; } + TimeDelta max_delay() const { return params_.maximum_upload_delay; } + TimeDelta retry_interval() const { return params_.upload_retry_interval; } + TimeDelta zero_delta() const { return base::TimeDelta::FromMicroseconds(0); } + + protected: + void ScheduleUploadCallback(TimeDelta min, TimeDelta max) { + callback_called_ = true; + callback_min_ = min; + callback_max_ = max; + } + + MockTime time_; + size_t num_collectors_; + DomainReliabilityScheduler::Params params_; + scoped_ptr<DomainReliabilityScheduler> scheduler_; + + bool callback_called_; + TimeDelta callback_min_; + TimeDelta callback_max_; +}; + +TEST_F(DomainReliabilitySchedulerTest, Create) { + CreateScheduler(1); +} + +TEST_F(DomainReliabilitySchedulerTest, UploadNotPendingWithoutBeacon) { + CreateScheduler(1); + + ASSERT_TRUE(CheckNoPendingUpload()); +} + +TEST_F(DomainReliabilitySchedulerTest, SuccessfulUploads) { + CreateScheduler(1); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + ASSERT_TRUE(CheckStartingUpload(0)); + NotifySuccessfulUpload(); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + ASSERT_TRUE(CheckStartingUpload(0)); + NotifySuccessfulUpload(); +} + +TEST_F(DomainReliabilitySchedulerTest, RetryAfter) { + CreateScheduler(1); + + base::TimeDelta retry_after_interval = base::TimeDelta::FromMinutes(30); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + ASSERT_TRUE(CheckStartingUpload(0)); + NotifyRetryAfterUpload(retry_after_interval); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(retry_after_interval, retry_after_interval)); + time_.Advance(retry_after_interval); + ASSERT_TRUE(CheckStartingUpload(0)); + NotifySuccessfulUpload(); +} + +TEST_F(DomainReliabilitySchedulerTest, Failover) { + CreateScheduler(2); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + ASSERT_TRUE(CheckStartingUpload(0)); + NotifyFailedUpload(); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(zero_delta(), max_delay() - min_delay())); + // Don't need to advance; should retry immediately. + ASSERT_TRUE(CheckStartingUpload(1)); + NotifySuccessfulUpload(); +} + +TEST_F(DomainReliabilitySchedulerTest, FailedAllCollectors) { + CreateScheduler(2); + + // T = 0 + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + + // T = min_delay + ASSERT_TRUE(CheckStartingUpload(0)); + NotifyFailedUpload(); + + ASSERT_TRUE(CheckPendingUpload(zero_delta(), max_delay() - min_delay())); + // Don't need to advance; should retry immediately. + ASSERT_TRUE(CheckStartingUpload(1)); + NotifyFailedUpload(); + + ASSERT_TRUE(CheckPendingUpload(retry_interval(), max_delay() - min_delay())); + time_.Advance(retry_interval()); + + // T = min_delay + retry_interval + ASSERT_TRUE(CheckStartingUpload(0)); + NotifyFailedUpload(); + + ASSERT_TRUE(CheckPendingUpload( + zero_delta(), + max_delay() - min_delay() - retry_interval())); + ASSERT_TRUE(CheckStartingUpload(1)); + NotifyFailedUpload(); +} + +// Make sure that the scheduler uses the first available collector at upload +// time, even if it wasn't available at scheduling time. +TEST_F(DomainReliabilitySchedulerTest, DetermineCollectorAtUpload) { + CreateScheduler(2); + + // T = 0 + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + + // T = min_delay + ASSERT_TRUE(CheckStartingUpload(0)); + NotifyFailedUpload(); + + ASSERT_TRUE(CheckPendingUpload(zero_delta(), max_delay() - min_delay())); + time_.Advance(retry_interval()); + + // T = min_delay + retry_interval; collector 0 should be active again. + ASSERT_TRUE(CheckStartingUpload(0)); + NotifySuccessfulUpload(); +} + +TEST_F(DomainReliabilitySchedulerTest, BeaconWhilePending) { + CreateScheduler(1); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + + // Second beacon should not call callback again. + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckNoPendingUpload()); + time_.Advance(min_delay()); + + // No pending upload after beacon. + ASSERT_TRUE(CheckStartingUpload(0)); + NotifySuccessfulUpload(); + ASSERT_TRUE(CheckNoPendingUpload()); +} + +TEST_F(DomainReliabilitySchedulerTest, BeaconWhileUploading) { + CreateScheduler(1); + + scheduler_->OnBeaconAdded(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + time_.Advance(min_delay()); + + // If a beacon arrives during the upload, a new upload should be pending. + ASSERT_TRUE(CheckStartingUpload(0)); + scheduler_->OnBeaconAdded(); + NotifySuccessfulUpload(); + ASSERT_TRUE(CheckPendingUpload(min_delay(), max_delay())); + + time_.Advance(min_delay()); + ASSERT_TRUE(CheckStartingUpload(0)); + NotifySuccessfulUpload(); + ASSERT_TRUE(CheckNoPendingUpload()); +} + +} // namespace +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/service.cc b/chromium/components/domain_reliability/service.cc new file mode 100644 index 00000000000..1d44453ff63 --- /dev/null +++ b/chromium/components/domain_reliability/service.cc @@ -0,0 +1,97 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/service.h" + +#include "base/bind.h" +#include "base/callback.h" +#include "base/location.h" +#include "base/single_thread_task_runner.h" +#include "base/task_runner_util.h" +#include "base/thread_task_runner_handle.h" +#include "components/domain_reliability/monitor.h" +#include "net/url_request/url_request_context_getter.h" + +namespace domain_reliability { + +namespace { + +scoped_ptr<base::Value> GetWebUIDataOnNetworkTaskRunner( + base::WeakPtr<DomainReliabilityMonitor> monitor) { + if (!monitor) { + base::DictionaryValue* dict = new base::DictionaryValue(); + dict->SetString("error", "no_monitor"); + return scoped_ptr<base::Value>(dict); + } + + return monitor->GetWebUIData(); +} + +} // namespace + +class DomainReliabilityServiceImpl : public DomainReliabilityService { + public: + explicit DomainReliabilityServiceImpl( + const std::string& upload_reporter_string) + : upload_reporter_string_(upload_reporter_string) {} + + ~DomainReliabilityServiceImpl() override {} + + // DomainReliabilityService implementation: + + scoped_ptr<DomainReliabilityMonitor> CreateMonitor( + scoped_refptr<base::SingleThreadTaskRunner> network_task_runner) + override { + DCHECK(!network_task_runner_.get()); + + scoped_ptr<DomainReliabilityMonitor> monitor(new DomainReliabilityMonitor( + upload_reporter_string_, base::ThreadTaskRunnerHandle::Get(), + network_task_runner)); + + monitor_ = monitor->MakeWeakPtr(); + network_task_runner_ = network_task_runner; + + return monitor; + } + + void ClearBrowsingData(DomainReliabilityClearMode clear_mode, + const base::Closure& callback) override { + DCHECK(network_task_runner_.get()); + + network_task_runner_->PostTaskAndReply( + FROM_HERE, + base::Bind(&DomainReliabilityMonitor::ClearBrowsingData, + monitor_, + clear_mode), + callback); + } + + void GetWebUIData(const base::Callback<void(scoped_ptr<base::Value>)>& + callback) const override { + DCHECK(network_task_runner_.get()); + + PostTaskAndReplyWithResult( + network_task_runner_.get(), + FROM_HERE, + base::Bind(&GetWebUIDataOnNetworkTaskRunner, monitor_), + callback); + } + + private: + std::string upload_reporter_string_; + base::WeakPtr<DomainReliabilityMonitor> monitor_; + scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_; +}; + +// static +DomainReliabilityService* DomainReliabilityService::Create( + const std::string& upload_reporter_string) { + return new DomainReliabilityServiceImpl(upload_reporter_string); +} + +DomainReliabilityService::~DomainReliabilityService() {} + +DomainReliabilityService::DomainReliabilityService() {} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/service.h b/chromium/components/domain_reliability/service.h new file mode 100644 index 00000000000..9625ae633ab --- /dev/null +++ b/chromium/components/domain_reliability/service.h @@ -0,0 +1,73 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_SERVICE_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_SERVICE_H_ + +#include <string> + +#include "base/callback_forward.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/single_thread_task_runner.h" +#include "components/domain_reliability/clear_mode.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "components/keyed_service/core/keyed_service.h" + +class PrefService; + +namespace base { +class Value; +} // namespace base + +namespace net { +class URLRequestContextGetter; +} // namespace net + +namespace domain_reliability { + +class DomainReliabilityMonitor; + +// DomainReliabilityService is a KeyedService that manages a Monitor that lives +// on another thread (as provided by the URLRequestContextGetter's task runner) +// and proxies (selected) method calls to it. Destruction of the Monitor (on +// that thread) is the responsibility of the caller. +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityService + : public KeyedService { + public: + // Creates a DomainReliabilityService that will contain a Monitor with the + // given upload reporter string. + static DomainReliabilityService* Create( + const std::string& upload_reporter_string); + + ~DomainReliabilityService() override; + + // Initializes the Service: given the task runner on which Monitor methods + // should be called, creates the Monitor and returns it. Can be called at + // most once, and must be called before any of the below methods can be + // called. The caller is responsible for destroying the Monitor on the given + // task runner when it is no longer needed. + virtual scoped_ptr<DomainReliabilityMonitor> CreateMonitor( + scoped_refptr<base::SingleThreadTaskRunner> network_task_runner) = 0; + + // Clears browsing data on the associated Monitor. |Init()| must have been + // called first. + virtual void ClearBrowsingData(DomainReliabilityClearMode clear_mode, + const base::Closure& callback) = 0; + + virtual void GetWebUIData( + const base::Callback<void(scoped_ptr<base::Value>)>& callback) + const = 0; + + protected: + DomainReliabilityService(); + + private: + DISALLOW_COPY_AND_ASSIGN(DomainReliabilityService); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_SERVICE_H_ diff --git a/chromium/components/domain_reliability/test_util.cc b/chromium/components/domain_reliability/test_util.cc new file mode 100644 index 00000000000..663ec5fa34e --- /dev/null +++ b/chromium/components/domain_reliability/test_util.cc @@ -0,0 +1,181 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/test_util.h" + +#include "base/bind.h" +#include "base/callback.h" +#include "components/domain_reliability/scheduler.h" +#include "net/url_request/url_request_status.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { + +namespace { + +class MockTimer : public MockableTime::Timer { + public: + MockTimer(MockTime* time) + : time_(time), + running_(false), + callback_sequence_number_(0), + weak_factory_(this) { + DCHECK(time); + } + + ~MockTimer() override {} + + // MockableTime::Timer implementation: + void Start(const tracked_objects::Location& posted_from, + base::TimeDelta delay, + const base::Closure& user_task) override { + DCHECK(!user_task.is_null()); + + if (running_) + ++callback_sequence_number_; + running_ = true; + user_task_ = user_task; + time_->AddTask(delay, + base::Bind(&MockTimer::OnDelayPassed, + weak_factory_.GetWeakPtr(), + callback_sequence_number_)); + } + + void Stop() override { + if (running_) { + ++callback_sequence_number_; + running_ = false; + } + } + + bool IsRunning() override { return running_; } + + private: + void OnDelayPassed(int expected_callback_sequence_number) { + if (callback_sequence_number_ != expected_callback_sequence_number) + return; + + DCHECK(running_); + running_ = false; + + // Grab user task in case it re-entrantly starts the timer again. + base::Closure task_to_run = user_task_; + user_task_.Reset(); + task_to_run.Run(); + } + + MockTime* time_; + bool running_; + int callback_sequence_number_; + base::Closure user_task_; + base::WeakPtrFactory<MockTimer> weak_factory_; +}; + +} // namespace + +TestCallback::TestCallback() + : callback_(base::Bind(&TestCallback::OnCalled, + base::Unretained(this))), + called_(false) {} + +TestCallback::~TestCallback() {} + +void TestCallback::OnCalled() { + EXPECT_FALSE(called_); + called_ = true; +} + +MockUploader::MockUploader(const UploadRequestCallback& callback) + : callback_(callback), + discard_uploads_(true) {} + +MockUploader::~MockUploader() {} + +bool MockUploader::discard_uploads() const { return discard_uploads_; } + +void MockUploader::UploadReport(const std::string& report_json, + int max_upload_depth, + const GURL& upload_url, + const UploadCallback& callback) { + callback_.Run(report_json, max_upload_depth, upload_url, callback); +} + +void MockUploader::set_discard_uploads(bool discard_uploads) { + discard_uploads_ = discard_uploads; +} + +MockTime::MockTime() + : now_(base::Time::Now()), + now_ticks_(base::TimeTicks::Now()), + epoch_ticks_(now_ticks_), + task_sequence_number_(0) { + VLOG(1) << "Creating mock time: T=" << elapsed_sec() << "s"; +} + +MockTime::~MockTime() {} + +base::Time MockTime::Now() { return now_; } +base::TimeTicks MockTime::NowTicks() { return now_ticks_; } + +scoped_ptr<MockableTime::Timer> MockTime::CreateTimer() { + return scoped_ptr<MockableTime::Timer>(new MockTimer(this)); +} + +void MockTime::Advance(base::TimeDelta delta) { + base::TimeTicks target_ticks = now_ticks_ + delta; + + while (!tasks_.empty() && tasks_.begin()->first.time <= target_ticks) { + TaskKey key = tasks_.begin()->first; + base::Closure task = tasks_.begin()->second; + tasks_.erase(tasks_.begin()); + + DCHECK(now_ticks_ <= key.time); + DCHECK(key.time <= target_ticks); + AdvanceToInternal(key.time); + VLOG(1) << "Advancing mock time: task at T=" << elapsed_sec() << "s"; + + task.Run(); + } + + DCHECK(now_ticks_ <= target_ticks); + AdvanceToInternal(target_ticks); + VLOG(1) << "Advanced mock time: T=" << elapsed_sec() << "s"; +} + +void MockTime::AddTask(base::TimeDelta delay, const base::Closure& task) { + tasks_[TaskKey(now_ticks_ + delay, task_sequence_number_++)] = task; +} + +void MockTime::AdvanceToInternal(base::TimeTicks target_ticks) { + base::TimeDelta delta = target_ticks - now_ticks_; + now_ += delta; + now_ticks_ += delta; +} + +DomainReliabilityScheduler::Params MakeTestSchedulerParams() { + DomainReliabilityScheduler::Params params; + params.minimum_upload_delay = base::TimeDelta::FromMinutes(1); + params.maximum_upload_delay = base::TimeDelta::FromMinutes(5); + params.upload_retry_interval = base::TimeDelta::FromSeconds(15); + return params; +} + +scoped_ptr<DomainReliabilityConfig> MakeTestConfig() { + return MakeTestConfigWithOrigin(GURL("https://example/")); +} + +scoped_ptr<DomainReliabilityConfig> MakeTestConfigWithOrigin( + const GURL& origin) { + DomainReliabilityConfig* config = new DomainReliabilityConfig(); + config->origin = origin; + config->collectors.push_back(new GURL("https://exampleuploader/upload")); + config->failure_sample_rate = 1.0; + config->success_sample_rate = 0.0; + + DCHECK(config->IsValid()); + + return scoped_ptr<DomainReliabilityConfig>(config); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/test_util.h b/chromium/components/domain_reliability/test_util.h new file mode 100644 index 00000000000..65e1072d480 --- /dev/null +++ b/chromium/components/domain_reliability/test_util.h @@ -0,0 +1,132 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_TEST_UTIL_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_TEST_UTIL_H_ + +#include "base/callback_forward.h" +#include "base/memory/scoped_ptr.h" +#include "components/domain_reliability/config.h" +#include "components/domain_reliability/scheduler.h" +#include "components/domain_reliability/uploader.h" +#include "components/domain_reliability/util.h" +#include "net/base/host_port_pair.h" +#include "url/gurl.h" + +namespace net { +class URLRequestStatus; +} // namespace net + +namespace domain_reliability { + +// A simple test callback that remembers whether it's been called. +class TestCallback { + public: + TestCallback(); + ~TestCallback(); + + // Returns a callback that can be called only once. + const base::Closure& callback() const { return callback_; } + // Returns whether the callback returned by |callback()| has been called. + bool called() const { return called_; } + + private: + void OnCalled(); + + base::Closure callback_; + bool called_; +}; + +class MockUploader : public DomainReliabilityUploader { + public: + typedef base::Callback<void(const std::string& report_json, + int max_upload_depth, + const GURL& upload_url, + const UploadCallback& upload_callback)> + UploadRequestCallback; + + MockUploader(const UploadRequestCallback& callback); + + ~MockUploader() override; + + virtual bool discard_uploads() const; + + // DomainReliabilityUploader implementation: + void UploadReport(const std::string& report_json, + int max_upload_depth, + const GURL& upload_url, + const UploadCallback& callback) override; + + void set_discard_uploads(bool discard_uploads) override; + + private: + UploadRequestCallback callback_; + bool discard_uploads_; +}; + +class MockTime : public MockableTime { + public: + MockTime(); + + // N.B.: Tasks (and therefore Timers) scheduled to run in the future will + // never be run if MockTime is destroyed before the mock time is advanced + // to their scheduled time. + ~MockTime() override; + + // MockableTime implementation: + base::Time Now() override; + base::TimeTicks NowTicks() override; + scoped_ptr<MockableTime::Timer> CreateTimer() override; + + // Pretends that |delta| has passed, and runs tasks that would've happened + // during that interval (with |Now()| returning proper values while they + // execute!) + void Advance(base::TimeDelta delta); + + // Queues |task| to be run after |delay|. (Lighter-weight than mocking an + // entire message pump.) + void AddTask(base::TimeDelta delay, const base::Closure& task); + + private: + // Key used to store tasks in the task map. Includes the time the task should + // run and a sequence number to disambiguate tasks with the same time. + struct TaskKey { + TaskKey(base::TimeTicks time, int sequence_number) + : time(time), + sequence_number(sequence_number) {} + + base::TimeTicks time; + int sequence_number; + }; + + // Comparator for TaskKey; sorts by time, then by sequence number. + struct TaskKeyCompare { + bool operator() (const TaskKey& lhs, const TaskKey& rhs) const { + return lhs.time < rhs.time || + (lhs.time == rhs.time && + lhs.sequence_number < rhs.sequence_number); + } + }; + + typedef std::map<TaskKey, base::Closure, TaskKeyCompare> TaskMap; + + void AdvanceToInternal(base::TimeTicks target_ticks); + + int elapsed_sec() { return (now_ticks_ - epoch_ticks_).InSeconds(); } + + base::Time now_; + base::TimeTicks now_ticks_; + base::TimeTicks epoch_ticks_; + int task_sequence_number_; + TaskMap tasks_; +}; + +scoped_ptr<DomainReliabilityConfig> MakeTestConfig(); +scoped_ptr<DomainReliabilityConfig> MakeTestConfigWithOrigin( + const GURL& origin); +DomainReliabilityScheduler::Params MakeTestSchedulerParams(); + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_TEST_UTIL_H_ diff --git a/chromium/components/domain_reliability/uploader.cc b/chromium/components/domain_reliability/uploader.cc new file mode 100644 index 00000000000..8ed2e0e3714 --- /dev/null +++ b/chromium/components/domain_reliability/uploader.cc @@ -0,0 +1,185 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/uploader.h" + +#include "base/bind.h" +#include "base/callback.h" +#include "base/metrics/sparse_histogram.h" +#include "base/stl_util.h" +#include "base/supports_user_data.h" +#include "components/data_use_measurement/core/data_use_user_data.h" +#include "components/domain_reliability/util.h" +#include "net/base/load_flags.h" +#include "net/base/net_errors.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_util.h" +#include "net/url_request/url_fetcher.h" +#include "net/url_request/url_fetcher_delegate.h" +#include "net/url_request/url_request_context_getter.h" + +namespace domain_reliability { + +namespace { + +const char kJsonMimeType[] = "application/json; charset=utf-8"; + +class UploadUserData : public base::SupportsUserData::Data { + public: + static net::URLFetcher::CreateDataCallback CreateCreateDataCallback( + int depth) { + return base::Bind(&UploadUserData::CreateUploadUserData, depth); + } + + static const void* const kUserDataKey; + + int depth() const { return depth_; } + + private: + UploadUserData(int depth) : depth_(depth) {} + + static base::SupportsUserData::Data* CreateUploadUserData(int depth) { + return new UploadUserData(depth); + } + + int depth_; +}; + +const void* const UploadUserData::kUserDataKey = + &UploadUserData::kUserDataKey; + +class DomainReliabilityUploaderImpl + : public DomainReliabilityUploader, net::URLFetcherDelegate { + public: + DomainReliabilityUploaderImpl( + MockableTime* time, + const scoped_refptr< + net::URLRequestContextGetter>& url_request_context_getter) + : time_(time), + url_request_context_getter_(url_request_context_getter), + discard_uploads_(true) {} + + ~DomainReliabilityUploaderImpl() override { + // Delete any in-flight URLFetchers. + STLDeleteContainerPairFirstPointers( + upload_callbacks_.begin(), upload_callbacks_.end()); + } + + // DomainReliabilityUploader implementation: + void UploadReport( + const std::string& report_json, + int max_upload_depth, + const GURL& upload_url, + const DomainReliabilityUploader::UploadCallback& callback) override { + VLOG(1) << "Uploading report to " << upload_url; + VLOG(2) << "Report JSON: " << report_json; + + if (discard_uploads_) { + VLOG(1) << "Discarding report instead of uploading."; + UploadResult result; + result.status = UploadResult::SUCCESS; + callback.Run(result); + return; + } + + net::URLFetcher* fetcher = + net::URLFetcher::Create(0, upload_url, net::URLFetcher::POST, this) + .release(); + data_use_measurement::DataUseUserData::AttachToFetcher( + fetcher, data_use_measurement::DataUseUserData::DOMAIN_RELIABILITY); + fetcher->SetRequestContext(url_request_context_getter_.get()); + fetcher->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); + fetcher->SetUploadData(kJsonMimeType, report_json); + fetcher->SetAutomaticallyRetryOn5xx(false); + fetcher->SetURLRequestUserData( + UploadUserData::kUserDataKey, + UploadUserData::CreateCreateDataCallback(max_upload_depth + 1)); + fetcher->Start(); + + upload_callbacks_[fetcher] = callback; + } + + void set_discard_uploads(bool discard_uploads) override { + discard_uploads_ = discard_uploads; + VLOG(1) << "Setting discard_uploads to " << discard_uploads; + } + + // net::URLFetcherDelegate implementation: + void OnURLFetchComplete(const net::URLFetcher* fetcher) override { + DCHECK(fetcher); + + UploadCallbackMap::iterator callback_it = upload_callbacks_.find(fetcher); + DCHECK(callback_it != upload_callbacks_.end()); + + int net_error = GetNetErrorFromURLRequestStatus(fetcher->GetStatus()); + int http_response_code = fetcher->GetResponseCode(); + base::TimeDelta retry_after; + { + std::string retry_after_string; + if (fetcher->GetResponseHeaders() && + fetcher->GetResponseHeaders()->EnumerateHeader(nullptr, + "Retry-After", + &retry_after_string)) { + net::HttpUtil::ParseRetryAfterHeader(retry_after_string, + time_->Now(), + &retry_after); + } + } + + VLOG(1) << "Upload finished with net error " << net_error + << ", response code " << http_response_code + << ", retry after " << retry_after; + + UMA_HISTOGRAM_SPARSE_SLOWLY("DomainReliability.UploadResponseCode", + http_response_code); + UMA_HISTOGRAM_SPARSE_SLOWLY("DomainReliability.UploadNetError", + -net_error); + + UploadResult result; + GetUploadResultFromResponseDetails(net_error, + http_response_code, + retry_after, + &result); + callback_it->second.Run(result); + + delete callback_it->first; + upload_callbacks_.erase(callback_it); + } + + private: + using DomainReliabilityUploader::UploadCallback; + typedef std::map<const net::URLFetcher*, UploadCallback> UploadCallbackMap; + + MockableTime* time_; + scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; + UploadCallbackMap upload_callbacks_; + bool discard_uploads_; +}; + +} // namespace + +DomainReliabilityUploader::DomainReliabilityUploader() {} +DomainReliabilityUploader::~DomainReliabilityUploader() {} + +// static +std::unique_ptr<DomainReliabilityUploader> DomainReliabilityUploader::Create( + MockableTime* time, + const scoped_refptr<net::URLRequestContextGetter>& + url_request_context_getter) { + return std::unique_ptr<DomainReliabilityUploader>( + new DomainReliabilityUploaderImpl(time, url_request_context_getter)); +} + +// static +int DomainReliabilityUploader::GetURLRequestUploadDepth( + const net::URLRequest& request) { + UploadUserData* data = static_cast<UploadUserData*>( + request.GetUserData(UploadUserData::kUserDataKey)); + if (!data) + return 0; + return data->depth(); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/uploader.h b/chromium/components/domain_reliability/uploader.h new file mode 100644 index 00000000000..700a4f8c294 --- /dev/null +++ b/chromium/components/domain_reliability/uploader.h @@ -0,0 +1,73 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_UPLOADER_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_UPLOADER_H_ + +#include <map> +#include <memory> + +#include "base/callback_forward.h" +#include "base/memory/ref_counted.h" +#include "base/time/time.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "url/gurl.h" + +namespace net { +class URLFetcher; +class URLRequest; +class URLRequestContextGetter; +} // namespace net + +namespace domain_reliability { + +class MockableTime; + +// Uploads Domain Reliability reports to collectors. +class DOMAIN_RELIABILITY_EXPORT DomainReliabilityUploader { + public: + struct UploadResult { + enum UploadStatus { + FAILURE, + SUCCESS, + RETRY_AFTER, + }; + + bool is_success() const { return status == SUCCESS; } + bool is_failure() const { return status == FAILURE; } + bool is_retry_after() const { return status == RETRY_AFTER; } + + UploadStatus status; + base::TimeDelta retry_after; + }; + + typedef base::Callback<void(const UploadResult& result)> UploadCallback; + + DomainReliabilityUploader(); + + virtual ~DomainReliabilityUploader(); + + // Creates an uploader that uses the given |url_request_context_getter| to + // get a URLRequestContext to use for uploads. (See test_util.h for a mock + // version.) + static std::unique_ptr<DomainReliabilityUploader> Create( + MockableTime* time, + const scoped_refptr<net::URLRequestContextGetter>& + url_request_context_getter); + + // Uploads |report_json| to |upload_url| and calls |callback| when the upload + // has either completed or failed. + virtual void UploadReport(const std::string& report_json, + int max_beacon_depth, + const GURL& upload_url, + const UploadCallback& callback) = 0; + + virtual void set_discard_uploads(bool discard_uploads) = 0; + + static int GetURLRequestUploadDepth(const net::URLRequest& request); +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_UPLOADER_H_ diff --git a/chromium/components/domain_reliability/uploader_unittest.cc b/chromium/components/domain_reliability/uploader_unittest.cc new file mode 100644 index 00000000000..7e697d0ffd8 --- /dev/null +++ b/chromium/components/domain_reliability/uploader_unittest.cc @@ -0,0 +1,279 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/uploader.h" + +#include <stddef.h> + +#include "base/bind.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/test/test_simple_task_runner.h" +#include "components/domain_reliability/test_util.h" +#include "net/base/load_flags.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_response_info.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_filter.h" +#include "net/url_request/url_request_interceptor.h" +#include "net/url_request/url_request_job.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +const char kUploadURL[] = "https://example/upload"; + +struct MockUploadResult { + int net_error; + int response_code; + scoped_refptr<net::HttpResponseHeaders> response_headers; +}; + +class UploadMockURLRequestJob : public net::URLRequestJob { + public: + UploadMockURLRequestJob(net::URLRequest* request, + net::NetworkDelegate* network_delegate, + MockUploadResult result) + : net::URLRequestJob(request, network_delegate), + upload_stream_(nullptr), + result_(result) { + int load_flags = request->load_flags(); + EXPECT_TRUE(load_flags & net::LOAD_DO_NOT_SEND_COOKIES); + EXPECT_TRUE(load_flags & net::LOAD_DO_NOT_SAVE_COOKIES); + } + + protected: + void Start() override { + int rv = upload_stream_->Init( + base::Bind(&UploadMockURLRequestJob::OnStreamInitialized, + base::Unretained(this))); + if (rv == net::ERR_IO_PENDING) + return; + OnStreamInitialized(rv); + } + + void SetUpload(net::UploadDataStream* upload_stream) override { + upload_stream_ = upload_stream; + } + + private: + ~UploadMockURLRequestJob() override {} + + void OnStreamInitialized(int rv) { + EXPECT_EQ(net::OK, rv); + + size_t upload_size = upload_stream_->size(); + upload_buffer_ = new net::IOBufferWithSize(upload_size); + rv = upload_stream_->Read( + upload_buffer_.get(), + upload_size, + base::Bind(&UploadMockURLRequestJob::OnStreamRead, + base::Unretained(this))); + if (rv == net::ERR_IO_PENDING) + return; + OnStreamRead(rv); + } + + void OnStreamRead(int rv) { + EXPECT_EQ(upload_buffer_->size(), rv); + + upload_data_ = std::string(upload_buffer_->data(), upload_buffer_->size()); + upload_buffer_ = nullptr; + + if (result_.net_error == net::OK) + NotifyHeadersComplete(); + else + NotifyStartError(net::URLRequestStatus::FromError(result_.net_error)); + } + + int GetResponseCode() const override { + return result_.response_code; + } + + void GetResponseInfo(net::HttpResponseInfo* info) override { + info->headers = result_.response_headers; + } + + net::UploadDataStream* upload_stream_; + scoped_refptr<net::IOBufferWithSize> upload_buffer_; + std::string upload_data_; + MockUploadResult result_; +}; + +class UploadInterceptor : public net::URLRequestInterceptor { + public: + UploadInterceptor() : last_upload_depth_(-1) {} + + ~UploadInterceptor() override { + EXPECT_TRUE(results_.empty()); + } + + net::URLRequestJob* MaybeInterceptRequest( + net::URLRequest* request, + net::NetworkDelegate* delegate) const override { + EXPECT_FALSE(results_.empty()); + MockUploadResult result = results_.front(); + results_.pop_front(); + + last_upload_depth_ = + DomainReliabilityUploader::GetURLRequestUploadDepth(*request); + + return new UploadMockURLRequestJob(request, delegate, result); + } + + void ExpectRequestAndReturnError(int net_error) { + MockUploadResult result; + result.net_error = net_error; + result.response_code = -1; + results_.push_back(result); + } + + void ExpectRequestAndReturnResponseCode(int response_code) { + MockUploadResult result; + result.net_error = net::OK; + result.response_code = response_code; + results_.push_back(result); + } + + void ExpectRequestAndReturnResponseCodeAndHeaders( + int response_code, + const char* headers) { + MockUploadResult result; + result.net_error = net::OK; + result.response_code = response_code; + result.response_headers = new net::HttpResponseHeaders( + net::HttpUtil::AssembleRawHeaders(headers, strlen(headers))); + results_.push_back(result); + } + + int last_upload_depth() const { return last_upload_depth_; } + + private: + mutable std::list<MockUploadResult> results_; + mutable int last_upload_depth_; +}; + +class TestUploadCallback { + public: + TestUploadCallback() : called_count_(0u) {} + + DomainReliabilityUploader::UploadCallback callback() { + return base::Bind(&TestUploadCallback::OnCalled, base::Unretained(this)); + } + + unsigned called_count() const { return called_count_; } + DomainReliabilityUploader::UploadResult last_result() const { + return last_result_; + } + + private: + void OnCalled(const DomainReliabilityUploader::UploadResult& result) { + called_count_++; + last_result_ = result; + } + + unsigned called_count_; + DomainReliabilityUploader::UploadResult last_result_; +}; + +class DomainReliabilityUploaderTest : public testing::Test { + protected: + DomainReliabilityUploaderTest() + : url_request_context_getter_(new net::TestURLRequestContextGetter( + message_loop_.task_runner())), + interceptor_(new UploadInterceptor()), + uploader_(DomainReliabilityUploader::Create( + &time_, url_request_context_getter_)) { + net::URLRequestFilter::GetInstance()->AddUrlInterceptor( + GURL(kUploadURL), make_scoped_ptr(interceptor_)); + uploader_->set_discard_uploads(false); + } + + ~DomainReliabilityUploaderTest() override { + net::URLRequestFilter::GetInstance()->ClearHandlers(); + } + + DomainReliabilityUploader* uploader() const { return uploader_.get(); } + UploadInterceptor* interceptor() const { return interceptor_; } + + private: + base::MessageLoopForIO message_loop_; + scoped_refptr<net::TestURLRequestContextGetter> url_request_context_getter_; + UploadInterceptor* interceptor_; + MockTime time_; + scoped_ptr<DomainReliabilityUploader> uploader_; +}; + +TEST_F(DomainReliabilityUploaderTest, Null) { +} + +TEST_F(DomainReliabilityUploaderTest, SuccessfulUpload) { + interceptor()->ExpectRequestAndReturnResponseCode(200); + + TestUploadCallback c; + uploader()->UploadReport("{}", 0, GURL(kUploadURL), c.callback()); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(1u, c.called_count()); + EXPECT_TRUE(c.last_result().is_success()); +} + +TEST_F(DomainReliabilityUploaderTest, NetworkErrorUpload) { + interceptor()->ExpectRequestAndReturnError(net::ERR_CONNECTION_REFUSED); + + TestUploadCallback c; + uploader()->UploadReport("{}", 0, GURL(kUploadURL), c.callback()); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(1u, c.called_count()); + EXPECT_TRUE(c.last_result().is_failure()); +} + +TEST_F(DomainReliabilityUploaderTest, ServerErrorUpload) { + interceptor()->ExpectRequestAndReturnResponseCode(500); + + TestUploadCallback c; + uploader()->UploadReport("{}", 0, GURL(kUploadURL), c.callback()); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(1u, c.called_count()); + EXPECT_TRUE(c.last_result().is_failure()); +} + +TEST_F(DomainReliabilityUploaderTest, RetryAfterUpload) { + interceptor()->ExpectRequestAndReturnResponseCodeAndHeaders( + 503, + "HTTP/1.1 503 Ugh\nRetry-After: 3600\n\n"); + + TestUploadCallback c; + uploader()->UploadReport("{}", 0, GURL(kUploadURL), c.callback()); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(1u, c.called_count()); + EXPECT_TRUE(c.last_result().is_retry_after()); +} + +TEST_F(DomainReliabilityUploaderTest, UploadDepth1) { + interceptor()->ExpectRequestAndReturnResponseCode(200); + + TestUploadCallback c; + uploader()->UploadReport("{}", 0, GURL(kUploadURL), c.callback()); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(1u, c.called_count()); + + EXPECT_EQ(1, interceptor()->last_upload_depth()); +} + +TEST_F(DomainReliabilityUploaderTest, UploadDepth2) { + interceptor()->ExpectRequestAndReturnResponseCode(200); + + TestUploadCallback c; + uploader()->UploadReport("{}", 1, GURL(kUploadURL), c.callback()); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(1u, c.called_count()); + + EXPECT_EQ(2, interceptor()->last_upload_depth()); +} + +} // namespace +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/util.cc b/chromium/components/domain_reliability/util.cc new file mode 100644 index 00000000000..8dd4e43e9f5 --- /dev/null +++ b/chromium/components/domain_reliability/util.cc @@ -0,0 +1,235 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/util.h" + +#include <stddef.h> + +#include "base/callback.h" +#include "base/logging.h" +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/time/time.h" +#include "base/timer/timer.h" +#include "net/base/net_errors.h" + +namespace domain_reliability { + +namespace { + +const struct NetErrorMapping { + int net_error; + const char* beacon_status; +} net_error_map[] = { + { net::OK, "ok" }, + { net::ERR_ABORTED, "aborted" }, + { net::ERR_TIMED_OUT, "tcp.connection.timed_out" }, + { net::ERR_CONNECTION_CLOSED, "tcp.connection.closed" }, + { net::ERR_CONNECTION_RESET, "tcp.connection.reset" }, + { net::ERR_CONNECTION_REFUSED, "tcp.connection.refused" }, + { net::ERR_CONNECTION_ABORTED, "tcp.connection.aborted" }, + { net::ERR_CONNECTION_FAILED, "tcp.connection.failed" }, + { net::ERR_NAME_NOT_RESOLVED, "dns" }, + { net::ERR_SSL_PROTOCOL_ERROR, "ssl.protocol.error" }, + { net::ERR_ADDRESS_INVALID, "tcp.connection.address_invalid" }, + { net::ERR_ADDRESS_UNREACHABLE, "tcp.connection.address_unreachable" }, + { net::ERR_CONNECTION_TIMED_OUT, "tcp.connection.timed_out" }, + { net::ERR_NAME_RESOLUTION_FAILED, "dns" }, + { net::ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN, + "ssl.cert.pinned_key_not_in_cert_chain" }, + { net::ERR_CERT_COMMON_NAME_INVALID, "ssl.cert.name_invalid" }, + { net::ERR_CERT_DATE_INVALID, "ssl.cert.date_invalid" }, + { net::ERR_CERT_AUTHORITY_INVALID, "ssl.cert.authority_invalid" }, + { net::ERR_CERT_REVOKED, "ssl.cert.revoked" }, + { net::ERR_CERT_INVALID, "ssl.cert.invalid" }, + { net::ERR_EMPTY_RESPONSE, "http.response.empty" }, + { net::ERR_SPDY_PING_FAILED, "spdy.ping_failed" }, + { net::ERR_SPDY_PROTOCOL_ERROR, "spdy.protocol" }, + { net::ERR_QUIC_PROTOCOL_ERROR, "quic.protocol" }, + { net::ERR_DNS_MALFORMED_RESPONSE, "dns.protocol" }, + { net::ERR_DNS_SERVER_FAILED, "dns.server" }, + { net::ERR_DNS_TIMED_OUT, "dns.timed_out" }, + { net::ERR_INSECURE_RESPONSE, "ssl" }, + { net::ERR_CONTENT_LENGTH_MISMATCH, "http.response.content_length_mismatch" }, + { net::ERR_INCOMPLETE_CHUNKED_ENCODING, + "http.response.incomplete_chunked_encoding" }, + { net::ERR_SSL_VERSION_OR_CIPHER_MISMATCH, + "ssl.version_or_cipher_mismatch" }, + { net::ERR_BAD_SSL_CLIENT_AUTH_CERT, "ssl.bad_client_auth_cert" }, + { net::ERR_INVALID_CHUNKED_ENCODING, + "http.response.invalid_chunked_encoding" }, + { net::ERR_RESPONSE_HEADERS_TRUNCATED, "http.response.headers.truncated" }, + { net::ERR_REQUEST_RANGE_NOT_SATISFIABLE, + "http.request.range_not_satisfiable" }, + { net::ERR_INVALID_RESPONSE, "http.response.invalid" }, + { net::ERR_RESPONSE_HEADERS_MULTIPLE_CONTENT_DISPOSITION, + "http.response.headers.multiple_content_disposition" }, + { net::ERR_RESPONSE_HEADERS_MULTIPLE_CONTENT_LENGTH, + "http.response.headers.multiple_content_length" }, + { net::ERR_SSL_UNRECOGNIZED_NAME_ALERT, "ssl.unrecognized_name_alert" } +}; + +bool CanReportFullBeaconURLToCollector(const GURL& beacon_url, + const GURL& collector_url) { + return beacon_url.GetOrigin() == collector_url.GetOrigin(); +} + +} // namespace + +// static +bool GetDomainReliabilityBeaconStatus( + int net_error, + int http_response_code, + std::string* beacon_status_out) { + if (net_error == net::OK) { + if (http_response_code >= 400 && http_response_code < 600) + *beacon_status_out = "http.error"; + else + *beacon_status_out = "ok"; + return true; + } + + // TODO(ttuttle): Consider sorting and using binary search? + for (size_t i = 0; i < arraysize(net_error_map); i++) { + if (net_error_map[i].net_error == net_error) { + *beacon_status_out = net_error_map[i].beacon_status; + return true; + } + } + return false; +} + +// TODO(ttuttle): Consider using NPN/ALPN instead, if there's a good way to +// differentiate HTTP and HTTPS. +std::string GetDomainReliabilityProtocol( + net::HttpResponseInfo::ConnectionInfo connection_info, + bool ssl_info_populated) { + switch (connection_info) { + case net::HttpResponseInfo::CONNECTION_INFO_UNKNOWN: + return ""; + case net::HttpResponseInfo::CONNECTION_INFO_HTTP1: + return ssl_info_populated ? "HTTPS" : "HTTP"; + case net::HttpResponseInfo::CONNECTION_INFO_DEPRECATED_SPDY2: + case net::HttpResponseInfo::CONNECTION_INFO_SPDY3: + case net::HttpResponseInfo::CONNECTION_INFO_HTTP2_14: + case net::HttpResponseInfo::CONNECTION_INFO_HTTP2_15: + case net::HttpResponseInfo::CONNECTION_INFO_HTTP2: + return "SPDY"; + case net::HttpResponseInfo::CONNECTION_INFO_QUIC1_SPDY3: + return "QUIC"; + case net::HttpResponseInfo::NUM_OF_CONNECTION_INFOS: + NOTREACHED(); + return ""; + } + NOTREACHED(); + return ""; +} + +int GetNetErrorFromURLRequestStatus(const net::URLRequestStatus& status) { + switch (status.status()) { + case net::URLRequestStatus::SUCCESS: + return net::OK; + case net::URLRequestStatus::CANCELED: + return net::ERR_ABORTED; + case net::URLRequestStatus::FAILED: + return status.error(); + default: + NOTREACHED(); + return net::ERR_FAILED; + } +} + +void GetUploadResultFromResponseDetails( + int net_error, + int http_response_code, + base::TimeDelta retry_after, + DomainReliabilityUploader::UploadResult* result) { + if (net_error == net::OK && http_response_code == 200) { + result->status = DomainReliabilityUploader::UploadResult::SUCCESS; + return; + } + + if (net_error == net::OK && + http_response_code == 503 && + retry_after != base::TimeDelta()) { + result->status = DomainReliabilityUploader::UploadResult::RETRY_AFTER; + result->retry_after = retry_after; + return; + } + + result->status = DomainReliabilityUploader::UploadResult::FAILURE; + return; +} + +// N.B. This uses a ScopedVector because that's what JSONValueConverter uses +// for repeated fields of any type, and Config uses JSONValueConverter to parse +// JSON configs. +GURL SanitizeURLForReport(const GURL& beacon_url, + const GURL& collector_url, + const ScopedVector<std::string>& path_prefixes) { + if (CanReportFullBeaconURLToCollector(beacon_url, collector_url)) + return beacon_url.GetAsReferrer(); + + std::string path = beacon_url.path(); + const std::string empty_path; + const std::string* longest_path_prefix = &empty_path; + for (const std::string* path_prefix : path_prefixes) { + if (path.substr(0, path_prefix->length()) == *path_prefix && + path_prefix->length() > longest_path_prefix->length()) { + longest_path_prefix = path_prefix; + } + } + + GURL::Replacements replacements; + replacements.ClearUsername(); + replacements.ClearPassword(); + replacements.SetPathStr(*longest_path_prefix); + replacements.ClearQuery(); + replacements.ClearRef(); + return beacon_url.ReplaceComponents(replacements); +} + +namespace { + +class ActualTimer : public MockableTime::Timer { + public: + // Initialize base timer with retain_user_info and is_repeating false. + ActualTimer() : base_timer_(false, false) {} + + ~ActualTimer() override {} + + // MockableTime::Timer implementation: + void Start(const tracked_objects::Location& posted_from, + base::TimeDelta delay, + const base::Closure& user_task) override { + base_timer_.Start(posted_from, delay, user_task); + } + + void Stop() override { base_timer_.Stop(); } + + bool IsRunning() override { return base_timer_.IsRunning(); } + + private: + base::Timer base_timer_; +}; + +} // namespace + +MockableTime::Timer::~Timer() {} +MockableTime::Timer::Timer() {} + +MockableTime::~MockableTime() {} +MockableTime::MockableTime() {} + +ActualTime::ActualTime() {} +ActualTime::~ActualTime() {} + +base::Time ActualTime::Now() { return base::Time::Now(); } +base::TimeTicks ActualTime::NowTicks() { return base::TimeTicks::Now(); } + +scoped_ptr<MockableTime::Timer> ActualTime::CreateTimer() { + return scoped_ptr<MockableTime::Timer>(new ActualTimer()); +} + +} // namespace domain_reliability diff --git a/chromium/components/domain_reliability/util.h b/chromium/components/domain_reliability/util.h new file mode 100644 index 00000000000..3cbf469b4f7 --- /dev/null +++ b/chromium/components/domain_reliability/util.h @@ -0,0 +1,112 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DOMAIN_RELIABILITY_UTIL_H_ +#define COMPONENTS_DOMAIN_RELIABILITY_UTIL_H_ + +#include <map> + +#include "base/callback_forward.h" +#include "base/compiler_specific.h" +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" +#include "base/time/clock.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "base/tracked_objects.h" +#include "components/domain_reliability/domain_reliability_export.h" +#include "components/domain_reliability/uploader.h" +#include "net/http/http_response_info.h" +#include "net/url_request/url_request_status.h" + +namespace domain_reliability { + +// Attempts to convert a net error and an HTTP response code into the status +// string that should be recorded in a beacon. Returns true if it could. +// +// N.B.: This functions as the whitelist of "safe" errors to report; network- +// local errors are purposefully not converted to avoid revealing +// information about the local network to the remote server. +bool GetDomainReliabilityBeaconStatus( + int net_error, + int http_response_code, + std::string* beacon_status_out); + +std::string GetDomainReliabilityProtocol( + net::HttpResponseInfo::ConnectionInfo connection_info, + bool ssl_info_populated); + +// Converts a URLRequestStatus into a network error. Returns the error code for +// FAILED; maps SUCCESS and CANCELED to OK and ERR_ABORTED, respectively; and +// returns ERR_ABORTED for any other status. +int GetNetErrorFromURLRequestStatus(const net::URLRequestStatus& status); + +// Based on the network error code, HTTP response code, and Retry-After value, +// fills |status| with the result of a report upload. +void GetUploadResultFromResponseDetails( + int net_error, + int http_response_code, + base::TimeDelta retry_after, + DomainReliabilityUploader::UploadResult* result); + +GURL SanitizeURLForReport(const GURL& beacon_url, + const GURL& collector_url, + const ScopedVector<std::string>& path_prefixes); + +// Mockable wrapper around TimeTicks::Now and Timer. Mock version is in +// test_util.h. +// TODO(ttuttle): Rename to Time{Provider,Source,?}. +class DOMAIN_RELIABILITY_EXPORT MockableTime : public base::Clock, + public base::TickClock { + public: + // Mockable wrapper around (a subset of) base::Timer. + class DOMAIN_RELIABILITY_EXPORT Timer { + public: + virtual ~Timer(); + + virtual void Start(const tracked_objects::Location& posted_from, + base::TimeDelta delay, + const base::Closure& user_task) = 0; + virtual void Stop() = 0; + virtual bool IsRunning() = 0; + + protected: + Timer(); + }; + + ~MockableTime() override; + + // Clock impl; returns base::Time::Now() or a mocked version thereof. + base::Time Now() override = 0; + // TickClock impl; returns base::TimeTicks::Now() or a mocked version thereof. + base::TimeTicks NowTicks() override = 0; + + // Returns a new Timer, or a mocked version thereof. + virtual scoped_ptr<MockableTime::Timer> CreateTimer() = 0; + + protected: + MockableTime(); + + private: + DISALLOW_COPY_AND_ASSIGN(MockableTime); +}; + +// Implementation of MockableTime that passes through to +// base::Time{,Ticks}::Now() and base::Timer. +class DOMAIN_RELIABILITY_EXPORT ActualTime : public MockableTime { + public: + ActualTime(); + + ~ActualTime() override; + + // MockableTime implementation: + base::Time Now() override; + base::TimeTicks NowTicks() override; + scoped_ptr<MockableTime::Timer> CreateTimer() override; +}; + +} // namespace domain_reliability + +#endif // COMPONENTS_DOMAIN_RELIABILITY_UTIL_H_ diff --git a/chromium/components/domain_reliability/util_unittest.cc b/chromium/components/domain_reliability/util_unittest.cc new file mode 100644 index 00000000000..3f21fccaa50 --- /dev/null +++ b/chromium/components/domain_reliability/util_unittest.cc @@ -0,0 +1,125 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/domain_reliability/util.h" + +#include "base/bind.h" +#include "components/domain_reliability/test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace domain_reliability { +namespace { + +using base::TimeDelta; +using base::TimeTicks; + +class DomainReliabilityMockTimeTest : public testing::Test { + protected: + MockTime time_; +}; + +TEST_F(DomainReliabilityMockTimeTest, Create) { +} + +TEST_F(DomainReliabilityMockTimeTest, NowAndAdvance) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + + TimeTicks initial = time_.NowTicks(); + time_.Advance(delta); + TimeTicks final = time_.NowTicks(); + EXPECT_EQ(delta, final - initial); +} + +TEST_F(DomainReliabilityMockTimeTest, AddTask) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + TestCallback callback; + + time_.AddTask(2 * delta, callback.callback()); + time_.Advance(delta); + EXPECT_FALSE(callback.called()); + time_.Advance(delta); + EXPECT_TRUE(callback.called()); +} + +TEST_F(DomainReliabilityMockTimeTest, TimerCreate) { + scoped_ptr<MockTime::Timer> timer(time_.CreateTimer()); +} + +TEST_F(DomainReliabilityMockTimeTest, TimerIsRunning) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + TestCallback callback; + + scoped_ptr<MockTime::Timer> timer(time_.CreateTimer()); + EXPECT_FALSE(timer->IsRunning()); + timer->Start(FROM_HERE, delta, callback.callback()); + EXPECT_TRUE(timer->IsRunning()); + timer->Stop(); + EXPECT_FALSE(timer->IsRunning()); +} + +TEST_F(DomainReliabilityMockTimeTest, TimerGoesOff) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + TestCallback callback; + + scoped_ptr<MockTime::Timer> timer(time_.CreateTimer()); + + timer->Start(FROM_HERE, 2 * delta, callback.callback()); + time_.Advance(delta); + EXPECT_FALSE(callback.called()); + time_.Advance(delta); + EXPECT_TRUE(callback.called()); +} + +TEST_F(DomainReliabilityMockTimeTest, TimerStopped) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + TestCallback callback; + + scoped_ptr<MockTime::Timer> timer(time_.CreateTimer()); + + timer->Start(FROM_HERE, 2 * delta, callback.callback()); + time_.Advance(delta); + timer->Stop(); + time_.Advance(delta); + EXPECT_FALSE(callback.called()); +} + +TEST_F(DomainReliabilityMockTimeTest, TimerRestarted) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + TestCallback callback; + + scoped_ptr<MockTime::Timer> timer(time_.CreateTimer()); + + timer->Start(FROM_HERE, 2 * delta, callback.callback()); + time_.Advance(delta); + timer->Start(FROM_HERE, 2 * delta, callback.callback()); + time_.Advance(delta); + EXPECT_FALSE(callback.called()); + time_.Advance(delta); + EXPECT_TRUE(callback.called()); +} + +TEST_F(DomainReliabilityMockTimeTest, TimerReentrantStart) { + const TimeDelta delta = TimeDelta::FromSeconds(1); + scoped_ptr<MockTime::Timer> timer(time_.CreateTimer()); + TestCallback callback; + + timer->Start( + FROM_HERE, + delta, + base::Bind( + &MockTime::Timer::Start, + base::Unretained(timer.get()), + FROM_HERE, + delta, + callback.callback())); + time_.Advance(delta); + EXPECT_FALSE(callback.called()); + EXPECT_TRUE(timer->IsRunning()); + time_.Advance(delta); + EXPECT_TRUE(callback.called()); + EXPECT_FALSE(timer->IsRunning()); +} + +} // namespace +} // namespace domain_reliability |