summaryrefslogtreecommitdiff
path: root/chromium/components/safe_browsing_db
diff options
context:
space:
mode:
authorAllan Sandfeld Jensen <allan.jensen@qt.io>2017-03-08 10:28:10 +0100
committerAllan Sandfeld Jensen <allan.jensen@qt.io>2017-03-20 13:40:30 +0000
commite733310db58160074f574c429d48f8308c0afe17 (patch)
treef8aef4b7e62a69928dbcf880620eece20f98c6df /chromium/components/safe_browsing_db
parent2f583e4aec1ae3a86fa047829c96b310dc12ecdf (diff)
downloadqtwebengine-chromium-e733310db58160074f574c429d48f8308c0afe17.tar.gz
BASELINE: Update Chromium to 56.0.2924.122
Change-Id: I4e04de8f47e47e501c46ed934c76a431c6337ced Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/safe_browsing_db')
-rw-r--r--chromium/components/safe_browsing_db/BUILD.gn50
-rw-r--r--chromium/components/safe_browsing_db/DEPS1
-rw-r--r--chromium/components/safe_browsing_db/PRESUBMIT.py12
-rw-r--r--chromium/components/safe_browsing_db/database_manager_unittest.cc3
-rw-r--r--chromium/components/safe_browsing_db/hit_report.h9
-rw-r--r--chromium/components/safe_browsing_db/prefix_set.cc2
-rw-r--r--chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc2
-rw-r--r--chromium/components/safe_browsing_db/safe_browsing_prefs.cc247
-rw-r--r--chromium/components/safe_browsing_db/safe_browsing_prefs.h113
-rw-r--r--chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc457
-rw-r--r--chromium/components/safe_browsing_db/safebrowsing.proto21
-rw-r--r--chromium/components/safe_browsing_db/testing_util.h26
-rw-r--r--chromium/components/safe_browsing_db/util.cc1
-rw-r--r--chromium/components/safe_browsing_db/v4_database.cc97
-rw-r--r--chromium/components/safe_browsing_db/v4_database.h40
-rw-r--r--chromium/components/safe_browsing_db/v4_database_unittest.cc183
-rw-r--r--chromium/components/safe_browsing_db/v4_feature_list.cc22
-rw-r--r--chromium/components/safe_browsing_db/v4_feature_list.h13
-rw-r--r--chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc78
-rw-r--r--chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h23
-rw-r--r--chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc73
-rw-r--r--chromium/components/safe_browsing_db/v4_local_database_manager.cc424
-rw-r--r--chromium/components/safe_browsing_db/v4_local_database_manager.h127
-rw-r--r--chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc572
-rw-r--r--chromium/components/safe_browsing_db/v4_protocol_manager_util.cc86
-rw-r--r--chromium/components/safe_browsing_db/v4_protocol_manager_util.h43
-rw-r--r--chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc48
-rw-r--r--chromium/components/safe_browsing_db/v4_store.cc452
-rw-r--r--chromium/components/safe_browsing_db/v4_store.h103
-rw-r--r--chromium/components/safe_browsing_db/v4_store_unittest.cc218
-rw-r--r--chromium/components/safe_browsing_db/v4_test_util.cc30
-rw-r--r--chromium/components/safe_browsing_db/v4_test_util.h23
-rw-r--r--chromium/components/safe_browsing_db/v4_update_protocol_manager.cc29
-rw-r--r--chromium/components/safe_browsing_db/v4_update_protocol_manager.h11
-rw-r--r--chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc92
35 files changed, 3071 insertions, 660 deletions
diff --git a/chromium/components/safe_browsing_db/BUILD.gn b/chromium/components/safe_browsing_db/BUILD.gn
index e14da2e5b0c..e7e788ceec9 100644
--- a/chromium/components/safe_browsing_db/BUILD.gn
+++ b/chromium/components/safe_browsing_db/BUILD.gn
@@ -31,6 +31,7 @@ group("safe_browsing_db_shared") {
":database_manager",
":hit_report",
":prefix_set",
+ ":safe_browsing_prefs",
":safebrowsing_proto",
":util",
]
@@ -78,6 +79,7 @@ static_library("hit_report") {
"hit_report.h",
]
deps = [
+ ":safe_browsing_prefs",
":util",
"//components/metrics",
"//url",
@@ -136,6 +138,17 @@ static_library("safe_browsing_api_handler_util") {
]
}
+static_library("safe_browsing_prefs") {
+ sources = [
+ "safe_browsing_prefs.cc",
+ "safe_browsing_prefs.h",
+ ]
+ deps = [
+ "//base:base",
+ "//components/prefs",
+ ]
+}
+
static_library("test_database_manager") {
sources = [
"test_database_manager.cc",
@@ -276,6 +289,18 @@ source_set("v4_store") {
]
}
+static_library("v4_test_util") {
+ testonly = true
+ sources = [
+ "v4_test_util.cc",
+ "v4_test_util.h",
+ ]
+ deps = [
+ ":util",
+ ":v4_protocol_manager_util",
+ ]
+}
+
static_library("v4_update_protocol_manager") {
sources = [
"v4_update_protocol_manager.cc",
@@ -318,6 +343,7 @@ source_set("v4_get_hash_protocol_manager_unittest") {
":v4_database",
":v4_get_hash_protocol_manager",
":v4_local_database_manager",
+ ":v4_test_util",
"//base",
"//base/test:test_support",
"//content/test:test_support",
@@ -335,6 +361,7 @@ source_set("v4_local_database_manager_unittest") {
deps = [
":v4_database",
":v4_local_database_manager",
+ ":v4_test_util",
"//base",
"//base/test:test_support",
"//content/test:test_support",
@@ -344,11 +371,30 @@ source_set("v4_local_database_manager_unittest") {
]
}
+source_set("v4_update_protocol_manager_unittest") {
+ testonly = true
+ sources = [
+ "v4_update_protocol_manager_unittest.cc",
+ ]
+ deps = [
+ ":safebrowsing_proto",
+ ":util",
+ ":v4_test_util",
+ ":v4_update_protocol_manager",
+ "//base",
+ "//base/test:test_support",
+ "//net",
+ "//net:test_support",
+ "//testing/gtest",
+ ]
+}
+
source_set("unit_tests") {
testonly = true
sources = [
"database_manager_unittest.cc",
"prefix_set_unittest.cc",
+ "safe_browsing_prefs_unittest.cc",
"util_unittest.cc",
"v4_database_unittest.cc",
"v4_get_hash_protocol_manager_unittest.cc",
@@ -361,6 +407,7 @@ source_set("unit_tests") {
deps = [
":database_manager",
":prefix_set",
+ ":safe_browsing_prefs",
":safebrowsing_proto",
":test_database_manager",
":util",
@@ -371,8 +418,10 @@ source_set("unit_tests") {
":v4_rice",
":v4_store",
":v4_store_proto",
+ ":v4_test_util",
":v4_update_protocol_manager",
"//base",
+ "//components/prefs:test_support",
"//content/test:test_support",
"//crypto",
"//net",
@@ -398,6 +447,7 @@ source_set("unit_tests_mobile") {
":safe_browsing_api_handler",
":safe_browsing_api_handler_util",
":util",
+ ":v4_test_util",
"//base",
"//components/variations",
"//testing/gtest",
diff --git a/chromium/components/safe_browsing_db/DEPS b/chromium/components/safe_browsing_db/DEPS
index 9d835a18bfe..91ea0ff090f 100644
--- a/chromium/components/safe_browsing_db/DEPS
+++ b/chromium/components/safe_browsing_db/DEPS
@@ -1,5 +1,6 @@
include_rules = [
"+components/data_use_measurement/core",
+ "+components/prefs",
"+components/variations",
"+content/public/browser",
"+content/public/common",
diff --git a/chromium/components/safe_browsing_db/PRESUBMIT.py b/chromium/components/safe_browsing_db/PRESUBMIT.py
new file mode 100644
index 00000000000..9a395c9243f
--- /dev/null
+++ b/chromium/components/safe_browsing_db/PRESUBMIT.py
@@ -0,0 +1,12 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Presubmit script for components/safe_browsing_db
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
+for more details about the presubmit API built into depot_tools.
+"""
+
+def CheckChangeOnUpload(input_api, output_api):
+ return input_api.canned_checks.CheckPatchFormatted(input_api, output_api)
diff --git a/chromium/components/safe_browsing_db/database_manager_unittest.cc b/chromium/components/safe_browsing_db/database_manager_unittest.cc
index 21338b86012..5180687b6f3 100644
--- a/chromium/components/safe_browsing_db/database_manager_unittest.cc
+++ b/chromium/components/safe_browsing_db/database_manager_unittest.cc
@@ -20,6 +20,7 @@
#include "base/threading/thread_task_runner_handle.h"
#include "components/safe_browsing_db/test_database_manager.h"
#include "components/safe_browsing_db/v4_protocol_manager_util.h"
+#include "components/safe_browsing_db/v4_test_util.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/test/test_browser_thread_bundle.h"
#include "crypto/sha2.h"
@@ -63,7 +64,7 @@ class SafeBrowsingDatabaseManagerTest : public testing::Test {
protected:
void SetUp() override {
db_manager_ = new TestSafeBrowsingDatabaseManager();
- db_manager_->StartOnIOThread(NULL, V4ProtocolConfig());
+ db_manager_->StartOnIOThread(NULL, GetTestV4ProtocolConfig());
}
void TearDown() override {
diff --git a/chromium/components/safe_browsing_db/hit_report.h b/chromium/components/safe_browsing_db/hit_report.h
index 158ec983d69..afa567637f1 100644
--- a/chromium/components/safe_browsing_db/hit_report.h
+++ b/chromium/components/safe_browsing_db/hit_report.h
@@ -7,6 +7,7 @@
#ifndef COMPONENTS_SAFE_BROWSING_DB_HIT_REPORT_H_
#define COMPONENTS_SAFE_BROWSING_DB_HIT_REPORT_H_
+#include "components/safe_browsing_db/safe_browsing_prefs.h"
#include "components/safe_browsing_db/util.h"
#include "url/gurl.h"
@@ -16,9 +17,9 @@ namespace safe_browsing {
enum class ThreatSource {
UNKNOWN,
DATA_SAVER, // From the Data Reduction service.
- LOCAL_PVER3, // From LocalSafeBrowingDatabaseManager, protocol v3
- LOCAL_PVER4, // From LocalSafeBrowingDatabaseManager, protocol v4
- REMOTE, // From RemoteSafeBrowingDatabaseManager
+ LOCAL_PVER3, // From LocalSafeBrowsingDatabaseManager, protocol v3
+ LOCAL_PVER4, // From V4LocalDatabaseManager, protocol v4
+ REMOTE, // From RemoteSafeBrowsingDatabaseManager
CLIENT_SIDE_DETECTION, // From ClientSideDetectionHost
};
@@ -41,7 +42,7 @@ struct HitReport {
// Opaque string used for tracking Pver4-based experiments
std::string population_id;
- bool is_extended_reporting;
+ ExtendedReportingLevel extended_reporting_level;
bool is_metrics_reporting_active;
std::string post_data;
diff --git a/chromium/components/safe_browsing_db/prefix_set.cc b/chromium/components/safe_browsing_db/prefix_set.cc
index 824bbd4f742..0701b8e25bc 100644
--- a/chromium/components/safe_browsing_db/prefix_set.cc
+++ b/chromium/components/safe_browsing_db/prefix_set.cc
@@ -15,7 +15,7 @@
#include "base/md5.h"
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram.h"
-#include "base/metrics/sparse_histogram.h"
+#include "base/metrics/histogram_macros.h"
namespace safe_browsing {
diff --git a/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc b/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc
index 8e91b098842..fbf07be6136 100644
--- a/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc
+++ b/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc
@@ -4,8 +4,8 @@
#include "components/safe_browsing_db/metadata.pb.h"
#include "components/safe_browsing_db/safe_browsing_api_handler_util.h"
-#include "components/safe_browsing_db/testing_util.h"
#include "components/safe_browsing_db/util.h"
+#include "components/safe_browsing_db/v4_test_util.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace safe_browsing {
diff --git a/chromium/components/safe_browsing_db/safe_browsing_prefs.cc b/chromium/components/safe_browsing_db/safe_browsing_prefs.cc
new file mode 100644
index 00000000000..c5a110aabdb
--- /dev/null
+++ b/chromium/components/safe_browsing_db/safe_browsing_prefs.cc
@@ -0,0 +1,247 @@
+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/command_line.h"
+#include "base/metrics/histogram_macros.h"
+#include "components/prefs/pref_service.h"
+#include "components/safe_browsing_db/safe_browsing_prefs.h"
+
+namespace {
+
+// Switch value which force the ScoutGroupSelected pref to true.
+const char kForceScoutGroupValueTrue[] = "true";
+// Switch value which force the ScoutGroupSelected pref to false.
+const char kForceScoutGroupValueFalse[] = "false";
+
+// Name of the Scout Transition UMA metric.
+const char kScoutTransitionMetricName[] = "SafeBrowsing.Pref.Scout.Transition";
+
+// Reasons that a state transition for Scout was performed.
+// These values are written to logs. New enum values can be added, but
+// existing enums must never be renumbered or deleted and reused.
+enum ScoutTransitionReason {
+ // Flag forced Scout Group to true
+ FORCE_SCOUT_FLAG_TRUE = 0,
+ // Flag forced Scout Group to false
+ FORCE_SCOUT_FLAG_FALSE = 1,
+ // User in OnlyShowScout group, enters Scout Group
+ ONLY_SHOW_SCOUT_OPT_IN = 2,
+ // User in CanShowScout group, enters Scout Group immediately
+ CAN_SHOW_SCOUT_OPT_IN_SCOUT_GROUP_ON = 3,
+ // User in CanShowScout group, waiting for interstitial to enter Scout Group
+ CAN_SHOW_SCOUT_OPT_IN_WAIT_FOR_INTERSTITIAL = 4,
+ // User in CanShowScout group saw the first interstitial and entered the Scout
+ // Group
+ CAN_SHOW_SCOUT_OPT_IN_SAW_FIRST_INTERSTITIAL = 5,
+ // User in Control group
+ CONTROL = 6,
+ // Rollback: SBER2 on on implies SBER1 can turn on
+ ROLLBACK_SBER2_IMPLIES_SBER1 = 7,
+ // Rollback: SBER2 off so SBER1 must be turned off
+ ROLLBACK_NO_SBER2_SET_SBER1_FALSE = 8,
+ // Rollback: SBER2 absent so SBER1 must be cleared
+ ROLLBACK_NO_SBER2_CLEAR_SBER1 = 9,
+ // New reasons must be added BEFORE MAX_REASONS
+ MAX_REASONS
+};
+} // namespace
+
+namespace prefs {
+const char kSafeBrowsingExtendedReportingEnabled[] =
+ "safebrowsing.extended_reporting_enabled";
+const char kSafeBrowsingScoutReportingEnabled[] =
+ "safebrowsing.scout_reporting_enabled";
+const char kSafeBrowsingScoutGroupSelected[] =
+ "safebrowsing.scout_group_selected";
+} // namespace prefs
+
+namespace safe_browsing {
+
+const char kSwitchForceScoutGroup[] = "force-scout-group";
+
+const base::Feature kCanShowScoutOptIn{"CanShowScoutOptIn",
+ base::FEATURE_DISABLED_BY_DEFAULT};
+
+const base::Feature kOnlyShowScoutOptIn{"OnlyShowScoutOptIn",
+ base::FEATURE_DISABLED_BY_DEFAULT};
+
+std::string ChooseOptInTextPreference(
+ const PrefService& prefs,
+ const std::string& extended_reporting_pref,
+ const std::string& scout_pref) {
+ return IsScout(prefs) ? scout_pref : extended_reporting_pref;
+}
+
+int ChooseOptInTextResource(const PrefService& prefs,
+ int extended_reporting_resource,
+ int scout_resource) {
+ return IsScout(prefs) ? scout_resource : extended_reporting_resource;
+}
+
+bool ExtendedReportingPrefExists(const PrefService& prefs) {
+ return prefs.HasPrefPath(GetExtendedReportingPrefName(prefs));
+}
+
+ExtendedReportingLevel GetExtendedReportingLevel(const PrefService& prefs) {
+ if (!IsExtendedReportingEnabled(prefs)) {
+ return SBER_LEVEL_OFF;
+ } else {
+ return IsScout(prefs) ? SBER_LEVEL_SCOUT : SBER_LEVEL_LEGACY;
+ }
+}
+
+const char* GetExtendedReportingPrefName(const PrefService& prefs) {
+ // The Scout pref is active if either of the experiment features are on, and
+ // ScoutGroupSelected is on as well.
+ if ((base::FeatureList::IsEnabled(kCanShowScoutOptIn) ||
+ base::FeatureList::IsEnabled(kOnlyShowScoutOptIn)) &&
+ prefs.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) {
+ return prefs::kSafeBrowsingScoutReportingEnabled;
+ }
+
+ // ..otherwise, either no experiment is on (ie: the Control group) or
+ // ScoutGroupSelected is off. So we use the SBER pref instead.
+ return prefs::kSafeBrowsingExtendedReportingEnabled;
+}
+
+void InitializeSafeBrowsingPrefs(PrefService* prefs) {
+ // First handle forcing kSafeBrowsingScoutGroupSelected pref via cmd-line.
+ if (base::CommandLine::ForCurrentProcess()->HasSwitch(
+ kSwitchForceScoutGroup)) {
+ std::string switch_value =
+ base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
+ kSwitchForceScoutGroup);
+ if (switch_value == kForceScoutGroupValueTrue) {
+ prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ FORCE_SCOUT_FLAG_TRUE, MAX_REASONS);
+ } else if (switch_value == kForceScoutGroupValueFalse) {
+ prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, false);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ FORCE_SCOUT_FLAG_FALSE, MAX_REASONS);
+ }
+
+ // If the switch is used, don't process the experiment state.
+ return;
+ }
+
+ // Handle the three possible experiment states.
+ if (base::FeatureList::IsEnabled(kOnlyShowScoutOptIn)) {
+ // OnlyShowScoutOptIn immediately turns on ScoutGroupSelected pref.
+ prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ ONLY_SHOW_SCOUT_OPT_IN, MAX_REASONS);
+ } else if (base::FeatureList::IsEnabled(kCanShowScoutOptIn)) {
+ // CanShowScoutOptIn will only turn on ScoutGroupSelected pref if the legacy
+ // SBER pref is false. Otherwise the legacy SBER pref will stay on and
+ // continue to be used until the next security incident, at which point
+ // the Scout pref will become the active one.
+ if (!prefs->GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)) {
+ prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ CAN_SHOW_SCOUT_OPT_IN_SCOUT_GROUP_ON,
+ MAX_REASONS);
+ } else {
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ CAN_SHOW_SCOUT_OPT_IN_WAIT_FOR_INTERSTITIAL,
+ MAX_REASONS);
+ }
+ } else {
+ // Both experiment features are off, so this is the Control group. We must
+ // handle the possibility that the user was previously in an experiment
+ // group (above) that was reverted. We want to restore the user to a
+ // reasonable state based on the ScoutGroup and ScoutReporting preferences.
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, CONTROL, MAX_REASONS);
+ if (prefs->GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled)) {
+ // User opted-in to Scout which is broader than legacy Extended Reporting.
+ // Opt them in to Extended Reporting.
+ prefs->SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, true);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ ROLLBACK_SBER2_IMPLIES_SBER1, MAX_REASONS);
+ } else if (prefs->GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) {
+ // User was in the Scout Group (ie: seeing the Scout opt-in text) but did
+ // NOT opt-in to Scout. Assume this was a conscious choice and remove
+ // their legacy Extended Reporting opt-in as well. The user will have a
+ // chance to evaluate their choice next time they see the opt-in text.
+
+ // We make the Extended Reporting pref mimic the state of the Scout
+ // Reporting pref. So we either Clear it or set it to False.
+ if (prefs->HasPrefPath(prefs::kSafeBrowsingScoutReportingEnabled)) {
+ // Scout Reporting pref was explicitly set to false, so set the SBER
+ // pref to false.
+ prefs->SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, false);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ ROLLBACK_NO_SBER2_SET_SBER1_FALSE,
+ MAX_REASONS);
+ } else {
+ // Scout Reporting pref is unset, so clear the SBER pref.
+ prefs->ClearPref(prefs::kSafeBrowsingExtendedReportingEnabled);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ ROLLBACK_NO_SBER2_CLEAR_SBER1, MAX_REASONS);
+ }
+ }
+
+ // Also clear both the Scout settings to start over from a clean state and
+ // avoid the above logic from triggering on next restart.
+ prefs->ClearPref(prefs::kSafeBrowsingScoutGroupSelected);
+ prefs->ClearPref(prefs::kSafeBrowsingScoutReportingEnabled);
+ }
+}
+
+bool IsExtendedReportingEnabled(const PrefService& prefs) {
+ return prefs.GetBoolean(GetExtendedReportingPrefName(prefs));
+}
+
+bool IsScout(const PrefService& prefs) {
+ return GetExtendedReportingPrefName(prefs) ==
+ prefs::kSafeBrowsingScoutReportingEnabled;
+}
+
+void RecordExtendedReportingMetrics(const PrefService& prefs) {
+ // This metric tracks the extended browsing opt-in based on whichever setting
+ // the user is currently seeing. It tells us whether extended reporting is
+ // happening for this user.
+ UMA_HISTOGRAM_BOOLEAN("SafeBrowsing.Pref.Extended",
+ IsExtendedReportingEnabled(prefs));
+
+ // These metrics track the Scout transition.
+ if (prefs.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) {
+ // Users in the Scout group: currently seeing the Scout opt-in.
+ UMA_HISTOGRAM_BOOLEAN(
+ "SafeBrowsing.Pref.Scout.ScoutGroup.SBER1Pref",
+ prefs.GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled));
+ UMA_HISTOGRAM_BOOLEAN(
+ "SafeBrowsing.Pref.Scout.ScoutGroup.SBER2Pref",
+ prefs.GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled));
+ } else {
+ // Users not in the Scout group: currently seeing the SBER opt-in.
+ UMA_HISTOGRAM_BOOLEAN(
+ "SafeBrowsing.Pref.Scout.NoScoutGroup.SBER1Pref",
+ prefs.GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled));
+ // The following metric is a corner case. User was previously in the
+ // Scout group and was able to opt-in to the Scout pref, but was since
+ // removed from the Scout group (eg: by rolling back a Scout experiment).
+ UMA_HISTOGRAM_BOOLEAN(
+ "SafeBrowsing.Pref.Scout.NoScoutGroup.SBER2Pref",
+ prefs.GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled));
+ }
+}
+
+void SetExtendedReportingPref(PrefService* prefs, bool value) {
+ prefs->SetBoolean(GetExtendedReportingPrefName(*prefs), value);
+}
+
+void UpdatePrefsBeforeSecurityInterstitial(PrefService* prefs) {
+ // Move the user into the Scout Group if the CanShowScoutOptIn experiment is
+ // enabled and they're not in the group already.
+ if (base::FeatureList::IsEnabled(kCanShowScoutOptIn) &&
+ !prefs->GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) {
+ prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true);
+ UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName,
+ CAN_SHOW_SCOUT_OPT_IN_SAW_FIRST_INTERSTITIAL,
+ MAX_REASONS);
+ }
+}
+
+} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/safe_browsing_prefs.h b/chromium/components/safe_browsing_db/safe_browsing_prefs.h
new file mode 100644
index 00000000000..c8856b40c1f
--- /dev/null
+++ b/chromium/components/safe_browsing_db/safe_browsing_prefs.h
@@ -0,0 +1,113 @@
+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// Safe Browsing preferences and some basic utility functions for using them.
+
+#ifndef COMPONENTS_SAFE_BROWSING_DB_SAFE_BROWSING_PREFS_H_
+#define COMPONENTS_SAFE_BROWSING_DB_SAFE_BROWSING_PREFS_H_
+
+#include "base/feature_list.h"
+
+class PrefService;
+
+namespace prefs {
+// Boolean that tell us whether Safe Browsing extended reporting is enabled.
+extern const char kSafeBrowsingExtendedReportingEnabled[];
+
+// Boolean indicating whether Safe Browsing Scout reporting is enabled, which
+// collects data for malware detection.
+extern const char kSafeBrowsingScoutReportingEnabled[];
+
+// Boolean indicating whether the Scout reporting workflow is enabled. This
+// affects which of SafeBrowsingExtendedReporting or SafeBrowsingScoutReporting
+// is used.
+extern const char kSafeBrowsingScoutGroupSelected[];
+}
+
+namespace safe_browsing {
+
+// Command-line switch for changing the scout_group_selected preference. Should
+// be set to either 'true' or 'false'. Primarily for testing purposes.
+// TODO: this is temporary (crbug.com/662944)
+extern const char kSwitchForceScoutGroup[];
+
+// When this feature is enabled, the Scout opt-in text will be displayed as of
+// the next security incident. Until then, the legacy SBER text will appear.
+// TODO: this is temporary (crbug.com/662944)
+extern const base::Feature kCanShowScoutOptIn;
+
+// When this feature is enabled, the Scout opt-in text will immediately be
+// displayed everywhere.
+// TODO: this is temporary (crbug.com/662944)
+extern const base::Feature kOnlyShowScoutOptIn;
+
+// Enumerates the level of Safe Browsing Extended Reporting that is currently
+// available.
+enum ExtendedReportingLevel {
+ // Extended reporting is off.
+ SBER_LEVEL_OFF = 0,
+ // The Legacy level of extended reporting is available, reporting happens in
+ // response to security incidents.
+ SBER_LEVEL_LEGACY = 1,
+ // The Scout level of extended reporting is available, some data can be
+ // collected to actively detect dangerous apps and sites.
+ SBER_LEVEL_SCOUT = 2,
+};
+
+// Determines which opt-in text should be used based on the currently active
+// preference. Will return either |extended_reporting_pref| if the legacy
+// Extended Reporting pref is active, or |scout_pref| if the Scout pref is
+// active. Used for Android.
+std::string ChooseOptInTextPreference(
+ const PrefService& prefs,
+ const std::string& extended_reporting_pref,
+ const std::string& scout_pref);
+
+// Determines which opt-in text should be used based on the currently active
+// preference. Will return either |extended_reporting_resource| if the legacy
+// Extended Reporting pref is active, or |scout_resource| if the Scout pref is
+// active.
+int ChooseOptInTextResource(const PrefService& prefs,
+ int extended_reporting_resource,
+ int scout_resource);
+
+// Returns whether the currently active Safe Browsing Extended Reporting
+// preference exists (eg: has been set before).
+bool ExtendedReportingPrefExists(const PrefService& prefs);
+
+// Returns the level of reporting available for the current user.
+ExtendedReportingLevel GetExtendedReportingLevel(const PrefService& prefs);
+
+// Returns the name of the Safe Browsing Extended Reporting pref that is
+// currently in effect. The specific pref in-use may change through experiments.
+const char* GetExtendedReportingPrefName(const PrefService& prefs);
+
+// Initializes Safe Browsing preferences based on data such as experiment state,
+// command line flags, etc.
+// TODO: this is temporary (crbug.com/662944)
+void InitializeSafeBrowsingPrefs(PrefService* prefs);
+
+// Returns whether Safe Browsing Extended Reporting is currently enabled.
+// This should be used to decide if any of the reporting preferences are set,
+// regardless of which specific one is set.
+bool IsExtendedReportingEnabled(const PrefService& prefs);
+
+// Returns whether the currently-active Extended Reporting pref is Scout.
+bool IsScout(const PrefService& prefs);
+
+// Updates UMA metrics about Safe Browsing Extended Reporting states.
+void RecordExtendedReportingMetrics(const PrefService& prefs);
+
+// Sets the currently active Safe Browsing Extended Reporting preference to the
+// specified value.
+void SetExtendedReportingPref(PrefService* prefs, bool value);
+
+// Called to indicate that a security interstitial is about to be shown to the
+// user. This may trigger the user to begin seeing the Scout opt-in text
+// depending on their experiment state.
+void UpdatePrefsBeforeSecurityInterstitial(PrefService* prefs);
+
+} // namespace safe_browsing
+
+#endif // COMPONENTS_SAFE_BROWSING_DB_SAFE_BROWSING_PREFS_H_
diff --git a/chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc b/chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc
new file mode 100644
index 00000000000..af38c78edf8
--- /dev/null
+++ b/chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc
@@ -0,0 +1,457 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+#include <vector>
+
+#include "base/command_line.h"
+#include "base/strings/string_util.h"
+#include "base/test/scoped_feature_list.h"
+#include "components/prefs/pref_registry_simple.h"
+#include "components/prefs/testing_pref_service.h"
+#include "components/safe_browsing_db/safe_browsing_prefs.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace safe_browsing {
+
+class SafeBrowsingPrefsTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ prefs_.registry()->RegisterBooleanPref(
+ prefs::kSafeBrowsingExtendedReportingEnabled, false);
+ prefs_.registry()->RegisterBooleanPref(
+ prefs::kSafeBrowsingScoutReportingEnabled, false);
+ prefs_.registry()->RegisterBooleanPref(
+ prefs::kSafeBrowsingScoutGroupSelected, false);
+
+ ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/false);
+ }
+
+ void ResetPrefs(bool sber_reporting, bool scout_reporting, bool scout_group) {
+ prefs_.SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled,
+ sber_reporting);
+ prefs_.SetBoolean(prefs::kSafeBrowsingScoutReportingEnabled,
+ scout_reporting);
+ prefs_.SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, scout_group);
+ }
+
+ void ResetExperiments(bool can_show_scout, bool only_show_scout) {
+ std::vector<std::string> enabled_features;
+ std::vector<std::string> disabled_features;
+
+ auto* target_vector =
+ can_show_scout ? &enabled_features : &disabled_features;
+ target_vector->push_back(kCanShowScoutOptIn.name);
+
+ target_vector = only_show_scout ? &enabled_features : &disabled_features;
+ target_vector->push_back(kOnlyShowScoutOptIn.name);
+
+ feature_list_.reset(new base::test::ScopedFeatureList);
+ feature_list_->InitFromCommandLine(
+ base::JoinString(enabled_features, ","),
+ base::JoinString(disabled_features, ","));
+ }
+
+ std::string GetActivePref() { return GetExtendedReportingPrefName(prefs_); }
+
+ // Convenience method for explicitly setting up all combinations of prefs and
+ // experiments.
+ void TestGetPrefName(bool sber_reporting,
+ bool scout_reporting,
+ bool scout_group,
+ bool can_show_scout,
+ bool only_show_scout,
+ const std::string& expected_pref) {
+ ResetPrefs(sber_reporting, scout_reporting, scout_group);
+ ResetExperiments(can_show_scout, only_show_scout);
+ EXPECT_EQ(expected_pref, GetActivePref())
+ << "sber=" << sber_reporting << " scout=" << scout_reporting
+ << " scout_group=" << scout_group
+ << " can_show_scout=" << can_show_scout
+ << " only_show_scout=" << only_show_scout;
+ }
+
+ void InitPrefs() { InitializeSafeBrowsingPrefs(&prefs_); }
+
+ bool IsScoutGroupSelected() {
+ return prefs_.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected);
+ }
+
+ void ExpectPrefs(bool sber_reporting,
+ bool scout_reporting,
+ bool scout_group) {
+ LOG(INFO) << "Pref values: sber=" << sber_reporting
+ << " scout=" << scout_reporting << " scout_group=" << scout_group;
+ EXPECT_EQ(sber_reporting,
+ prefs_.GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled));
+ EXPECT_EQ(scout_reporting,
+ prefs_.GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled));
+ EXPECT_EQ(scout_group,
+ prefs_.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected));
+ }
+
+ void ExpectPrefsExist(bool sber_reporting,
+ bool scout_reporting,
+ bool scout_group) {
+ LOG(INFO) << "Prefs exist: sber=" << sber_reporting
+ << " scout=" << scout_reporting << " scout_group=" << scout_group;
+ EXPECT_EQ(sber_reporting,
+ prefs_.HasPrefPath(prefs::kSafeBrowsingExtendedReportingEnabled));
+ EXPECT_EQ(scout_reporting,
+ prefs_.HasPrefPath(prefs::kSafeBrowsingScoutReportingEnabled));
+ EXPECT_EQ(scout_group,
+ prefs_.HasPrefPath(prefs::kSafeBrowsingScoutGroupSelected));
+ }
+ TestingPrefServiceSimple prefs_;
+
+ private:
+ std::unique_ptr<base::test::ScopedFeatureList> feature_list_;
+};
+
+// This test ensures that we correctly select between SBER and Scout as the
+// active preference in a number of common scenarios.
+TEST_F(SafeBrowsingPrefsTest, GetExtendedReportingPrefName_Common) {
+ const std::string& sber = prefs::kSafeBrowsingExtendedReportingEnabled;
+ const std::string& scout = prefs::kSafeBrowsingScoutReportingEnabled;
+
+ // By default (all prefs and experiment features disabled), SBER pref is used.
+ TestGetPrefName(false, false, false, false, false, sber);
+
+ // Changing any prefs (including ScoutGroupSelected) keeps SBER as the active
+ // pref because the experiment remains in the Control group.
+ TestGetPrefName(/*sber=*/true, false, false, false, false, sber);
+ TestGetPrefName(false, /*scout=*/true, false, false, false, sber);
+ TestGetPrefName(false, false, /*scout_group=*/true, false, false, sber);
+
+ // Being in either experiment group with ScoutGroup selected makes Scout the
+ // active pref.
+ TestGetPrefName(false, false, /*scout_group=*/true, /*can_show_scout=*/true,
+ false, scout);
+ TestGetPrefName(false, false, /*scout_group=*/true, false,
+ /*only_show_scout=*/true, scout);
+
+ // When ScoutGroup is not selected then SBER remains the active pref,
+ // regardless which experiment is enabled.
+ TestGetPrefName(false, false, false, /*can_show_scout=*/true, false, sber);
+ TestGetPrefName(false, false, false, false, /*only_show_scout=*/true, sber);
+}
+
+// Here we exhaustively check all combinations of pref and experiment states.
+// This should help catch regressions.
+TEST_F(SafeBrowsingPrefsTest, GetExtendedReportingPrefName_Exhaustive) {
+ const std::string& sber = prefs::kSafeBrowsingExtendedReportingEnabled;
+ const std::string& scout = prefs::kSafeBrowsingScoutReportingEnabled;
+ TestGetPrefName(false, false, false, false, false, sber);
+ TestGetPrefName(false, false, false, false, true, sber);
+ TestGetPrefName(false, false, false, true, false, sber);
+ TestGetPrefName(false, false, false, true, true, sber);
+ TestGetPrefName(false, false, true, false, false, sber);
+ TestGetPrefName(false, false, true, false, true, scout);
+ TestGetPrefName(false, false, true, true, false, scout);
+ TestGetPrefName(false, false, true, true, true, scout);
+ TestGetPrefName(false, true, false, false, false, sber);
+ TestGetPrefName(false, true, false, false, true, sber);
+ TestGetPrefName(false, true, false, true, false, sber);
+ TestGetPrefName(false, true, false, true, true, sber);
+ TestGetPrefName(false, true, true, false, false, sber);
+ TestGetPrefName(false, true, true, false, true, scout);
+ TestGetPrefName(false, true, true, true, false, scout);
+ TestGetPrefName(false, true, true, true, true, scout);
+ TestGetPrefName(true, false, false, false, false, sber);
+ TestGetPrefName(true, false, false, false, true, sber);
+ TestGetPrefName(true, false, false, true, false, sber);
+ TestGetPrefName(true, false, false, true, true, sber);
+ TestGetPrefName(true, false, true, false, false, sber);
+ TestGetPrefName(true, false, true, false, true, scout);
+ TestGetPrefName(true, false, true, true, false, scout);
+ TestGetPrefName(true, false, true, true, true, scout);
+ TestGetPrefName(true, true, false, false, false, sber);
+ TestGetPrefName(true, true, false, false, true, sber);
+ TestGetPrefName(true, true, false, true, false, sber);
+ TestGetPrefName(true, true, false, true, true, sber);
+ TestGetPrefName(true, true, true, false, false, sber);
+ TestGetPrefName(true, true, true, false, true, scout);
+ TestGetPrefName(true, true, true, true, false, scout);
+ TestGetPrefName(true, true, true, true, true, scout);
+}
+
+// Basic test that command-line flags can force the ScoutGroupSelected pref on
+// or off.
+TEST_F(SafeBrowsingPrefsTest, InitPrefs_ForceScoutGroupOnOff) {
+ // By default ScoutGroupSelected is off.
+ EXPECT_FALSE(IsScoutGroupSelected());
+
+ // Command-line flag can force it on during initialization.
+ base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+ kSwitchForceScoutGroup, "true");
+ InitPrefs();
+ EXPECT_TRUE(IsScoutGroupSelected());
+
+ // ScoutGroup remains on if switches are cleared, but only if an experiment
+ // is active (since being in the Control group automatically clears the
+ // Scout prefs).
+ base::CommandLine::StringVector empty;
+ base::CommandLine::ForCurrentProcess()->InitFromArgv(empty);
+ ResetExperiments(/*can_show_scout=*/true, /*only_show_scout=*/false);
+ EXPECT_TRUE(IsScoutGroupSelected());
+
+ // Nonsense values are ignored and ScoutGroup is unchanged.
+ base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+ kSwitchForceScoutGroup, "foo");
+ InitPrefs();
+ EXPECT_TRUE(IsScoutGroupSelected());
+
+ // ScoutGroup can also be forced off during initialization.
+ base::CommandLine::ForCurrentProcess()->InitFromArgv(empty);
+ base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+ kSwitchForceScoutGroup, "false");
+ InitPrefs();
+ EXPECT_FALSE(IsScoutGroupSelected());
+}
+
+// Test all combinations of prefs during initialization when neither experiment
+// is on (ie: control group). In all cases the Scout prefs should be cleared,
+// and the SBER pref may get switched.
+TEST_F(SafeBrowsingPrefsTest, InitPrefs_Control) {
+ // Turn both experiments off.
+ ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/false);
+
+ // Default case (everything off) - no change on init.
+ ResetPrefs(false, false, false);
+ InitPrefs();
+ ExpectPrefs(false, false, false);
+ // SBER pref exists because it was set to false above.
+ ExpectPrefsExist(true, false, false);
+
+ // ScoutGroup on - SBER cleared since Scout opt-in was shown but Scout pref
+ // was not chosen. Scout prefs cleared.
+ ResetPrefs(false, false, true);
+ InitPrefs();
+ ExpectPrefs(false, false, false);
+ ExpectPrefsExist(true, false, false);
+
+ // ScoutReporting on without ScoutGroup - SBER turns on since user opted-in to
+ // broader Scout reporting, we can continue collecting the SBER subset. Scout
+ // prefs cleared.
+ ResetPrefs(false, true, false);
+ InitPrefs();
+ ExpectPrefs(true, false, false);
+ ExpectPrefsExist(true, false, false);
+
+ // ScoutReporting and ScoutGroup on - SBER turns on since user opted-in to
+ // broader Scout reporting, we can continue collecting the SBER subset. Scout
+ // prefs cleared.
+ ResetPrefs(false, true, true);
+ InitPrefs();
+ ExpectPrefs(true, false, false);
+ ExpectPrefsExist(true, false, false);
+
+ // SBER on - no change on init since ScoutGroup is off implying that user
+ // never saw Scout opt-in text. Scout prefs remain cleared.
+ ResetPrefs(true, false, false);
+ InitPrefs();
+ ExpectPrefs(true, false, false);
+ ExpectPrefsExist(true, false, false);
+
+ // SBER and ScoutGroup on - SBER cleared. User previously opted-in to SBER
+ // and they saw Scout opt-in text (ie. ScoutGroup on), but chose not to opt-in
+ // to Scout reporting. We want them to re-evaluate their choice of SBER since
+ // the lack of Scout opt-in was a conscious choice. Scout cleared.
+ ResetPrefs(true, false, true);
+ InitPrefs();
+ ExpectPrefs(false, false, false);
+ ExpectPrefsExist(true, false, false);
+
+ // SBER and ScoutReporting on. User has opted-in to broader level of reporting
+ // so SBER stays on. Scout prefs cleared.
+ ResetPrefs(true, true, false);
+ InitPrefs();
+ ExpectPrefs(true, false, false);
+ ExpectPrefsExist(true, false, false);
+
+ // Everything on. User has opted-in to broader level of reporting so SBER
+ // stays on. Scout prefs cleared.
+ ResetPrefs(true, true, true);
+ InitPrefs();
+ ExpectPrefs(true, false, false);
+ ExpectPrefs(true, false, false);
+}
+
+// Tests a unique case where the Extended Reporting pref will be Cleared instead
+// of set to False in order to mimic the state of the Scout reporting pref.
+// This happens when a user is in the OnlyShowScoutOptIn experiment but never
+// encounters a security issue so never sees the Scout opt-in. This user then
+// returns to the Control group having never seen the Scout opt-in, so their
+// Scout Reporting pref is un-set. We want to return their SBER pref to the
+// unset state as well.
+TEST_F(SafeBrowsingPrefsTest, InitPrefs_Control_SberPrefCleared) {
+ // Turn both experiments off.
+ ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/false);
+
+ // Set the user's old SBER pref to on to be explicit.
+ prefs_.SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, true);
+ // User is in the OnlyShowScoutOptIn experiment so they go directly to the
+ // Scout Group.
+ prefs_.SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true);
+ // But they never see a security popup or change the setting manually so the
+ // Scout pref remains unset.
+ prefs_.ClearPref(prefs::kSafeBrowsingScoutReportingEnabled);
+
+ InitPrefs();
+
+ // All pref values should be false and unset.
+ ExpectPrefs(false, false, false);
+ ExpectPrefsExist(false, false, false);
+}
+
+// Test all combinations of prefs during initialization when the CanShowScout
+// experiment is on.
+TEST_F(SafeBrowsingPrefsTest, InitPrefs_CanShowScout) {
+ // Turn the CanShowScout experiment on.
+ ResetExperiments(/*can_show_scout=*/true, /*only_show_scout=*/false);
+
+ // Default case (everything off) - ScoutGroup turns on because SBER is off.
+ ResetPrefs(false, false, false);
+ InitPrefs();
+ ExpectPrefs(false, false, true);
+
+ // ScoutGroup on - no change on init since ScoutGroup is already on.
+ ResetPrefs(false, false, true);
+ InitPrefs();
+ ExpectPrefs(false, false, true);
+
+ // ScoutReporting on without ScoutGroup - ScoutGroup turns on because SBER is
+ // off.
+ ResetPrefs(false, true, false);
+ InitPrefs();
+ ExpectPrefs(false, true, true);
+
+ // ScoutReporting and ScoutGroup on - no change on init since ScoutGroup is
+ // already on.
+ ResetPrefs(false, true, true);
+ InitPrefs();
+ ExpectPrefs(false, true, true);
+
+ // SBER on - no change on init. Will wait for first security incident before
+ // turning on ScoutGroup and displaying the Scout opt-in.
+ ResetPrefs(true, false, false);
+ InitPrefs();
+ ExpectPrefs(true, false, false);
+
+ // SBER and ScoutGroup on - no change on init since ScoutGroup is already on.
+ ResetPrefs(true, false, true);
+ InitPrefs();
+ ExpectPrefs(true, false, true);
+
+ // SBER and ScoutReporting on - no change on init because SBER is on.
+ // ScoutGroup will turn on on next security incident.
+ ResetPrefs(true, true, false);
+ InitPrefs();
+ ExpectPrefs(true, true, false);
+
+ // Everything on - no change on init since ScoutGroup is already on.
+ ResetPrefs(true, true, true);
+ InitPrefs();
+ ExpectPrefs(true, true, true);
+}
+
+// Test all combinations of prefs during initialization when the OnlyShowScout
+// experiment is on.
+TEST_F(SafeBrowsingPrefsTest, InitPrefs_OnlyShowScout) {
+ // Turn the OnlyShowScout experiment on.
+ ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/true);
+
+ // Default case (everything off) - ScoutGroup turns on.
+ ResetPrefs(false, false, false);
+ InitPrefs();
+ ExpectPrefs(false, false, true);
+
+ // ScoutGroup on - no change on init since ScoutGroup is already on.
+ ResetPrefs(false, false, true);
+ InitPrefs();
+ ExpectPrefs(false, false, true);
+
+ // ScoutReporting on without ScoutGroup - ScoutGroup turns on.
+ ResetPrefs(false, true, false);
+ InitPrefs();
+ ExpectPrefs(false, true, true);
+
+ // ScoutReporting and ScoutGroup on - no change on init since ScoutGroup is
+ // already on.
+ ResetPrefs(false, true, true);
+ InitPrefs();
+ ExpectPrefs(false, true, true);
+
+ // SBER on - ScoutGroup turns on immediately, not waiting for first security
+ // incident.
+ ResetPrefs(true, false, false);
+ InitPrefs();
+ ExpectPrefs(true, false, true);
+
+ // SBER and ScoutGroup on - no change on init since ScoutGroup is already on.
+ ResetPrefs(true, false, true);
+ InitPrefs();
+ ExpectPrefs(true, false, true);
+
+ // SBER and ScoutReporting on - ScoutGroup turns on immediately, not waiting
+ // for first security incident.
+ ResetPrefs(true, true, false);
+ InitPrefs();
+ ExpectPrefs(true, true, true);
+
+ // Everything on - no change on init since ScoutGroup is already on.
+ ResetPrefs(true, true, true);
+ InitPrefs();
+ ExpectPrefs(true, true, true);
+}
+
+TEST_F(SafeBrowsingPrefsTest, ChooseOptInText) {
+ const int kSberResource = 100;
+ const int kScoutResource = 500;
+ // By default, SBER opt-in is used
+ EXPECT_EQ(kSberResource,
+ ChooseOptInTextResource(prefs_, kSberResource, kScoutResource));
+
+ // Enabling Scout switches to the Scout opt-in text.
+ ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/true);
+ ResetPrefs(/*sber=*/false, /*scout=*/false, /*scout_group=*/true);
+ EXPECT_EQ(kScoutResource,
+ ChooseOptInTextResource(prefs_, kSberResource, kScoutResource));
+}
+
+TEST_F(SafeBrowsingPrefsTest, GetSafeBrowsingExtendedReportingLevel) {
+ // By Default, SBER is off
+ EXPECT_EQ(SBER_LEVEL_OFF, GetExtendedReportingLevel(prefs_));
+
+ // Opt-in to Legacy SBER gives Legacy reporting leve.
+ ResetPrefs(/*sber=*/true, /*scout_reporting=*/false, /*scout_group=*/false);
+ EXPECT_EQ(SBER_LEVEL_LEGACY, GetExtendedReportingLevel(prefs_));
+
+ // The value of the Scout pref doesn't change the reporting level if the user
+ // is outside of the Scout Group and/or no experiment is running.
+ // No scout group.
+ ResetPrefs(/*sber=*/true, /*scout_reporting=*/true, /*scout_group=*/false);
+ EXPECT_EQ(SBER_LEVEL_LEGACY, GetExtendedReportingLevel(prefs_));
+ // Scout group but no experiment.
+ ResetPrefs(/*sber=*/true, /*scout_reporting=*/true, /*scout_group=*/true);
+ EXPECT_EQ(SBER_LEVEL_LEGACY, GetExtendedReportingLevel(prefs_));
+
+ // Remaining in the Scout Group and adding an experiment will switch to the
+ // Scout pref to determine reporting level.
+ ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/true);
+ // Both reporting prefs off, so reporting is off.
+ ResetPrefs(/*sber=*/false, /*scout_reporting=*/false, /*scout_group=*/true);
+ EXPECT_EQ(SBER_LEVEL_OFF, GetExtendedReportingLevel(prefs_));
+ // Legacy pref on when we're using Scout - reporting remains off.
+ ResetPrefs(/*sber=*/true, /*scout_reporting=*/false, /*scout_group=*/true);
+ EXPECT_EQ(SBER_LEVEL_OFF, GetExtendedReportingLevel(prefs_));
+ // Turning on Scout gives us Scout level reporting
+ ResetPrefs(/*sber=*/false, /*scout_reporting=*/true, /*scout_group=*/true);
+ EXPECT_EQ(SBER_LEVEL_SCOUT, GetExtendedReportingLevel(prefs_));
+ // .. and the legacy pref doesn't affect this.
+ ResetPrefs(/*sber=*/true, /*scout_reporting=*/true, /*scout_group=*/true);
+ EXPECT_EQ(SBER_LEVEL_SCOUT, GetExtendedReportingLevel(prefs_));
+}
+
+} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/safebrowsing.proto b/chromium/components/safe_browsing_db/safebrowsing.proto
index c9c24deb953..fb5785d28d7 100644
--- a/chromium/components/safe_browsing_db/safebrowsing.proto
+++ b/chromium/components/safe_browsing_db/safebrowsing.proto
@@ -267,6 +267,18 @@ enum ThreatType {
// API abuse threat type.
API_ABUSE = 6;
+
+ // Malicious binary threat type.
+ MALICIOUS_BINARY = 7;
+
+ // Client side detection whitelist threat type.
+ CSD_WHITELIST = 8;
+
+ // Client side download detection whitelist threat type.
+ CSD_DOWNLOAD_WHITELIST = 9;
+
+ // Client incident threat type.
+ CLIENT_INCIDENT = 10;
}
// Types of platforms.
@@ -354,6 +366,15 @@ enum ThreatEntryType {
// An IP range.
IP_RANGE = 3;
+
+ // Chrome extension.
+ CHROME_EXTENSION = 4;
+
+ // Filename.
+ FILENAME = 5;
+
+ // CERT.
+ CERT = 6;
}
// A set of threats that should be added or removed from a client's local
diff --git a/chromium/components/safe_browsing_db/testing_util.h b/chromium/components/safe_browsing_db/testing_util.h
deleted file mode 100644
index 8447162681c..00000000000
--- a/chromium/components/safe_browsing_db/testing_util.h
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright (c) 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-//
-// Utilities to be used in tests of safe_browsing_db/ component.
-
-#ifndef COMPONENTS_SAFE_BROWSING_DB_TESTING_UTIL_H_
-#define COMPONENTS_SAFE_BROWSING_DB_TESTING_UTIL_H_
-
-#include "components/safe_browsing_db/util.h"
-
-#include <ostream>
-
-namespace safe_browsing {
-
-inline std::ostream& operator<<(std::ostream& os, const ThreatMetadata& meta) {
- os << "{threat_pattern_type=" << static_cast<int>(meta.threat_pattern_type)
- << ", api_permissions=[";
- for (auto p : meta.api_permissions)
- os << p << ",";
- return os << "], population_id=" << meta.population_id << "}";
-}
-
-} // namespace safe_browsing
-
-#endif // COMPONENTS_SAFE_BROWSING_DB_TESTING_UTIL_H_
diff --git a/chromium/components/safe_browsing_db/util.cc b/chromium/components/safe_browsing_db/util.cc
index 3f34fe5fb09..0f38dee61bd 100644
--- a/chromium/components/safe_browsing_db/util.cc
+++ b/chromium/components/safe_browsing_db/util.cc
@@ -134,6 +134,7 @@ bool GetListName(ListType list_id, std::string* list) {
break;
case MODULEWHITELIST:
*list = kModuleWhitelist;
+ break;
case RESOURCEBLACKLIST:
*list = kResourceBlacklist;
break;
diff --git a/chromium/components/safe_browsing_db/v4_database.cc b/chromium/components/safe_browsing_db/v4_database.cc
index 6e14e5f099d..5edeb3e9422 100644
--- a/chromium/components/safe_browsing_db/v4_database.cc
+++ b/chromium/components/safe_browsing_db/v4_database.cc
@@ -8,14 +8,22 @@
#include "base/debug/leak_annotations.h"
#include "base/files/file_util.h"
#include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_macros.h"
#include "base/threading/thread_task_runner_handle.h"
#include "components/safe_browsing_db/v4_database.h"
#include "content/public/browser/browser_thread.h"
using content::BrowserThread;
+using base::TimeTicks;
namespace safe_browsing {
+namespace {
+
+const char kV4DatabaseSizeMetric[] = "SafeBrowsing.V4Database.Size";
+
+} // namespace
+
// static
V4StoreFactory* V4Database::factory_ = NULL;
@@ -31,9 +39,9 @@ void V4Database::Create(
const scoped_refptr<base::SingleThreadTaskRunner> callback_task_runner =
base::ThreadTaskRunnerHandle::Get();
db_task_runner->PostTask(
- FROM_HERE,
- base::Bind(&V4Database::CreateOnTaskRunner, db_task_runner, base_path,
- list_infos, callback_task_runner, new_db_callback));
+ FROM_HERE, base::Bind(&V4Database::CreateOnTaskRunner, db_task_runner,
+ base_path, list_infos, callback_task_runner,
+ new_db_callback, TimeTicks::Now()));
}
// static
@@ -42,7 +50,8 @@ void V4Database::CreateOnTaskRunner(
const base::FilePath& base_path,
const ListInfos& list_infos,
const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner,
- NewDatabaseReadyCallback new_db_callback) {
+ NewDatabaseReadyCallback new_db_callback,
+ const TimeTicks create_start_time) {
DCHECK(db_task_runner->RunsTasksOnCurrentThread());
if (!factory_) {
@@ -69,9 +78,12 @@ void V4Database::CreateOnTaskRunner(
new V4Database(db_task_runner, std::move(store_map)));
// Database is done loading, pass it to the new_db_callback on the caller's
- // thread.
+ // thread. This would unblock resource loads.
callback_task_runner->PostTask(
FROM_HERE, base::Bind(new_db_callback, base::Passed(&v4_database)));
+
+ UMA_HISTOGRAM_TIMES("SafeBrowsing.V4DatabaseOpen.Time",
+ TimeTicks::Now() - create_start_time);
}
V4Database::V4Database(
@@ -81,11 +93,11 @@ V4Database::V4Database(
store_map_(std::move(store_map)),
pending_store_updates_(0) {
DCHECK(db_task_runner->RunsTasksOnCurrentThread());
- // TODO(vakh): Implement skeleton
}
// static
void V4Database::Destroy(std::unique_ptr<V4Database> v4_database) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
V4Database* v4_database_raw = v4_database.release();
if (v4_database_raw) {
v4_database_raw->db_task_runner_->DeleteSoon(FROM_HERE, v4_database_raw);
@@ -142,7 +154,9 @@ void V4Database::UpdatedStoreReady(ListIdentifier identifier,
DCHECK_CURRENTLY_ON(BrowserThread::IO);
DCHECK(pending_store_updates_);
if (new_store) {
- (*store_map_)[identifier] = std::move(new_store);
+ (*store_map_)[identifier].swap(new_store);
+ // |new_store| now is the store that needs to be destroyed on task runner.
+ V4Store::Destroy(std::move(new_store));
}
pending_store_updates_--;
@@ -152,17 +166,6 @@ void V4Database::UpdatedStoreReady(ListIdentifier identifier,
}
}
-bool V4Database::ResetDatabase() {
- DCHECK(db_task_runner_->RunsTasksOnCurrentThread());
- bool reset_success = true;
- for (const auto& store_map_iter : *store_map_) {
- if (!store_map_iter.second->Reset()) {
- reset_success = false;
- }
- }
- return reset_success;
-}
-
std::unique_ptr<StoreStateMap> V4Database::GetStoreStateMap() {
std::unique_ptr<StoreStateMap> store_state_map =
base::MakeUnique<StoreStateMap>();
@@ -172,6 +175,19 @@ std::unique_ptr<StoreStateMap> V4Database::GetStoreStateMap() {
return store_state_map;
}
+bool V4Database::AreStoresAvailable(
+ const StoresToCheck& stores_to_check) const {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+ for (const ListIdentifier& identifier : stores_to_check) {
+ const auto& store_pair = store_map_->find(identifier);
+ if (store_pair == store_map_->end())
+ return false; // Store not in our list
+ if (!store_pair->second->HasValidData())
+ return false; // Store never properly populated.
+ }
+ return true;
+}
+
void V4Database::GetStoresMatchingFullHash(
const FullHash& full_hash,
const StoresToCheck& stores_to_check,
@@ -189,6 +205,51 @@ void V4Database::GetStoresMatchingFullHash(
}
}
+void V4Database::ResetStores(
+ const std::vector<ListIdentifier>& stores_to_reset) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+ for (const ListIdentifier& identifier : stores_to_reset) {
+ store_map_->at(identifier)->Reset();
+ }
+}
+
+void V4Database::VerifyChecksum(
+ DatabaseReadyForUpdatesCallback db_ready_for_updates_callback) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+ // TODO(vakh): Consider using PostTaskAndReply instead.
+ const scoped_refptr<base::SingleThreadTaskRunner> callback_task_runner =
+ base::ThreadTaskRunnerHandle::Get();
+ db_task_runner_->PostTask(
+ FROM_HERE, base::Bind(&V4Database::VerifyChecksumOnTaskRunner,
+ base::Unretained(this), callback_task_runner,
+ db_ready_for_updates_callback));
+}
+
+void V4Database::VerifyChecksumOnTaskRunner(
+ const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner,
+ DatabaseReadyForUpdatesCallback db_ready_for_updates_callback) {
+ std::vector<ListIdentifier> stores_to_reset;
+ for (const auto& store_map_iter : *store_map_) {
+ if (!store_map_iter.second->VerifyChecksum()) {
+ stores_to_reset.push_back(store_map_iter.first);
+ }
+ }
+
+ callback_task_runner->PostTask(
+ FROM_HERE, base::Bind(db_ready_for_updates_callback, stores_to_reset));
+}
+
+void V4Database::RecordFileSizeHistograms() {
+ int64_t db_size = 0;
+ for (const auto& store_map_iter : *store_map_) {
+ const int64_t size =
+ store_map_iter.second->RecordAndReturnFileSize(kV4DatabaseSizeMetric);
+ db_size += size;
+ }
+ const int64_t db_size_kilobytes = static_cast<int64_t>(db_size / 1024);
+ UMA_HISTOGRAM_COUNTS(kV4DatabaseSizeMetric, db_size_kilobytes);
+}
+
ListInfo::ListInfo(const bool fetch_updates,
const std::string& filename,
const ListIdentifier& list_id,
diff --git a/chromium/components/safe_browsing_db/v4_database.h b/chromium/components/safe_browsing_db/v4_database.h
index bddba598850..19fee93a53b 100644
--- a/chromium/components/safe_browsing_db/v4_database.h
+++ b/chromium/components/safe_browsing_db/v4_database.h
@@ -17,9 +17,17 @@ namespace safe_browsing {
class V4Database;
+// Scheduled when the database has been read from disk and is ready to process
+// resource reputation requests.
typedef base::Callback<void(std::unique_ptr<V4Database>)>
NewDatabaseReadyCallback;
+// Scheduled when the checksum for all the stores in the database has been
+// verified to match the expected value. Stores for which the checksum did not
+// match are passed as the argument and need to be reset.
+typedef base::Callback<void(const std::vector<ListIdentifier>&)>
+ DatabaseReadyForUpdatesCallback;
+
// This callback is scheduled once the database has finished processing the
// update requests for all stores and is ready to process the next set of update
// requests.
@@ -108,6 +116,13 @@ class V4Database {
// Returns the current state of each of the stores being managed.
std::unique_ptr<StoreStateMap> GetStoreStateMap();
+ // Check if all the selected stores are available and populated.
+ // Returns false if any of |stores_to_check| don't have valid data.
+ // A store may be unavailble if either it hasn't yet gotten a proper
+ // full-update (just after install, or corrupted/missing file), or if it's
+ // not supported in this build (i.e. Chromium).
+ virtual bool AreStoresAvailable(const StoresToCheck& stores_to_check) const;
+
// Searches for a hash prefix matching the |full_hash| in stores in the
// database, filtered by |stores_to_check|, and returns the identifier of the
// store along with the matching hash prefix in |matched_hash_prefix_map|.
@@ -116,8 +131,21 @@ class V4Database {
const StoresToCheck& stores_to_check,
StoreAndHashPrefixes* matched_store_and_full_hashes);
- // Deletes the current database and creates a new one.
- virtual bool ResetDatabase();
+ // Resets the stores in |stores_to_reset| to an empty state. This is done if
+ // the checksum doesn't match the expected value.
+ void ResetStores(const std::vector<ListIdentifier>& stores_to_reset);
+
+ // Schedules verification of the checksum of each store read from disk on task
+ // runner. If the checksum doesn't match, that store is passed to the
+ // |db_ready_for_updates_callback|. At the end,
+ // |db_ready_for_updates_callback| is scheduled (on the same thread as it was
+ // called) to indicate that the database updates can now be scheduled.
+ void VerifyChecksum(
+ DatabaseReadyForUpdatesCallback db_ready_for_updates_callback);
+
+ // Records the size of each of the stores managed by this database, along
+ // with the combined size of all the stores.
+ void RecordFileSizeHistograms();
protected:
V4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
@@ -147,7 +175,8 @@ class V4Database {
const base::FilePath& base_path,
const ListInfos& list_infos,
const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner,
- NewDatabaseReadyCallback callback);
+ NewDatabaseReadyCallback callback,
+ const base::TimeTicks create_start_time);
// Callback called when a new store has been created and is ready to be used.
// This method updates the store_map_ to point to the new store, which causes
@@ -155,6 +184,11 @@ class V4Database {
void UpdatedStoreReady(ListIdentifier identifier,
std::unique_ptr<V4Store> store);
+ // See |VerifyChecksum|.
+ void VerifyChecksumOnTaskRunner(
+ const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner,
+ DatabaseReadyForUpdatesCallback db_ready_for_updates_callback);
+
const scoped_refptr<base::SequencedTaskRunner> db_task_runner_;
// Map of ListIdentifier to the V4Store.
diff --git a/chromium/components/safe_browsing_db/v4_database_unittest.cc b/chromium/components/safe_browsing_db/v4_database_unittest.cc
index 2ce05ee3ec1..a9133d74f2b 100644
--- a/chromium/components/safe_browsing_db/v4_database_unittest.cc
+++ b/chromium/components/safe_browsing_db/v4_database_unittest.cc
@@ -19,50 +19,42 @@ class FakeV4Store : public V4Store {
public:
FakeV4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
const base::FilePath& store_path,
- const bool reset_succeeds,
const bool hash_prefix_matches)
: V4Store(
task_runner,
base::FilePath(store_path.value() + FILE_PATH_LITERAL(".fake"))),
- hash_prefix_should_match_(hash_prefix_matches),
- reset_succeeds_(reset_succeeds) {}
-
- bool Reset() override { return reset_succeeds_; }
+ hash_prefix_should_match_(hash_prefix_matches) {}
HashPrefix GetMatchingHashPrefix(const FullHash& full_hash) override {
return hash_prefix_should_match_ ? full_hash : HashPrefix();
}
+ bool HasValidData() const override { return true; }
+
void set_hash_prefix_matches(bool hash_prefix_matches) {
hash_prefix_should_match_ = hash_prefix_matches;
}
private:
bool hash_prefix_should_match_;
- bool reset_succeeds_;
};
-// This factory creates a "fake" store. It allows the caller to specify that the
-// first store should fail on Reset() i.e. return false. All subsequent stores
-// always return true. This is used to test the Reset() method in V4Database.
+// This factory creates a "fake" store. It allows the caller to specify whether
+// the store has a hash prefix matching a full hash. This is used to test the
+// |GetStoresMatchingFullHash()| method in |V4Database|.
class FakeV4StoreFactory : public V4StoreFactory {
public:
- FakeV4StoreFactory(bool next_store_reset_fails, bool hash_prefix_matches)
- : hash_prefix_should_match_(hash_prefix_matches),
- next_store_reset_fails_(next_store_reset_fails) {}
+ FakeV4StoreFactory(bool hash_prefix_matches)
+ : hash_prefix_should_match_(hash_prefix_matches) {}
V4Store* CreateV4Store(
const scoped_refptr<base::SequencedTaskRunner>& task_runner,
const base::FilePath& store_path) override {
- bool reset_succeeds = !next_store_reset_fails_;
- next_store_reset_fails_ = false;
- return new FakeV4Store(task_runner, store_path, reset_succeeds,
- hash_prefix_should_match_);
+ return new FakeV4Store(task_runner, store_path, hash_prefix_should_match_);
}
private:
bool hash_prefix_should_match_;
- bool next_store_reset_fails_;
};
class V4DatabaseTest : public PlatformTest {
@@ -81,6 +73,7 @@ class V4DatabaseTest : public PlatformTest {
created_but_not_called_back_ = false;
created_and_called_back_ = false;
+ verify_checksum_called_back_ = false;
callback_db_updated_ =
base::Bind(&V4DatabaseTest::DatabaseUpdated, base::Unretained(this));
@@ -97,10 +90,8 @@ class V4DatabaseTest : public PlatformTest {
PlatformTest::TearDown();
}
- void RegisterFactory(bool fails_first_reset,
- bool hash_prefix_matches = true) {
- factory_.reset(
- new FakeV4StoreFactory(fails_first_reset, hash_prefix_matches));
+ void RegisterFactory(bool hash_prefix_matches = true) {
+ factory_.reset(new FakeV4StoreFactory(hash_prefix_matches));
V4Database::RegisterStoreFactoryForTest(factory_.get());
}
@@ -137,8 +128,6 @@ class V4DatabaseTest : public PlatformTest {
EXPECT_EQ(expected_store_path, store->store_path());
}
- EXPECT_EQ(expected_resets_successfully_, v4_database->ResetDatabase());
-
EXPECT_FALSE(created_and_called_back_);
created_and_called_back_ = true;
@@ -195,6 +184,16 @@ class V4DatabaseTest : public PlatformTest {
}
}
+ void VerifyChecksumCallback(const std::vector<ListIdentifier>& stores) {
+ EXPECT_FALSE(verify_checksum_called_back_);
+ verify_checksum_called_back_ = true;
+ }
+
+ void WaitForTasksOnTaskRunner() {
+ task_runner_->RunPendingTasks();
+ base::RunLoop().RunUntilIdle();
+ }
+
scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
std::unique_ptr<V4Database> v4_database_;
base::FilePath database_dirname_;
@@ -202,10 +201,10 @@ class V4DatabaseTest : public PlatformTest {
content::TestBrowserThreadBundle thread_bundle_;
bool created_but_not_called_back_;
bool created_and_called_back_;
+ bool verify_checksum_called_back_;
ListInfos list_infos_;
std::vector<ListIdentifier> expected_identifiers_;
std::vector<base::FilePath> expected_store_paths_;
- bool expected_resets_successfully_;
std::unique_ptr<FakeV4StoreFactory> factory_;
DatabaseUpdatedCallback callback_db_updated_;
NewDatabaseReadyCallback callback_db_ready_;
@@ -216,42 +215,23 @@ class V4DatabaseTest : public PlatformTest {
// Test to set up the database with fake stores.
TEST_F(V4DatabaseTest, TestSetupDatabaseWithFakeStores) {
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_);
+ RegisterFactory();
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
-
- base::RunLoop().RunUntilIdle();
- EXPECT_EQ(true, created_and_called_back_);
-}
-
-// Test to set up the database with fake stores that fail to reset.
-TEST_F(V4DatabaseTest, TestSetupDatabaseWithFakeStoresFailsReset) {
- expected_resets_successfully_ = false;
- RegisterFactory(!expected_resets_successfully_);
-
- V4Database::Create(task_runner_, database_dirname_, list_infos_,
- callback_db_ready_);
- created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
-
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
EXPECT_EQ(true, created_and_called_back_);
}
// Test to check database updates as expected.
TEST_F(V4DatabaseTest, TestApplyUpdateWithNewStates) {
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_);
+ RegisterFactory();
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
// The database has now been created. Time to try to update it.
EXPECT_TRUE(v4_database_);
@@ -267,22 +247,23 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithNewStates) {
CreateFakeServerResponse(expected_store_state_map_, true),
callback_db_updated_);
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ // Wait for the ApplyUpdate callback to get called.
+ WaitForTasksOnTaskRunner();
VerifyExpectedStoresState(true);
+
+ // Wait for the old stores to get destroyed on task runner.
+ WaitForTasksOnTaskRunner();
}
// Test to ensure no state updates leads to no store updates.
TEST_F(V4DatabaseTest, TestApplyUpdateWithNoNewState) {
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_);
+ RegisterFactory();
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
// The database has now been created. Time to try to update it.
EXPECT_TRUE(v4_database_);
@@ -298,22 +279,19 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithNoNewState) {
CreateFakeServerResponse(expected_store_state_map_, true),
callback_db_updated_);
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
VerifyExpectedStoresState(false);
}
// Test to ensure no updates leads to no store updates.
TEST_F(V4DatabaseTest, TestApplyUpdateWithEmptyUpdate) {
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_);
+ RegisterFactory();
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
// The database has now been created. Time to try to update it.
EXPECT_TRUE(v4_database_);
@@ -330,22 +308,19 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithEmptyUpdate) {
v4_database_->ApplyUpdate(std::move(parsed_server_response),
callback_db_updated_);
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
VerifyExpectedStoresState(false);
}
// Test to ensure invalid update leads to no store changes.
TEST_F(V4DatabaseTest, TestApplyUpdateWithInvalidUpdate) {
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_);
+ RegisterFactory();
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
// The database has now been created. Time to try to update it.
EXPECT_TRUE(v4_database_);
@@ -360,8 +335,7 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithInvalidUpdate) {
v4_database_->ApplyUpdate(
CreateFakeServerResponse(expected_store_state_map_, false),
callback_db_updated_);
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
VerifyExpectedStoresState(false);
}
@@ -369,15 +343,12 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithInvalidUpdate) {
// Test to ensure the case that all stores match a given full hash.
TEST_F(V4DatabaseTest, TestAllStoresMatchFullHash) {
bool hash_prefix_matches = true;
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_, hash_prefix_matches);
+ RegisterFactory(hash_prefix_matches);
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
-
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
EXPECT_EQ(true, created_and_called_back_);
StoresToCheck stores_to_check({linux_malware_id_, win_malware_id_});
@@ -395,15 +366,12 @@ TEST_F(V4DatabaseTest, TestAllStoresMatchFullHash) {
// Test to ensure the case that no stores match a given full hash.
TEST_F(V4DatabaseTest, TestNoStoreMatchesFullHash) {
bool hash_prefix_matches = false;
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_, hash_prefix_matches);
+ RegisterFactory(hash_prefix_matches);
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
-
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
EXPECT_EQ(true, created_and_called_back_);
StoreAndHashPrefixes store_and_hash_prefixes;
@@ -417,15 +385,12 @@ TEST_F(V4DatabaseTest, TestNoStoreMatchesFullHash) {
TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHash) {
// Setup stores to not match the full hash.
bool hash_prefix_matches = false;
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_, hash_prefix_matches);
+ RegisterFactory(hash_prefix_matches);
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
-
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
EXPECT_EQ(true, created_and_called_back_);
// Set the store corresponding to linux_malware_id_ to match the full hash.
@@ -447,15 +412,12 @@ TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHash) {
TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHashBecauseOfStoresToMatch) {
// Setup all stores to match the full hash.
bool hash_prefix_matches = true;
- expected_resets_successfully_ = true;
- RegisterFactory(!expected_resets_successfully_, hash_prefix_matches);
+ RegisterFactory(hash_prefix_matches);
V4Database::Create(task_runner_, database_dirname_, list_infos_,
callback_db_ready_);
created_but_not_called_back_ = true;
- task_runner_->RunPendingTasks();
-
- base::RunLoop().RunUntilIdle();
+ WaitForTasksOnTaskRunner();
EXPECT_EQ(true, created_and_called_back_);
// Don't add win_malware_id_ to the StoresToCheck.
@@ -467,4 +429,51 @@ TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHashBecauseOfStoresToMatch) {
EXPECT_FALSE(store_and_hash_prefixes.begin()->hash_prefix.empty());
}
+TEST_F(V4DatabaseTest, VerifyChecksumCalledAsync) {
+ bool hash_prefix_matches = true;
+ RegisterFactory(hash_prefix_matches);
+
+ V4Database::Create(task_runner_, database_dirname_, list_infos_,
+ callback_db_ready_);
+ created_but_not_called_back_ = true;
+ WaitForTasksOnTaskRunner();
+ EXPECT_EQ(true, created_and_called_back_);
+
+ // verify_checksum_called_back_ set to false in the constructor.
+ EXPECT_FALSE(verify_checksum_called_back_);
+ // Now call VerifyChecksum and pass the callback that sets
+ // verify_checksum_called_back_ to true.
+ v4_database_->VerifyChecksum(base::Bind(
+ &V4DatabaseTest::VerifyChecksumCallback, base::Unretained(this)));
+ // verify_checksum_called_back_ should still be false since the checksum
+ // verification is async.
+ EXPECT_FALSE(verify_checksum_called_back_);
+ WaitForTasksOnTaskRunner();
+ EXPECT_TRUE(verify_checksum_called_back_);
+}
+
+// Test that we can properly check for unsupported stores
+TEST_F(V4DatabaseTest, TestStoresAvailable) {
+ bool hash_prefix_matches = false;
+ RegisterFactory(hash_prefix_matches);
+
+ V4Database::Create(task_runner_, database_dirname_, list_infos_,
+ callback_db_ready_);
+ created_but_not_called_back_ = true;
+ WaitForTasksOnTaskRunner();
+ EXPECT_EQ(true, created_and_called_back_);
+
+ // Doesn't exist in out list
+ const ListIdentifier bogus_id(LINUX_PLATFORM, CHROME_EXTENSION,
+ CSD_WHITELIST);
+
+ EXPECT_TRUE(v4_database_->AreStoresAvailable(
+ StoresToCheck({linux_malware_id_, win_malware_id_})));
+
+ EXPECT_FALSE(v4_database_->AreStoresAvailable(
+ StoresToCheck({linux_malware_id_, bogus_id})));
+
+ EXPECT_FALSE(v4_database_->AreStoresAvailable(StoresToCheck({bogus_id})));
+}
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_feature_list.cc b/chromium/components/safe_browsing_db/v4_feature_list.cc
index a8435edf051..323ad4db74f 100644
--- a/chromium/components/safe_browsing_db/v4_feature_list.cc
+++ b/chromium/components/safe_browsing_db/v4_feature_list.cc
@@ -14,17 +14,27 @@ const base::Feature kLocalDatabaseManagerEnabled{
"SafeBrowsingV4LocalDatabaseManagerEnabled",
base::FEATURE_DISABLED_BY_DEFAULT};
-const base::Feature kParallelCheckEnabled{"SafeBrowingV4ParallelCheckEnabled",
- base::FEATURE_DISABLED_BY_DEFAULT};
+const base::Feature kV4HybridEnabled{"SafeBrowsingV4HybridEnabled",
+ base::FEATURE_DISABLED_BY_DEFAULT};
+
+const base::Feature kV4OnlyEnabled{"SafeBrowsingV4OnlyEnabled",
+ base::FEATURE_DISABLED_BY_DEFAULT};
+
} // namespace
bool IsLocalDatabaseManagerEnabled() {
- return IsParallelCheckEnabled() ||
- base::FeatureList::IsEnabled(kLocalDatabaseManagerEnabled);
+ return base::FeatureList::IsEnabled(kLocalDatabaseManagerEnabled) ||
+ IsV4HybridEnabled() || IsV4OnlyEnabled();
+}
+
+bool IsV4HybridEnabled() {
+ return base::FeatureList::IsEnabled(kV4HybridEnabled);
}
-bool IsParallelCheckEnabled() {
- return base::FeatureList::IsEnabled(kParallelCheckEnabled);
+bool IsV4OnlyEnabled() {
+ // TODO(vakh): Enable this only when all the lists can be synced from the
+ // server. See http://b/33182208
+ return base::FeatureList::IsEnabled(kV4OnlyEnabled);
}
} // namespace V4FeatureList
diff --git a/chromium/components/safe_browsing_db/v4_feature_list.h b/chromium/components/safe_browsing_db/v4_feature_list.h
index b30554143a1..f549a9556ab 100644
--- a/chromium/components/safe_browsing_db/v4_feature_list.h
+++ b/chromium/components/safe_browsing_db/v4_feature_list.h
@@ -11,11 +11,18 @@ namespace safe_browsing {
// through Finch.
namespace V4FeatureList {
-// Is the Pver4 database manager enabled?
+// Is the PVer4 database manager enabled? Should be true if either of those
+// below are true.
bool IsLocalDatabaseManagerEnabled();
-// Is the Pver4 database manager doing resource checks in paralled with PVer3?
-bool IsParallelCheckEnabled();
+// Is the PVer4 database being checked for resource reputation? If this returns
+// true, use PVer4 database for CheckBrowseUrl, otherwise use PVer3.
+bool IsV4HybridEnabled();
+
+// Is only the PVer4 database being checked for resource reputation? If this
+// returns true, use PVer4 database for all SafeBrowsing operations, and don't
+// update the PVer3 database at all. This is the launch configuration.
+bool IsV4OnlyEnabled();
} // namespace V4FeatureList
diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc
index 96bafa951e3..f0dd8cb1db6 100644
--- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc
+++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc
@@ -10,6 +10,7 @@
#include "base/macros.h"
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram_macros.h"
+#include "base/stl_util.h"
#include "base/timer/timer.h"
#include "content/public/browser/browser_thread.h"
#include "net/base/load_flags.h"
@@ -27,7 +28,7 @@ namespace {
// Record a GetHash result.
void RecordGetHashResult(safe_browsing::V4OperationResult result) {
UMA_HISTOGRAM_ENUMERATION(
- "SafeBrowsing.GetV4HashResult", result,
+ "SafeBrowsing.V4GetHash.Result", result,
safe_browsing::V4OperationResult::OPERATION_RESULT_MAX);
}
@@ -67,7 +68,7 @@ enum ParseResultType {
// Record parsing errors of a GetHash result.
void RecordParseGetHashResult(ParseResultType result_type) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.ParseV4HashResult", result_type,
+ UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHash.Parse.Result", result_type,
PARSE_RESULT_TYPE_MAX);
}
@@ -90,8 +91,8 @@ enum V4FullHashCacheResultType {
// Record a full hash cache hit result.
void RecordV4FullHashCacheResult(V4FullHashCacheResultType result_type) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4FullHashCacheResult", result_type,
- FULL_HASH_CACHE_RESULT_MAX);
+ UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHash.CacheHit.Result",
+ result_type, FULL_HASH_CACHE_RESULT_MAX);
}
// Enumerate GetHash hits/misses for histogramming purposes. DO NOT CHANGE THE
@@ -113,7 +114,7 @@ enum V4GetHashCheckResultType {
// Record a GetHash hit result.
void RecordV4GetHashCheckResult(V4GetHashCheckResultType result_type) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHashCheckResult", result_type,
+ UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHash.Check.Result", result_type,
GET_HASH_CHECK_RESULT_MAX);
}
@@ -131,9 +132,6 @@ const char kPhishing[] = "PHISHING";
namespace safe_browsing {
-const char kUmaV4HashResponseMetricName[] =
- "SafeBrowsing.GetV4HashHttpResponseOrErrorCode";
-
// The default V4GetHashProtocolManagerFactory.
class V4GetHashProtocolManagerFactoryImpl
: public V4GetHashProtocolManagerFactory {
@@ -171,12 +169,14 @@ FullHashCallbackInfo::FullHashCallbackInfo(
std::unique_ptr<net::URLFetcher> fetcher,
const FullHashToStoreAndHashPrefixesMap&
full_hash_to_store_and_hash_prefixes,
- const FullHashCallback& callback)
+ const FullHashCallback& callback,
+ const base::Time& network_start_time)
: cached_full_hash_infos(cached_full_hash_infos),
callback(callback),
fetcher(std::move(fetcher)),
full_hash_to_store_and_hash_prefixes(
full_hash_to_store_and_hash_prefixes),
+ network_start_time(network_start_time),
prefixes_requested(prefixes_requested) {}
FullHashCallbackInfo::~FullHashCallbackInfo() {}
@@ -239,11 +239,18 @@ V4GetHashProtocolManager::V4GetHashProtocolManager(
url_fetcher_id_(0),
clock_(new base::DefaultClock()) {
DCHECK(!stores_to_check.empty());
+ std::set<PlatformType> platform_types;
+ std::set<ThreatEntryType> threat_entry_types;
+ std::set<ThreatType> threat_types;
for (const ListIdentifier& store : stores_to_check) {
- platform_types_.insert(store.platform_type());
- threat_entry_types_.insert(store.threat_entry_type());
- threat_types_.insert(store.threat_type());
+ platform_types.insert(store.platform_type());
+ threat_entry_types.insert(store.threat_entry_type());
+ threat_types.insert(store.threat_type());
}
+ platform_types_.assign(platform_types.begin(), platform_types.end());
+ threat_entry_types_.assign(threat_entry_types.begin(),
+ threat_entry_types.end());
+ threat_types_.assign(threat_types.begin(), threat_types.end());
}
V4GetHashProtocolManager::~V4GetHashProtocolManager() {}
@@ -298,7 +305,7 @@ void V4GetHashProtocolManager::GetFullHashes(
net::URLFetcher* fetcher = owned_fetcher.get();
pending_hash_requests_[fetcher].reset(new FullHashCallbackInfo(
cached_full_hash_infos, prefixes_to_request, std::move(owned_fetcher),
- full_hash_to_store_and_hash_prefixes, callback));
+ full_hash_to_store_and_hash_prefixes, callback, clock_->Now()));
fetcher->SetExtraRequestHeaders(headers.ToString());
fetcher->SetLoadFlags(net::LOAD_DISABLE_CACHE);
@@ -312,7 +319,7 @@ void V4GetHashProtocolManager::GetFullHashesWithApis(
DCHECK_CURRENTLY_ON(BrowserThread::IO);
DCHECK(url.SchemeIs(url::kHttpScheme) || url.SchemeIs(url::kHttpsScheme));
- std::unordered_set<FullHash> full_hashes;
+ std::vector<FullHash> full_hashes;
V4ProtocolManagerUtil::UrlToFullHashes(url.GetOrigin(), &full_hashes);
FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
@@ -470,12 +477,13 @@ void V4GetHashProtocolManager::HandleGetHashError(const Time& now) {
void V4GetHashProtocolManager::OnFullHashForApi(
const ThreatMetadataForApiCallback& api_callback,
- const std::unordered_set<FullHash>& full_hashes,
+ const std::vector<FullHash>& full_hashes,
const std::vector<FullHashInfo>& full_hash_infos) {
ThreatMetadata md;
for (const FullHashInfo& full_hash_info : full_hash_infos) {
DCHECK_EQ(GetChromeUrlApiId(), full_hash_info.list_id);
- DCHECK(full_hashes.find(full_hash_info.full_hash) != full_hashes.end());
+ DCHECK(std::find(full_hashes.begin(), full_hashes.end(),
+ full_hash_info.full_hash) != full_hashes.end());
md.api_permissions.insert(full_hash_info.metadata.api_permissions.begin(),
full_hash_info.metadata.api_permissions.end());
}
@@ -529,6 +537,15 @@ bool V4GetHashProtocolManager::ParseHashResponse(
ListIdentifier list_id(match.platform_type(), match.threat_entry_type(),
match.threat_type());
+ if (!base::ContainsValue(platform_types_, list_id.platform_type()) ||
+ !base::ContainsValue(threat_entry_types_,
+ list_id.threat_entry_type()) ||
+ !base::ContainsValue(threat_types_, list_id.threat_type())) {
+ // The server may send a ThreatMatch response for lists that we didn't ask
+ // for so ignore those ThreatMatch responses.
+ continue;
+ }
+
base::Time positive_expiry;
if (match.has_cache_duration()) {
// Seconds resolution is good enough so we ignore the nanos field.
@@ -539,35 +556,33 @@ bool V4GetHashProtocolManager::ParseHashResponse(
}
FullHashInfo full_hash_info(match.threat().hash(), list_id,
positive_expiry);
- if (!ParseMetadata(match, &full_hash_info.metadata)) {
- return false;
- }
-
+ ParseMetadata(match, &full_hash_info.metadata);
full_hash_infos->push_back(full_hash_info);
}
return true;
}
-bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match,
+// static
+void V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match,
ThreatMetadata* metadata) {
// Different threat types will handle the metadata differently.
if (match.threat_type() == API_ABUSE) {
if (!match.has_platform_type() ||
match.platform_type() != CHROME_PLATFORM) {
RecordParseGetHashResult(UNEXPECTED_PLATFORM_TYPE_ERROR);
- return false;
+ return;
}
if (!match.has_threat_entry_metadata()) {
RecordParseGetHashResult(NO_METADATA_ERROR);
- return false;
+ return;
}
// For API Abuse, store a list of the returned permissions.
for (const ThreatEntryMetadata::MetadataEntry& m :
match.threat_entry_metadata().entries()) {
if (m.key() != kPermission) {
RecordParseGetHashResult(UNEXPECTED_METADATA_VALUE_ERROR);
- return false;
+ return;
}
metadata->api_permissions.insert(m.value());
}
@@ -575,7 +590,6 @@ bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match,
match.threat_type() == POTENTIALLY_HARMFUL_APPLICATION) {
for (const ThreatEntryMetadata::MetadataEntry& m :
match.threat_entry_metadata().entries()) {
- // TODO: Need to confirm the below key/value pairs with CSD backend.
if (m.key() == kPhaPatternType || m.key() == kMalwarePatternType) {
if (m.value() == kLanding) {
metadata->threat_pattern_type = ThreatPatternType::MALWARE_LANDING;
@@ -586,7 +600,7 @@ bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match,
break;
} else {
RecordParseGetHashResult(UNEXPECTED_METADATA_VALUE_ERROR);
- return false;
+ return;
}
}
}
@@ -607,16 +621,14 @@ bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match,
break;
} else {
RecordParseGetHashResult(UNEXPECTED_METADATA_VALUE_ERROR);
- return false;
+ return;
}
}
}
- } else {
+ } else if (match.has_threat_entry_metadata() &&
+ match.threat_entry_metadata().entries_size() > 1) {
RecordParseGetHashResult(UNEXPECTED_THREAT_TYPE_ERROR);
- return false;
}
-
- return true;
}
void V4GetHashProtocolManager::ResetGetHashErrors() {
@@ -698,7 +710,7 @@ void V4GetHashProtocolManager::OnURLFetchComplete(
int response_code = source->GetResponseCode();
net::URLRequestStatus status = source->GetStatus();
V4ProtocolManagerUtil::RecordHttpResponseOrErrorCode(
- kUmaV4HashResponseMetricName, status, response_code);
+ "SafeBrowsing.V4GetHash.Network.Result", status, response_code);
std::vector<FullHashInfo> full_hash_infos;
Time negative_cache_expire;
@@ -726,6 +738,8 @@ void V4GetHashProtocolManager::OnURLFetchComplete(
}
const std::unique_ptr<FullHashCallbackInfo>& fhci = it->second;
+ UMA_HISTOGRAM_LONG_TIMES("SafeBrowsing.V4GetHash.Network.Time",
+ clock_->Now() - fhci->network_start_time);
UpdateCache(fhci->prefixes_requested, full_hash_infos, negative_cache_expire);
MergeResults(fhci->full_hash_to_store_and_hash_prefixes, full_hash_infos,
&fhci->cached_full_hash_infos);
diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h
index 720e09684cb..ffd4ea89056 100644
--- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h
+++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h
@@ -106,7 +106,8 @@ struct FullHashCallbackInfo {
std::unique_ptr<net::URLFetcher> fetcher,
const FullHashToStoreAndHashPrefixesMap&
full_hash_to_store_and_hash_prefixes,
- const FullHashCallback& callback);
+ const FullHashCallback& callback,
+ const base::Time& network_start_time);
~FullHashCallbackInfo();
// The FullHashInfo objects retrieved from cache. These are merged with the
@@ -125,6 +126,10 @@ struct FullHashCallbackInfo {
// which to look for a full hash match.
FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
+ // Used to measure how long did it take to fetch the full hash response from
+ // the server.
+ base::Time network_start_time;
+
// The prefixes that were requested from the server.
std::vector<HashPrefix> prefixes_requested;
};
@@ -254,7 +259,7 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate,
// permission API metadata for full hashes in those |full_hash_infos| that
// have a full hash in |full_hashes|.
void OnFullHashForApi(const ThreatMetadataForApiCallback& api_callback,
- const std::unordered_set<FullHash>& full_hashes,
+ const std::vector<FullHash>& full_hashes,
const std::vector<FullHashInfo>& full_hash_infos);
// Parses a FindFullHashesResponse protocol buffer and fills the results in
@@ -266,10 +271,10 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate,
std::vector<FullHashInfo>* full_hash_infos,
base::Time* negative_cache_expire);
- // Parses the store specific |metadata| information from |match|. Returns
- // true if the metadata information was parsed correctly and was consistent
- // with what's expected from that corresponding store; false otherwise.
- bool ParseMetadata(const ThreatMatch& match, ThreatMetadata* metadata);
+ // Parses the store specific |metadata| information from |match|. Logs errors
+ // to UMA if the metadata information was not parsed correctly or was
+ // inconsistent with what's expected from that corresponding store.
+ static void ParseMetadata(const ThreatMatch& match, ThreatMetadata* metadata);
// Resets the gethash error counter and multiplier.
void ResetGetHashErrors();
@@ -330,9 +335,9 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate,
// The following sets represent the combination of lists that we would always
// request from the server, irrespective of which list we found the hash
// prefix match in.
- std::unordered_set<PlatformType> platform_types_;
- std::unordered_set<ThreatEntryType> threat_entry_types_;
- std::unordered_set<ThreatType> threat_types_;
+ std::vector<PlatformType> platform_types_;
+ std::vector<ThreatEntryType> threat_entry_types_;
+ std::vector<ThreatType> threat_types_;
DISALLOW_COPY_AND_ASSIGN(V4GetHashProtocolManager);
};
diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc
index fa37e0622f3..85a42c16c65 100644
--- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc
+++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc
@@ -14,8 +14,8 @@
#include "base/test/simple_test_clock.h"
#include "base/time/time.h"
#include "components/safe_browsing_db/safebrowsing.pb.h"
-#include "components/safe_browsing_db/testing_util.h"
#include "components/safe_browsing_db/util.h"
+#include "components/safe_browsing_db/v4_test_util.h"
#include "content/public/test/test_browser_thread_bundle.h"
#include "net/base/escape.h"
#include "net/base/load_flags.h"
@@ -26,14 +26,6 @@
using base::Time;
using base::TimeDelta;
-namespace {
-
-const char kClient[] = "unittest";
-const char kAppVer[] = "1.0";
-const char kKeyParam[] = "test_key_param";
-
-} // namespace
-
namespace safe_browsing {
namespace {
@@ -76,12 +68,13 @@ class V4GetHashProtocolManagerTest : public PlatformTest {
}
std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager() {
- V4ProtocolConfig config;
- config.client_name = kClient;
- config.version = kAppVer;
- config.key_param = kKeyParam;
- StoresToCheck stores_to_check({GetUrlMalwareId(), GetChromeUrlApiId()});
- return V4GetHashProtocolManager::Create(NULL, stores_to_check, config);
+ StoresToCheck stores_to_check(
+ {GetUrlMalwareId(), GetChromeUrlApiId(),
+ ListIdentifier(CHROME_PLATFORM, URL, SOCIAL_ENGINEERING_PUBLIC),
+ ListIdentifier(CHROME_PLATFORM, URL,
+ POTENTIALLY_HARMFUL_APPLICATION)});
+ return V4GetHashProtocolManager::Create(NULL, stores_to_check,
+ GetTestV4ProtocolConfig());
}
static void SetupFetcherToReturnOKResponse(
@@ -285,13 +278,18 @@ TEST_F(V4GetHashProtocolManagerTest,
TEST_F(V4GetHashProtocolManagerTest, TestGetHashRequest) {
FindFullHashesRequest req;
ThreatInfo* info = req.mutable_threat_info();
- info->add_platform_types(GetCurrentPlatformType());
- info->add_platform_types(CHROME_PLATFORM);
+ for (const PlatformType& p :
+ std::set<PlatformType>{GetCurrentPlatformType(), CHROME_PLATFORM}) {
+ info->add_platform_types(p);
+ }
info->add_threat_entry_types(URL);
- info->add_threat_types(MALWARE_THREAT);
- info->add_threat_types(API_ABUSE);
+ for (const ThreatType& tt :
+ std::set<ThreatType>{MALWARE_THREAT, SOCIAL_ENGINEERING_PUBLIC,
+ POTENTIALLY_HARMFUL_APPLICATION, API_ABUSE}) {
+ info->add_threat_types(tt);
+ }
HashPrefix one = "hashone";
HashPrefix two = "hashtwo";
@@ -331,6 +329,14 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashResponse) {
m->mutable_threat_entry_metadata()->add_entries();
e->set_key("permission");
e->set_value("NOTIFICATIONS");
+ // Add another ThreatMatch for a list we don't track. This response should
+ // get dropped.
+ m = res.add_matches();
+ m->set_threat_type(THREAT_TYPE_UNSPECIFIED);
+ m->set_platform_type(CHROME_PLATFORM);
+ m->set_threat_entry_type(URL);
+ m->mutable_cache_duration()->set_seconds(300);
+ m->mutable_threat()->set_hash(full_hash);
// Serialize.
std::string res_data;
@@ -341,6 +347,8 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashResponse) {
EXPECT_TRUE(pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
+ // Even though the server responded with two ThreatMatch responses, one
+ // should have been dropped.
ASSERT_EQ(1ul, full_hash_infos.size());
const FullHashInfo& fhi = full_hash_infos[0];
EXPECT_EQ(full_hash, fhi.full_hash);
@@ -408,6 +416,8 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashThreatPatternType) {
pm->ParseHashResponse(se_data, &full_hash_infos, &cache_expire));
EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
+ // Ensure that the threat remains valid since we found a full hash match,
+ // even though the metadata information could not be parsed correctly.
ASSERT_EQ(1ul, full_hash_infos.size());
const FullHashInfo& fhi = full_hash_infos[0];
EXPECT_EQ(full_hash, fhi.full_hash);
@@ -470,9 +480,18 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashThreatPatternType) {
invalid_res.SerializeToString(&invalid_data);
std::vector<FullHashInfo> full_hash_infos;
base::Time cache_expire;
- EXPECT_FALSE(
+ EXPECT_TRUE(
pm->ParseHashResponse(invalid_data, &full_hash_infos, &cache_expire));
- EXPECT_EQ(0ul, full_hash_infos.size());
+
+ // Ensure that the threat remains valid since we found a full hash match,
+ // even though the metadata information could not be parsed correctly.
+ ASSERT_EQ(1ul, full_hash_infos.size());
+ const auto& fhi = full_hash_infos[0];
+ EXPECT_EQ(full_hash, fhi.full_hash);
+ EXPECT_EQ(
+ ListIdentifier(CHROME_PLATFORM, URL, POTENTIALLY_HARMFUL_APPLICATION),
+ fhi.list_id);
+ EXPECT_EQ(ThreatPatternType::NONE, fhi.metadata.threat_pattern_type);
}
}
@@ -484,13 +503,14 @@ TEST_F(V4GetHashProtocolManagerTest,
base::Time now = base::Time::UnixEpoch();
SetTestClock(now, pm.get());
+ FullHash full_hash("Not to fret.");
FindFullHashesResponse res;
res.mutable_negative_cache_duration()->set_seconds(600);
ThreatMatch* m = res.add_matches();
m->set_threat_type(API_ABUSE);
m->set_platform_type(CHROME_PLATFORM);
m->set_threat_entry_type(URL);
- m->mutable_threat()->set_hash(FullHash("Not to fret."));
+ m->mutable_threat()->set_hash(full_hash);
ThreatEntryMetadata::MetadataEntry* e =
m->mutable_threat_entry_metadata()->add_entries();
e->set_key("notpermission");
@@ -502,11 +522,14 @@ TEST_F(V4GetHashProtocolManagerTest,
std::vector<FullHashInfo> full_hash_infos;
base::Time cache_expire;
- EXPECT_FALSE(
- pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
+ EXPECT_TRUE(pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
- EXPECT_EQ(0ul, full_hash_infos.size());
+ ASSERT_EQ(1ul, full_hash_infos.size());
+ const auto& fhi = full_hash_infos[0];
+ EXPECT_EQ(full_hash, fhi.full_hash);
+ EXPECT_EQ(GetChromeUrlApiId(), fhi.list_id);
+ EXPECT_TRUE(fhi.metadata.api_permissions.empty());
}
TEST_F(V4GetHashProtocolManagerTest,
diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager.cc b/chromium/components/safe_browsing_db/v4_local_database_manager.cc
index 1a81f7fcda2..ffef84ef025 100644
--- a/chromium/components/safe_browsing_db/v4_local_database_manager.cc
+++ b/chromium/components/safe_browsing_db/v4_local_database_manager.cc
@@ -13,10 +13,14 @@
#include "base/callback.h"
#include "base/memory/ptr_util.h"
#include "base/memory/ref_counted.h"
+#include "base/metrics/histogram_macros.h"
#include "components/safe_browsing_db/v4_feature_list.h"
+#include "components/safe_browsing_db/v4_protocol_manager_util.h"
#include "content/public/browser/browser_thread.h"
+#include "crypto/sha2.h"
using content::BrowserThread;
+using base::TimeTicks;
namespace safe_browsing {
@@ -26,12 +30,50 @@ const ThreatSeverity kLeastSeverity =
std::numeric_limits<ThreatSeverity>::max();
ListInfos GetListInfos() {
- return ListInfos(
- {ListInfo(true, "UrlMalware.store", GetUrlMalwareId(),
- SB_THREAT_TYPE_URL_MALWARE),
- ListInfo(true, "UrlSoceng.store", GetUrlSocEngId(),
- SB_THREAT_TYPE_URL_PHISHING),
- ListInfo(false, "", GetChromeUrlApiId(), SB_THREAT_TYPE_API_ABUSE)});
+ // NOTE(vakh): When adding a store here, add the corresponding store-specific
+ // histograms also.
+ // NOTE(vakh): Delete file "AnyIpMalware.store". It has been renamed to
+ // "IpMalware.store". If it exists, it should be 75 bytes long.
+ // The first argument to ListInfo specifies whether to sync hash prefixes for
+ // that list. This can be false for two reasons:
+ // - The server doesn't support that list yet. Once the server adds support
+ // for it, it can be changed to true.
+ // - The list doesn't have hash prefixes to match. All requests lead to full
+ // hash checks. For instance: GetChromeUrlApiId()
+
+#if defined(GOOGLE_CHROME_BUILD)
+ const bool kSyncOnlyOnChromeBuilds = true;
+#else
+ const bool kSyncOnlyOnChromeBuilds = false;
+#endif
+ const bool kSyncAlways = true;
+ const bool kSyncNever = false;
+ return ListInfos({
+ ListInfo(kSyncOnlyOnChromeBuilds, "CertCsdDownloadWhitelist.store",
+ GetCertCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED),
+ ListInfo(kSyncOnlyOnChromeBuilds, "ChromeFilenameClientIncident.store",
+ GetChromeFilenameClientIncidentId(), SB_THREAT_TYPE_UNUSED),
+ ListInfo(kSyncAlways, "IpMalware.store", GetIpMalwareId(),
+ SB_THREAT_TYPE_UNUSED),
+ ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdDownloadWhitelist.store",
+ GetUrlCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED),
+ ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdWhitelist.store",
+ GetUrlCsdWhitelistId(), SB_THREAT_TYPE_UNUSED),
+ ListInfo(kSyncAlways, "UrlSoceng.store", GetUrlSocEngId(),
+ SB_THREAT_TYPE_URL_PHISHING),
+ ListInfo(kSyncAlways, "UrlMalware.store", GetUrlMalwareId(),
+ SB_THREAT_TYPE_URL_MALWARE),
+ ListInfo(kSyncAlways, "UrlUws.store", GetUrlUwsId(),
+ SB_THREAT_TYPE_URL_UNWANTED),
+ ListInfo(kSyncAlways, "UrlMalBin.store", GetUrlMalBinId(),
+ SB_THREAT_TYPE_BINARY_MALWARE_URL),
+ ListInfo(kSyncAlways, "ChromeExtMalware.store", GetChromeExtMalwareId(),
+ SB_THREAT_TYPE_EXTENSION),
+ ListInfo(kSyncOnlyOnChromeBuilds, "ChromeUrlClientIncident.store",
+ GetChromeUrlClientIncidentId(),
+ SB_THREAT_TYPE_BLACKLISTED_RESOURCE),
+ ListInfo(kSyncNever, "", GetChromeUrlApiId(), SB_THREAT_TYPE_API_ABUSE),
+ });
}
// Returns the severity information about a given SafeBrowsing list. The lowest
@@ -40,11 +82,15 @@ ThreatSeverity GetThreatSeverity(const ListIdentifier& list_id) {
switch (list_id.threat_type()) {
case MALWARE_THREAT:
case SOCIAL_ENGINEERING_PUBLIC:
+ case MALICIOUS_BINARY:
return 0;
- case API_ABUSE:
+ case UNWANTED_SOFTWARE:
return 1;
+ case API_ABUSE:
+ return 2;
default:
- NOTREACHED() << "Unexpected ThreatType encountered in GetThreatSeverity";
+ NOTREACHED() << "Unexpected ThreatType encountered: "
+ << list_id.threat_type();
return kLeastSeverity;
}
}
@@ -55,13 +101,27 @@ V4LocalDatabaseManager::PendingCheck::PendingCheck(
Client* client,
ClientCallbackType client_callback_type,
const StoresToCheck& stores_to_check,
- const GURL& url)
+ const std::vector<GURL>& urls)
: client(client),
client_callback_type(client_callback_type),
result_threat_type(SB_THREAT_TYPE_SAFE),
stores_to_check(stores_to_check),
- url(url) {
- DCHECK_GT(ClientCallbackType::CHECK_MAX, client_callback_type);
+ urls(urls) {
+ for (const auto& url : urls) {
+ V4ProtocolManagerUtil::UrlToFullHashes(url, &full_hashes);
+ }
+}
+
+V4LocalDatabaseManager::PendingCheck::PendingCheck(
+ Client* client,
+ ClientCallbackType client_callback_type,
+ const StoresToCheck& stores_to_check,
+ const std::set<FullHash>& full_hashes_set)
+ : client(client),
+ client_callback_type(client_callback_type),
+ result_threat_type(SB_THREAT_TYPE_SAFE),
+ stores_to_check(stores_to_check) {
+ full_hashes.assign(full_hashes_set.begin(), full_hashes_set.end());
}
V4LocalDatabaseManager::PendingCheck::~PendingCheck() {}
@@ -100,9 +160,11 @@ void V4LocalDatabaseManager::CancelCheck(Client* client) {
DCHECK_CURRENTLY_ON(BrowserThread::IO);
DCHECK(enabled_);
- auto it = pending_clients_.find(client);
- if (it != pending_clients_.end()) {
- pending_clients_.erase(it);
+ auto pending_it = std::find_if(
+ std::begin(pending_checks_), std::end(pending_checks_),
+ [client](const PendingCheck* check) { return check->client == client; });
+ if (pending_it != pending_checks_.end()) {
+ pending_checks_.erase(pending_it);
}
auto queued_it =
@@ -139,73 +201,137 @@ bool V4LocalDatabaseManager::CheckBrowseUrl(const GURL& url, Client* client) {
std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
client, ClientCallbackType::CHECK_BROWSE_URL,
- StoresToCheck({GetUrlMalwareId(), GetUrlSocEngId()}), url);
- if (!v4_database_) {
- queued_checks_.push_back(std::move(check));
- return false;
- }
+ StoresToCheck({GetUrlMalwareId(), GetUrlSocEngId(), GetUrlUwsId()}),
+ std::vector<GURL>(1, url));
- FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
- if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) {
- return true;
- }
-
- PerformFullHashCheck(std::move(check), full_hash_to_store_and_hash_prefixes);
- return false;
+ return HandleCheck(std::move(check));
}
bool V4LocalDatabaseManager::CheckDownloadUrl(
const std::vector<GURL>& url_chain,
Client* client) {
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- // TODO(vakh): Implement this skeleton.
- return true;
+
+ if (!enabled_ || url_chain.empty()) {
+ return true;
+ }
+
+ std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
+ client, ClientCallbackType::CHECK_DOWNLOAD_URLS,
+ StoresToCheck({GetUrlMalBinId()}), url_chain);
+
+ return HandleCheck(std::move(check));
}
bool V4LocalDatabaseManager::CheckExtensionIDs(
- const std::set<std::string>& extension_ids,
+ const std::set<FullHash>& extension_ids,
Client* client) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+
+ if (!enabled_) {
+ return true;
+ }
+
+ std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
+ client, ClientCallbackType::CHECK_EXTENSION_IDS,
+ StoresToCheck({GetChromeExtMalwareId()}), extension_ids);
+
+ return HandleCheck(std::move(check));
}
bool V4LocalDatabaseManager::CheckResourceUrl(const GURL& url, Client* client) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+
+ StoresToCheck stores_to_check({GetChromeUrlClientIncidentId()});
+
+ if (!CanCheckUrl(url) || !AreStoresAvailableNow(stores_to_check)) {
+ // Fail open: Mark resource as safe immediately.
+ // TODO(nparker): This should queue the request if the DB isn't yet
+ // loaded, and later decide if this store is available.
+ // Currently this is the only store that requires full-hash-checks
+ // AND isn't supported on Chromium, so it's unique.
+ return true;
+ }
+
+ std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
+ client, ClientCallbackType::CHECK_RESOURCE_URL, stores_to_check,
+ std::vector<GURL>(1, url));
+
+ return HandleCheck(std::move(check));
}
bool V4LocalDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+
+ StoresToCheck stores_to_check({GetUrlCsdWhitelistId()});
+ if (!AreStoresAvailableNow(stores_to_check)) {
+ // Fail open: Whitelist everything. Otherwise we may run the
+ // CSD phishing/malware detector on popular domains and generate
+ // undue load on the client and server. This has the effect of disabling
+ // CSD phishing/malware detection until the store is first synced.
+ return true;
+ }
+
+ return HandleUrlSynchronously(url, stores_to_check);
}
bool V4LocalDatabaseManager::MatchDownloadWhitelistString(
const std::string& str) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+
+ StoresToCheck stores_to_check({GetCertCsdDownloadWhitelistId()});
+ if (!AreStoresAvailableNow(stores_to_check)) {
+ // Fail close: Whitelist nothing. This may generate download-protection
+ // pings for whitelisted binaries, but that's fine.
+ return false;
+ }
+
+ return HandleHashSynchronously(str, stores_to_check);
}
bool V4LocalDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+
+ StoresToCheck stores_to_check({GetUrlCsdDownloadWhitelistId()});
+ if (!AreStoresAvailableNow(stores_to_check)) {
+ // Fail close: Whitelist nothing. This may generate download-protection
+ // pings for whitelisted domains, but that's fine.
+ return false;
+ }
+
+ return HandleUrlSynchronously(url, stores_to_check);
}
bool V4LocalDatabaseManager::MatchMalwareIP(const std::string& ip_address) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return false;
+ if (!enabled_ || !v4_database_) {
+ return false;
+ }
+
+ FullHash hashed_encoded_ip;
+ if (!V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash(ip_address,
+ &hashed_encoded_ip)) {
+ return false;
+ }
+
+ return HandleHashSynchronously(hashed_encoded_ip,
+ StoresToCheck({GetIpMalwareId()}));
}
bool V4LocalDatabaseManager::MatchModuleWhitelistString(
const std::string& str) {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+
+ StoresToCheck stores_to_check({GetChromeFilenameClientIncidentId()});
+ if (!AreStoresAvailableNow(stores_to_check)) {
+ // Fail open: Whitelist everything. This has the effect of marking
+ // all DLLs as safe until the DB is synced and loaded.
+ return true;
+ }
+
+ // str is the module's filename. Convert to hash.
+ FullHash hash = crypto::SHA256HashString(str);
+ return HandleHashSynchronously(hash, stores_to_check);
}
ThreatSource V4LocalDatabaseManager::GetThreatSource() const {
@@ -213,9 +339,8 @@ ThreatSource V4LocalDatabaseManager::GetThreatSource() const {
}
bool V4LocalDatabaseManager::IsCsdWhitelistKillSwitchOn() {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+ return false;
}
bool V4LocalDatabaseManager::IsDownloadProtectionEnabled() const {
@@ -225,9 +350,8 @@ bool V4LocalDatabaseManager::IsDownloadProtectionEnabled() const {
}
bool V4LocalDatabaseManager::IsMalwareKillSwitchOn() {
- // TODO(vakh): Implement this skeleton.
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- return true;
+ return false;
}
bool V4LocalDatabaseManager::IsSupported() const {
@@ -253,7 +377,7 @@ void V4LocalDatabaseManager::StopOnIOThread(bool shutdown) {
enabled_ = false;
- pending_clients_.clear();
+ pending_checks_.clear();
RespondSafeToQueuedChecks();
@@ -275,7 +399,7 @@ void V4LocalDatabaseManager::StopOnIOThread(bool shutdown) {
// End: SafeBrowsingDatabaseManager implementation
//
-void V4LocalDatabaseManager::DatabaseReady(
+void V4LocalDatabaseManager::DatabaseReadyForChecks(
std::unique_ptr<V4Database> v4_database) {
DCHECK_CURRENTLY_ON(BrowserThread::IO);
@@ -284,9 +408,15 @@ void V4LocalDatabaseManager::DatabaseReady(
if (enabled_) {
v4_database_ = std::move(v4_database);
- // The database is in place. Start fetching updates now.
- v4_update_protocol_manager_->ScheduleNextUpdate(
- v4_database_->GetStoreStateMap());
+ v4_database_->RecordFileSizeHistograms();
+
+ // The consistency of the stores read from the disk needs to verified. Post
+ // that task on the task runner. It calls |DatabaseReadyForUpdates|
+ // callback with the stores to reset, if any, and then we can schedule the
+ // database updates.
+ v4_database_->VerifyChecksum(
+ base::Bind(&V4LocalDatabaseManager::DatabaseReadyForUpdates,
+ weak_factory_.GetWeakPtr()));
ProcessQueuedChecks();
} else {
@@ -295,8 +425,20 @@ void V4LocalDatabaseManager::DatabaseReady(
}
}
+void V4LocalDatabaseManager::DatabaseReadyForUpdates(
+ const std::vector<ListIdentifier>& stores_to_reset) {
+ if (enabled_) {
+ v4_database_->ResetStores(stores_to_reset);
+
+ // The database is ready to process updates. Schedule them now.
+ v4_update_protocol_manager_->ScheduleNextUpdate(
+ v4_database_->GetStoreStateMap());
+ }
+}
+
void V4LocalDatabaseManager::DatabaseUpdated() {
if (enabled_) {
+ v4_database_->RecordFileSizeHistograms();
v4_update_protocol_manager_->ScheduleNextUpdate(
v4_database_->GetStoreStateMap());
}
@@ -309,15 +451,18 @@ bool V4LocalDatabaseManager::GetPrefixMatches(
DCHECK(enabled_);
DCHECK(v4_database_);
- DCHECK_GT(ClientCallbackType::CHECK_MAX, check->client_callback_type);
-
- if (check->client_callback_type == ClientCallbackType::CHECK_BROWSE_URL) {
- std::unordered_set<FullHash> full_hashes;
- V4ProtocolManagerUtil::UrlToFullHashes(check->url, &full_hashes);
- StoreAndHashPrefixes matched_store_and_hash_prefixes;
- for (const auto& full_hash : full_hashes) {
- matched_store_and_hash_prefixes.clear();
+ const base::TimeTicks before = TimeTicks::Now();
+ if (check->client_callback_type == ClientCallbackType::CHECK_BROWSE_URL ||
+ check->client_callback_type == ClientCallbackType::CHECK_DOWNLOAD_URLS ||
+ check->client_callback_type == ClientCallbackType::CHECK_RESOURCE_URL ||
+ check->client_callback_type == ClientCallbackType::CHECK_EXTENSION_IDS ||
+ check->client_callback_type == ClientCallbackType::CHECK_OTHER) {
+ DCHECK(!check->full_hashes.empty());
+
+ full_hash_to_store_and_hash_prefixes->clear();
+ for (const auto& full_hash : check->full_hashes) {
+ StoreAndHashPrefixes matched_store_and_hash_prefixes;
v4_database_->GetStoresMatchingFullHash(full_hash, check->stores_to_check,
&matched_store_and_hash_prefixes);
if (!matched_store_and_hash_prefixes.empty()) {
@@ -325,22 +470,31 @@ bool V4LocalDatabaseManager::GetPrefixMatches(
matched_store_and_hash_prefixes;
}
}
-
- // No hash prefixes found in the local database so that resource must be
- // safe.
- return !full_hash_to_store_and_hash_prefixes->empty();
+ } else {
+ NOTREACHED() << "Unexpected client_callback_type encountered.";
}
- NOTREACHED() << "Unexpected client_callback_type encountered";
- return false;
+ // TODO(vakh): Only log SafeBrowsing.V4GetPrefixMatches.Time once PVer3 code
+ // is removed.
+ // NOTE(vakh): This doesn't distinguish which stores it's searching through.
+ // However, the vast majority of the entries in this histogram will be from
+ // searching the three CHECK_BROWSE_URL stores.
+ base::TimeDelta diff = TimeTicks::Now() - before;
+ UMA_HISTOGRAM_TIMES("SB2.FilterCheck", diff);
+ UMA_HISTOGRAM_CUSTOM_TIMES("SafeBrowsing.V4GetPrefixMatches.Time", diff,
+ base::TimeDelta::FromMicroseconds(20),
+ base::TimeDelta::FromSeconds(1), 50);
+ return !full_hash_to_store_and_hash_prefixes->empty();
}
void V4LocalDatabaseManager::GetSeverestThreatTypeAndMetadata(
SBThreatType* result_threat_type,
ThreatMetadata* metadata,
+ FullHash* matching_full_hash,
const std::vector<FullHashInfo>& full_hash_infos) {
DCHECK(result_threat_type);
DCHECK(metadata);
+ DCHECK(matching_full_hash);
ThreatSeverity most_severe_yet = kLeastSeverity;
for (const FullHashInfo& fhi : full_hash_infos) {
@@ -349,6 +503,7 @@ void V4LocalDatabaseManager::GetSeverestThreatTypeAndMetadata(
most_severe_yet = severity;
*result_threat_type = GetSBThreatTypeForList(fhi.list_id);
*metadata = fhi.metadata;
+ *matching_full_hash = fhi.full_hash;
}
}
}
@@ -369,31 +524,84 @@ SBThreatType V4LocalDatabaseManager::GetSBThreatTypeForList(
[&list_id](ListInfo const& li) { return li.list_id() == list_id; });
DCHECK(list_infos_.end() != it);
DCHECK_NE(SB_THREAT_TYPE_SAFE, it->sb_threat_type());
+ DCHECK_NE(SB_THREAT_TYPE_UNUSED, it->sb_threat_type());
return it->sb_threat_type();
}
+bool V4LocalDatabaseManager::HandleCheck(std::unique_ptr<PendingCheck> check) {
+ if (!v4_database_) {
+ queued_checks_.push_back(std::move(check));
+ return false;
+ }
+
+ FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
+ if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) {
+ return true;
+ }
+
+ // Add check to pending_checks_ before scheduling PerformFullHashCheck so that
+ // even if the client calls CancelCheck before PerformFullHashCheck gets
+ // called, the check can be found in pending_checks_.
+ pending_checks_.insert(check.get());
+
+ // Post on the IO thread to enforce async behavior.
+ BrowserThread::PostTask(
+ BrowserThread::IO, FROM_HERE,
+ base::Bind(&V4LocalDatabaseManager::PerformFullHashCheck, this,
+ base::Passed(std::move(check)),
+ full_hash_to_store_and_hash_prefixes));
+
+ return false;
+}
+
+bool V4LocalDatabaseManager::HandleHashSynchronously(
+ const FullHash& hash,
+ const StoresToCheck& stores_to_check) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+ std::set<FullHash> hashes{hash};
+ std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
+ nullptr, ClientCallbackType::CHECK_OTHER, stores_to_check, hashes);
+
+ FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
+ return GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes);
+}
+
+bool V4LocalDatabaseManager::HandleUrlSynchronously(
+ const GURL& url,
+ const StoresToCheck& stores_to_check) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+ std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
+ nullptr, ClientCallbackType::CHECK_OTHER, stores_to_check,
+ std::vector<GURL>(1, url));
+
+ FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
+ return GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes);
+}
+
void V4LocalDatabaseManager::OnFullHashResponse(
- std::unique_ptr<PendingCheck> pending_check,
+ std::unique_ptr<PendingCheck> check,
const std::vector<FullHashInfo>& full_hash_infos) {
DCHECK_CURRENTLY_ON(BrowserThread::IO);
if (!enabled_) {
- DCHECK(pending_clients_.empty());
+ DCHECK(pending_checks_.empty());
return;
}
- auto it = pending_clients_.find(pending_check->client);
- if (it == pending_clients_.end()) {
+ const auto it = pending_checks_.find(check.get());
+ if (it == pending_checks_.end()) {
// The check has since been cancelled.
return;
}
// Find out the most severe threat, if any, to report to the client.
- GetSeverestThreatTypeAndMetadata(&pending_check->result_threat_type,
- &pending_check->url_metadata,
- full_hash_infos);
- pending_clients_.erase(it);
- RespondToClient(std::move(pending_check));
+ GetSeverestThreatTypeAndMetadata(&check->result_threat_type,
+ &check->url_metadata,
+ &check->matching_full_hash, full_hash_infos);
+ pending_checks_.erase(it);
+ RespondToClient(std::move(check));
}
void V4LocalDatabaseManager::PerformFullHashCheck(
@@ -405,8 +613,6 @@ void V4LocalDatabaseManager::PerformFullHashCheck(
DCHECK(enabled_);
DCHECK(!full_hash_to_store_and_hash_prefixes.empty());
- pending_clients_.insert(check->client);
-
v4_get_hash_protocol_manager_->GetFullHashes(
full_hash_to_store_and_hash_prefixes,
base::Bind(&V4LocalDatabaseManager::OnFullHashResponse,
@@ -415,31 +621,66 @@ void V4LocalDatabaseManager::PerformFullHashCheck(
void V4LocalDatabaseManager::ProcessQueuedChecks() {
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- for (auto& it : queued_checks_) {
+
+ // Steal the queue to protect against reentrant CancelCheck() calls.
+ QueuedChecks checks;
+ checks.swap(queued_checks_);
+
+ for (auto& it : checks) {
FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
if (!GetPrefixMatches(it, &full_hash_to_store_and_hash_prefixes)) {
RespondToClient(std::move(it));
} else {
+ pending_checks_.insert(it.get());
PerformFullHashCheck(std::move(it), full_hash_to_store_and_hash_prefixes);
}
}
- queued_checks_.clear();
}
void V4LocalDatabaseManager::RespondSafeToQueuedChecks() {
DCHECK_CURRENTLY_ON(BrowserThread::IO);
- for (std::unique_ptr<PendingCheck>& it : queued_checks_) {
+
+ // Steal the queue to protect against reentrant CancelCheck() calls.
+ QueuedChecks checks;
+ checks.swap(queued_checks_);
+
+ for (std::unique_ptr<PendingCheck>& it : checks) {
RespondToClient(std::move(it));
}
- queued_checks_.clear();
}
void V4LocalDatabaseManager::RespondToClient(
- std::unique_ptr<PendingCheck> pending_check) {
- DCHECK(pending_check.get());
- DCHECK_EQ(ClientCallbackType::CHECK_BROWSE_URL,
- pending_check->client_callback_type);
- // TODO(vakh): Implement this skeleton.
+ std::unique_ptr<PendingCheck> check) {
+ DCHECK(check.get());
+
+ switch (check->client_callback_type) {
+ case ClientCallbackType::CHECK_BROWSE_URL:
+ DCHECK_EQ(1u, check->urls.size());
+ check->client->OnCheckBrowseUrlResult(
+ check->urls[0], check->result_threat_type, check->url_metadata);
+ break;
+
+ case ClientCallbackType::CHECK_DOWNLOAD_URLS:
+ check->client->OnCheckDownloadUrlResult(check->urls,
+ check->result_threat_type);
+ break;
+
+ case ClientCallbackType::CHECK_RESOURCE_URL:
+ DCHECK_EQ(1u, check->urls.size());
+ check->client->OnCheckResourceUrlResult(
+ check->urls[0], check->result_threat_type, check->matching_full_hash);
+ break;
+
+ case ClientCallbackType::CHECK_EXTENSION_IDS: {
+ const std::set<FullHash> extension_ids(check->full_hashes.begin(),
+ check->full_hashes.end());
+ check->client->OnCheckExtensionsResult(extension_ids);
+ break;
+ }
+
+ case ClientCallbackType::CHECK_OTHER:
+ NOTREACHED() << "Unexpected client_callback_type encountered";
+ }
}
void V4LocalDatabaseManager::SetupDatabase() {
@@ -458,8 +699,9 @@ void V4LocalDatabaseManager::SetupDatabase() {
// Do not create the database on the IO thread since this may be an expensive
// operation. Instead, do that on the task_runner and when the new database
// has been created, swap it out on the IO thread.
- NewDatabaseReadyCallback db_ready_callback = base::Bind(
- &V4LocalDatabaseManager::DatabaseReady, weak_factory_.GetWeakPtr());
+ NewDatabaseReadyCallback db_ready_callback =
+ base::Bind(&V4LocalDatabaseManager::DatabaseReadyForChecks,
+ weak_factory_.GetWeakPtr());
V4Database::Create(task_runner_, base_path_, list_infos_, db_ready_callback);
}
@@ -481,4 +723,10 @@ void V4LocalDatabaseManager::UpdateRequestCompleted(
db_updated_callback_);
}
+bool V4LocalDatabaseManager::AreStoresAvailableNow(
+ const StoresToCheck& stores_to_check) const {
+ return enabled_ && v4_database_ &&
+ v4_database_->AreStoresAvailable(stores_to_check);
+}
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager.h b/chromium/components/safe_browsing_db/v4_local_database_manager.h
index 0449b8c5ac5..41598243228 100644
--- a/chromium/components/safe_browsing_db/v4_local_database_manager.h
+++ b/chromium/components/safe_browsing_db/v4_local_database_manager.h
@@ -9,6 +9,7 @@
// and database that holds the downloaded updates.
#include <memory>
+#include <unordered_set>
#include "base/memory/weak_ptr.h"
#include "components/safe_browsing_db/database_manager.h"
@@ -45,7 +46,11 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
bool CheckBrowseUrl(const GURL& url, Client* client) override;
bool CheckDownloadUrl(const std::vector<GURL>& url_chain,
Client* client) override;
- bool CheckExtensionIDs(const std::set<std::string>& extension_ids,
+ // TODO(vakh): |CheckExtensionIDs| in the base class accepts a set of
+ // std::strings but the overriding method in this class accepts a set of
+ // FullHash objects. Since FullHash is currently std::string, it compiles,
+ // but this difference should be eliminated.
+ bool CheckExtensionIDs(const std::set<FullHash>& extension_ids,
Client* client) override;
bool CheckResourceUrl(const GURL& url, Client* client) override;
bool MatchCsdWhitelistUrl(const GURL& url) override;
@@ -71,24 +76,48 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
// Must be initialized by calling StartOnIOThread() before using.
V4LocalDatabaseManager(const base::FilePath& base_path);
+ ~V4LocalDatabaseManager() override;
+
+ void SetTaskRunnerForTest(
+ const scoped_refptr<base::SequencedTaskRunner>& task_runner) {
+ task_runner_ = task_runner;
+ }
+
enum class ClientCallbackType {
// This represents the case when we're trying to determine if a URL is
// unsafe from the following perspectives: Malware, Phishing, UwS.
CHECK_BROWSE_URL = 0,
- // This should always be the last value.
- CHECK_MAX
+ // This represents the case when we're trying to determine if any of the
+ // URLs in a vector of URLs is unsafe for downloading binaries.
+ CHECK_DOWNLOAD_URLS = 1,
+
+ // This represents the case when we're trying to determine if a URL is an
+ // unsafe resource.
+ CHECK_RESOURCE_URL = 2,
+
+ // This represents the case when we're trying to determine if a Chrome
+ // extension is a unsafe.
+ CHECK_EXTENSION_IDS = 3,
+
+ // This represents the other cases when a check is being performed
+ // synchronously so a client callback isn't required. For instance, when
+ // trying to determing if an IP address is unsafe due to hosting Malware.
+ CHECK_OTHER = 4,
};
// The information we need to process a URL safety reputation request and
// respond to the SafeBrowsing client that asked for it.
- // TODO(vakh): In its current form, it only includes information for
- // |CheckBrowseUrl| method. Extend it to serve other methods on |client|.
struct PendingCheck {
PendingCheck(Client* client,
ClientCallbackType client_callback_type,
const StoresToCheck& stores_to_check,
- const GURL& url);
+ const std::vector<GURL>& urls);
+
+ PendingCheck(Client* client,
+ ClientCallbackType client_callback_type,
+ const StoresToCheck& stores_to_check,
+ const std::set<FullHash>& full_hashes);
~PendingCheck();
@@ -97,20 +126,34 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
// Determines which funtion from the |client| needs to be called once we
// know whether the URL in |url| is safe or unsafe.
- ClientCallbackType client_callback_type;
+ const ClientCallbackType client_callback_type;
// The threat verdict for the URL being checked.
SBThreatType result_threat_type;
+ // When the check was sent to the SafeBrowsing service. Used to record the
+ // time it takes to get the uncached full hashes from the service (or a
+ // cached full hash response).
+ base::TimeTicks full_hash_check_start;
+
// The SafeBrowsing lists to check hash prefixes in.
- StoresToCheck stores_to_check;
+ const StoresToCheck stores_to_check;
+
+ // The URLs that are being checked for being unsafe. The size of exactly
+ // one of |full_hashes| and |urls| should be greater than 0.
+ const std::vector<GURL> urls;
- // The URL that is being checked for being unsafe.
- GURL url;
+ // The full hashes that are being checked for being safe. The size of
+ // exactly one of |full_hashes| and |urls| should be greater than 0.
+ std::vector<FullHash> full_hashes;
// The metadata associated with the full hash of the severest match found
// for that URL.
ThreatMetadata url_metadata;
+
+ // The full hash that matched for a blacklisted resource URL. Used only for
+ // |CheckResourceUrl| case.
+ FullHash matching_full_hash;
};
typedef std::vector<std::unique_ptr<PendingCheck>> QueuedChecks;
@@ -121,42 +164,61 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
private:
friend class V4LocalDatabaseManagerTest;
- void SetTaskRunnerForTest(
- const scoped_refptr<base::SequencedTaskRunner>& task_runner) {
- task_runner_ = task_runner;
- }
FRIEND_TEST_ALL_PREFIXES(V4LocalDatabaseManagerTest,
TestGetSeverestThreatTypeAndMetadata);
- // The set of clients awaiting a full hash response. It is used for tracking
- // which clients have cancelled their outstanding request.
- typedef std::unordered_set<Client*> PendingClients;
-
- ~V4LocalDatabaseManager() override;
+ // The checks awaiting a full hash response from SafeBrowsing service.
+ typedef std::unordered_set<const PendingCheck*> PendingChecks;
// Called when all the stores managed by the database have been read from
- // disk after startup and the database is ready for use.
- void DatabaseReady(std::unique_ptr<V4Database> v4_database);
+ // disk after startup and the database is ready for checking resource
+ // reputation.
+ void DatabaseReadyForChecks(std::unique_ptr<V4Database> v4_database);
+
+ // Called when all the stores managed by the database have been verified for
+ // checksum correctness after startup and the database is ready for applying
+ // updates.
+ void DatabaseReadyForUpdates(
+ const std::vector<ListIdentifier>& stores_to_reset);
// Called when the database has been updated and schedules the next update.
void DatabaseUpdated();
- // Return the prefixes and the store they matched in, for a given URL. Returns
- // true if a hash prefix match is found; false otherwise.
+ // Identifies the prefixes and the store they matched in, for a given |check|.
+ // Returns true if one or more hash prefix matches are found; false otherwise.
bool GetPrefixMatches(
const std::unique_ptr<PendingCheck>& check,
FullHashToStoreAndHashPrefixesMap* full_hash_to_store_and_hash_prefixes);
- // Finds the most severe |SBThreatType| and the corresponding |metadata| from
- // |full_hash_infos|.
+ // Finds the most severe |SBThreatType| and the corresponding |metadata|, and
+ // |matching_full_hash| from |full_hash_infos|.
void GetSeverestThreatTypeAndMetadata(
SBThreatType* result_threat_type,
ThreatMetadata* metadata,
+ FullHash* matching_full_hash,
const std::vector<FullHashInfo>& full_hash_infos);
// Returns the SBThreatType for a given ListIdentifier.
SBThreatType GetSBThreatTypeForList(const ListIdentifier& list_id);
+ // Queues the check for async response if the database isn't ready yet.
+ // If the database is ready, checks the database for prefix matches and
+ // returns true immediately if there's no match. If a match is found, it
+ // schedules a task to perform full hash check and returns false.
+ bool HandleCheck(std::unique_ptr<PendingCheck> check);
+
+ // Checks |stores_to_check| in database synchronously for hash prefixes
+ // matching |hash|. Returns true if there's a match; false otherwise. This is
+ // used for lists that have full hash information in the database.
+ bool HandleHashSynchronously(const FullHash& hash,
+ const StoresToCheck& stores_to_check);
+
+ // Checks |stores_to_check| in database synchronously for hash prefixes
+ // matching the full hashes for |url|. See |HandleHashSynchronously| for
+ // details.
+ bool HandleUrlSynchronously(const GURL& url,
+ const StoresToCheck& stores_to_check);
+
// Called when the |v4_get_hash_protocol_manager_| has the full hash response
// available for the URL that we requested. It determines the severest
// threat type and responds to the |client| with that information.
@@ -164,9 +226,9 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
const std::vector<FullHashInfo>& full_hash_infos);
// Performs the full hash checking of the URL in |check|.
- void PerformFullHashCheck(std::unique_ptr<PendingCheck> check,
- const FullHashToStoreAndHashPrefixesMap&
- full_hash_to_store_and_hash_prefixes);
+ virtual void PerformFullHashCheck(std::unique_ptr<PendingCheck> check,
+ const FullHashToStoreAndHashPrefixesMap&
+ full_hash_to_store_and_hash_prefixes);
// When the database is ready to use, process the checks that were queued
// while the database was loading from disk.
@@ -194,6 +256,10 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
void UpdateRequestCompleted(
std::unique_ptr<ParsedServerResponse> parsed_server_response);
+ // Return true if we're enabled and have loaded real data for all of
+ // these stores.
+ bool AreStoresAvailableNow(const StoresToCheck& stores_to_check) const;
+
// The base directory under which to create the files that contain hashes.
const base::FilePath base_path_;
@@ -210,9 +276,8 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager {
// name of the file on disk that would contain the prefixes, if applicable.
ListInfos list_infos_;
- // The set of clients that are waiting for a full hash response from the
- // SafeBrowsing service.
- PendingClients pending_clients_;
+ // The checks awaiting for a full hash response from the SafeBrowsing service.
+ PendingChecks pending_checks_;
// The checks that need to be scheduled when the database becomes ready for
// use.
diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc
index 6574a9515d0..85df2f85adb 100644
--- a/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc
+++ b/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc
@@ -7,47 +7,202 @@
#include "base/memory/ref_counted.h"
#include "base/run_loop.h"
#include "base/test/test_simple_task_runner.h"
+#include "base/threading/thread_task_runner_handle.h"
#include "components/safe_browsing_db/v4_database.h"
#include "components/safe_browsing_db/v4_local_database_manager.h"
+#include "components/safe_browsing_db/v4_test_util.h"
#include "content/public/test/test_browser_thread_bundle.h"
+#include "crypto/sha2.h"
#include "net/url_request/test_url_fetcher_factory.h"
#include "testing/platform_test.h"
namespace safe_browsing {
+namespace {
+
+// Utility function for populating hashes.
+FullHash HashForUrl(const GURL& url) {
+ std::vector<FullHash> full_hashes;
+ V4ProtocolManagerUtil::UrlToFullHashes(url, &full_hashes);
+ // ASSERT_GE(full_hashes.size(), 1u);
+ return full_hashes[0];
+}
+
+// Always returns misses from GetFullHashes().
+class FakeGetHashProtocolManager : public V4GetHashProtocolManager {
+ public:
+ FakeGetHashProtocolManager(
+ net::URLRequestContextGetter* request_context_getter,
+ const StoresToCheck& stores_to_check,
+ const V4ProtocolConfig& config)
+ : V4GetHashProtocolManager(request_context_getter,
+ stores_to_check,
+ config) {}
+
+ void GetFullHashes(const FullHashToStoreAndHashPrefixesMap&,
+ FullHashCallback callback) override {
+ std::vector<FullHashInfo> full_hash_infos;
+
+ // Async, since the real manager might use a fetcher.
+ base::ThreadTaskRunnerHandle::Get()->PostTask(
+ FROM_HERE, base::Bind(callback, full_hash_infos));
+ }
+};
+
+class FakeGetHashProtocolManagerFactory
+ : public V4GetHashProtocolManagerFactory {
+ public:
+ std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager(
+ net::URLRequestContextGetter* request_context_getter,
+ const StoresToCheck& stores_to_check,
+ const V4ProtocolConfig& config) override {
+ return base::MakeUnique<FakeGetHashProtocolManager>(
+ request_context_getter, stores_to_check, config);
+ }
+};
+
+// Use FakeGetHashProtocolManagerFactory in scope, then reset.
+class ScopedFakeGetHashProtocolManagerFactory {
+ public:
+ ScopedFakeGetHashProtocolManagerFactory() {
+ V4GetHashProtocolManager::RegisterFactory(
+ base::MakeUnique<FakeGetHashProtocolManagerFactory>());
+ }
+ ~ScopedFakeGetHashProtocolManagerFactory() {
+ V4GetHashProtocolManager::RegisterFactory(nullptr);
+ }
+};
+
+} // namespace
+
class FakeV4Database : public V4Database {
public:
- FakeV4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
- std::unique_ptr<StoreMap> store_map,
- const StoreAndHashPrefixes& store_and_hash_prefixes)
- : V4Database(db_task_runner, std::move(store_map)),
- store_and_hash_prefixes_(store_and_hash_prefixes) {}
+ static void Create(
+ const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
+ std::unique_ptr<StoreMap> store_map,
+ const StoreAndHashPrefixes& store_and_hash_prefixes,
+ NewDatabaseReadyCallback new_db_callback,
+ bool stores_available) {
+ // Mimics V4Database::Create
+ const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner =
+ base::MessageLoop::current()->task_runner();
+ db_task_runner->PostTask(
+ FROM_HERE,
+ base::Bind(&FakeV4Database::CreateOnTaskRunner, db_task_runner,
+ base::Passed(&store_map), store_and_hash_prefixes,
+ callback_task_runner, new_db_callback, stores_available));
+ }
+ // V4Database implementation
void GetStoresMatchingFullHash(
const FullHash& full_hash,
const StoresToCheck& stores_to_check,
StoreAndHashPrefixes* store_and_hash_prefixes) override {
- *store_and_hash_prefixes = store_and_hash_prefixes_;
+ store_and_hash_prefixes->clear();
+ for (const StoreAndHashPrefix& stored_sahp : store_and_hash_prefixes_) {
+ const PrefixSize& prefix_size = stored_sahp.hash_prefix.size();
+ if (!full_hash.compare(0, prefix_size, stored_sahp.hash_prefix)) {
+ store_and_hash_prefixes->push_back(stored_sahp);
+ }
+ }
+ }
+
+ bool AreStoresAvailable(const StoresToCheck& stores_to_check) const override {
+ return stores_available_;
}
private:
- const StoreAndHashPrefixes& store_and_hash_prefixes_;
+ static void CreateOnTaskRunner(
+ const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
+ std::unique_ptr<StoreMap> store_map,
+ const StoreAndHashPrefixes& store_and_hash_prefixes,
+ const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner,
+ NewDatabaseReadyCallback new_db_callback,
+ bool stores_available) {
+ // Mimics the semantics of V4Database::CreateOnTaskRunner
+ std::unique_ptr<FakeV4Database> fake_v4_database(
+ new FakeV4Database(db_task_runner, std::move(store_map),
+ store_and_hash_prefixes, stores_available));
+ callback_task_runner->PostTask(
+ FROM_HERE,
+ base::Bind(new_db_callback, base::Passed(&fake_v4_database)));
+ }
+
+ FakeV4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
+ std::unique_ptr<StoreMap> store_map,
+ const StoreAndHashPrefixes& store_and_hash_prefixes,
+ bool stores_available)
+ : V4Database(db_task_runner, std::move(store_map)),
+ store_and_hash_prefixes_(store_and_hash_prefixes),
+ stores_available_(stores_available) {}
+
+ const StoreAndHashPrefixes store_and_hash_prefixes_;
+ const bool stores_available_;
};
class TestClient : public SafeBrowsingDatabaseManager::Client {
public:
- TestClient(SBThreatType sb_threat_type, const GURL& url)
- : expected_sb_threat_type(sb_threat_type), expected_url(url) {}
+ TestClient(SBThreatType sb_threat_type,
+ const GURL& url,
+ V4LocalDatabaseManager* manager_to_cancel = nullptr)
+ : expected_sb_threat_type(sb_threat_type),
+ expected_url(url),
+ on_check_browse_url_result_called_(false),
+ on_check_resource_url_result_called_(false),
+ manager_to_cancel_(manager_to_cancel) {}
void OnCheckBrowseUrlResult(const GURL& url,
SBThreatType threat_type,
const ThreatMetadata& metadata) override {
DCHECK_EQ(expected_url, url);
DCHECK_EQ(expected_sb_threat_type, threat_type);
+ on_check_browse_url_result_called_ = true;
+ if (manager_to_cancel_) {
+ manager_to_cancel_->CancelCheck(this);
+ }
+ }
+
+ void OnCheckResourceUrlResult(const GURL& url,
+ SBThreatType threat_type,
+ const std::string& threat_hash) override {
+ DCHECK_EQ(expected_url, url);
+ DCHECK_EQ(expected_sb_threat_type, threat_type);
+ // |threat_hash| is empty because GetFullHashes calls back with empty
+ // |full_hash_infos|.
+ DCHECK(threat_hash.empty());
+ on_check_resource_url_result_called_ = true;
}
SBThreatType expected_sb_threat_type;
GURL expected_url;
+ bool on_check_browse_url_result_called_;
+ bool on_check_resource_url_result_called_;
+ V4LocalDatabaseManager* manager_to_cancel_;
+};
+
+class FakeV4LocalDatabaseManager : public V4LocalDatabaseManager {
+ public:
+ void PerformFullHashCheck(std::unique_ptr<PendingCheck> check,
+ const FullHashToStoreAndHashPrefixesMap&
+ full_hash_to_store_and_hash_prefixes) override {
+ perform_full_hash_check_called_ = true;
+ }
+
+ FakeV4LocalDatabaseManager(const base::FilePath& base_path)
+ : V4LocalDatabaseManager(base_path),
+ perform_full_hash_check_called_(false) {}
+
+ static bool PerformFullHashCheckCalled(
+ scoped_refptr<safe_browsing::V4LocalDatabaseManager>& v4_ldbm) {
+ FakeV4LocalDatabaseManager* fake =
+ static_cast<FakeV4LocalDatabaseManager*>(v4_ldbm.get());
+ return fake->perform_full_hash_check_called_;
+ }
+
+ private:
+ ~FakeV4LocalDatabaseManager() override {}
+
+ bool perform_full_hash_check_called_;
};
class V4LocalDatabaseManagerTest : public PlatformTest {
@@ -62,7 +217,7 @@ class V4LocalDatabaseManagerTest : public PlatformTest {
v4_local_database_manager_ =
make_scoped_refptr(new V4LocalDatabaseManager(base_dir_.GetPath()));
- v4_local_database_manager_->SetTaskRunnerForTest(task_runner_);
+ SetTaskRunnerForTest();
StartLocalDatabaseManager();
}
@@ -77,22 +232,55 @@ class V4LocalDatabaseManagerTest : public PlatformTest {
v4_local_database_manager_->enabled_ = false;
}
+ void ForceEnableLocalDatabaseManager() {
+ v4_local_database_manager_->enabled_ = true;
+ }
+
const V4LocalDatabaseManager::QueuedChecks& GetQueuedChecks() {
return v4_local_database_manager_->queued_checks_;
}
- void ReplaceV4Database(const StoreAndHashPrefixes& store_and_hash_prefixes) {
- v4_local_database_manager_->v4_database_.reset(new FakeV4Database(
- task_runner_, base::MakeUnique<StoreMap>(), store_and_hash_prefixes));
+ void ReplaceV4Database(const StoreAndHashPrefixes& store_and_hash_prefixes,
+ bool stores_available = false) {
+ // Disable the V4LocalDatabaseManager first so that if the callback to
+ // verify checksum has been scheduled, then it doesn't do anything when it
+ // is called back.
+ ForceDisableLocalDatabaseManager();
+ // Wait to make sure that the callback gets executed if it has already been
+ // scheduled.
+ WaitForTasksOnTaskRunner();
+ // Re-enable the V4LocalDatabaseManager otherwise the checks won't work and
+ // the fake database won't be set either.
+ ForceEnableLocalDatabaseManager();
+
+ NewDatabaseReadyCallback db_ready_callback =
+ base::Bind(&V4LocalDatabaseManager::DatabaseReadyForChecks,
+ base::Unretained(v4_local_database_manager_.get()));
+ FakeV4Database::Create(task_runner_, base::MakeUnique<StoreMap>(),
+ store_and_hash_prefixes, db_ready_callback,
+ stores_available);
+ WaitForTasksOnTaskRunner();
+ }
+
+ void ResetLocalDatabaseManager() {
+ StopLocalDatabaseManager();
+ v4_local_database_manager_ =
+ make_scoped_refptr(new V4LocalDatabaseManager(base_dir_.GetPath()));
+ SetTaskRunnerForTest();
+ StartLocalDatabaseManager();
+ }
+
+ void ResetV4Database() {
+ V4Database::Destroy(std::move(v4_local_database_manager_->v4_database_));
}
- void ResetV4Database() { v4_local_database_manager_->v4_database_.reset(); }
+ void SetTaskRunnerForTest() {
+ v4_local_database_manager_->SetTaskRunnerForTest(task_runner_);
+ }
void StartLocalDatabaseManager() {
- v4_local_database_manager_->StartOnIOThread(NULL, V4ProtocolConfig());
-
- task_runner_->RunPendingTasks();
- base::RunLoop().RunUntilIdle();
+ v4_local_database_manager_->StartOnIOThread(NULL,
+ GetTestV4ProtocolConfig());
}
void StopLocalDatabaseManager() {
@@ -101,10 +289,28 @@ class V4LocalDatabaseManagerTest : public PlatformTest {
}
// Force destruction of the database.
+ WaitForTasksOnTaskRunner();
+ }
+
+ void WaitForTasksOnTaskRunner() {
+ // Wait for tasks on the task runner so we're sure that the
+ // V4LocalDatabaseManager has read the data from disk.
task_runner_->RunPendingTasks();
base::RunLoop().RunUntilIdle();
}
+ // For those tests that need the fake manager
+ void SetupFakeManager() {
+ // StopLocalDatabaseManager before resetting it because that's what
+ // ~V4LocalDatabaseManager expects.
+ StopLocalDatabaseManager();
+ v4_local_database_manager_ =
+ make_scoped_refptr(new FakeV4LocalDatabaseManager(base_dir_.GetPath()));
+ SetTaskRunnerForTest();
+ StartLocalDatabaseManager();
+ WaitForTasksOnTaskRunner();
+ }
+
base::ScopedTempDir base_dir_;
scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
content::TestBrowserThreadBundle thread_bundle_;
@@ -112,15 +318,18 @@ class V4LocalDatabaseManagerTest : public PlatformTest {
};
TEST_F(V4LocalDatabaseManagerTest, TestGetThreatSource) {
+ WaitForTasksOnTaskRunner();
EXPECT_EQ(ThreatSource::LOCAL_PVER4,
v4_local_database_manager_->GetThreatSource());
}
TEST_F(V4LocalDatabaseManagerTest, TestIsSupported) {
+ WaitForTasksOnTaskRunner();
EXPECT_TRUE(v4_local_database_manager_->IsSupported());
}
TEST_F(V4LocalDatabaseManagerTest, TestCanCheckUrl) {
+ WaitForTasksOnTaskRunner();
EXPECT_TRUE(
v4_local_database_manager_->CanCheckUrl(GURL("http://example.com/a/")));
EXPECT_TRUE(
@@ -133,28 +342,32 @@ TEST_F(V4LocalDatabaseManagerTest, TestCanCheckUrl) {
TEST_F(V4LocalDatabaseManagerTest,
TestCheckBrowseUrlWithEmptyStoresReturnsNoMatch) {
+ WaitForTasksOnTaskRunner();
// Both the stores are empty right now so CheckBrowseUrl should return true.
EXPECT_TRUE(v4_local_database_manager_->CheckBrowseUrl(
GURL("http://example.com/a/"), nullptr));
}
TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithFakeDbReturnsMatch) {
+ WaitForTasksOnTaskRunner();
net::TestURLFetcherFactory factory;
StoreAndHashPrefixes store_and_hash_prefixes;
- store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), HashPrefix("aaaa"));
+ store_and_hash_prefixes.emplace_back(GetUrlMalwareId(),
+ HashPrefix("eW\x1A\xF\xA9"));
ReplaceV4Database(store_and_hash_prefixes);
// The fake database returns a matched hash prefix.
EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(
GURL("http://example.com/a/"), nullptr));
+
+ // Wait for PerformFullHashCheck to complete.
+ WaitForTasksOnTaskRunner();
}
TEST_F(V4LocalDatabaseManagerTest,
TestCheckBrowseUrlReturnsNoMatchWhenDisabled) {
- StoreAndHashPrefixes store_and_hash_prefixes;
- store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), HashPrefix("aaaa"));
- ReplaceV4Database(store_and_hash_prefixes);
+ WaitForTasksOnTaskRunner();
// The same URL returns |false| in the previous test because
// v4_local_database_manager_ is enabled.
@@ -165,8 +378,10 @@ TEST_F(V4LocalDatabaseManagerTest,
}
TEST_F(V4LocalDatabaseManagerTest, TestGetSeverestThreatTypeAndMetadata) {
- FullHashInfo fhi_malware(FullHash("Malware"), GetUrlMalwareId(),
- base::Time::Now());
+ WaitForTasksOnTaskRunner();
+
+ FullHash full_hash("Malware");
+ FullHashInfo fhi_malware(full_hash, GetUrlMalwareId(), base::Time::Now());
fhi_malware.metadata.population_id = "malware_popid";
FullHashInfo fhi_api(FullHash("api"), GetChromeUrlApiId(), base::Time::Now());
@@ -176,18 +391,21 @@ TEST_F(V4LocalDatabaseManagerTest, TestGetSeverestThreatTypeAndMetadata) {
SBThreatType result_threat_type;
ThreatMetadata metadata;
+ FullHash matching_full_hash;
v4_local_database_manager_->GetSeverestThreatTypeAndMetadata(
- &result_threat_type, &metadata, fhis);
+ &result_threat_type, &metadata, &matching_full_hash, fhis);
EXPECT_EQ(SB_THREAT_TYPE_URL_MALWARE, result_threat_type);
EXPECT_EQ("malware_popid", metadata.population_id);
+ EXPECT_EQ(full_hash, matching_full_hash);
// Reversing the list has no effect.
std::reverse(std::begin(fhis), std::end(fhis));
v4_local_database_manager_->GetSeverestThreatTypeAndMetadata(
- &result_threat_type, &metadata, fhis);
+ &result_threat_type, &metadata, &matching_full_hash, fhis);
EXPECT_EQ(SB_THREAT_TYPE_URL_MALWARE, result_threat_type);
EXPECT_EQ("malware_popid", metadata.population_id);
+ EXPECT_EQ(full_hash, matching_full_hash);
}
TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) {
@@ -195,17 +413,11 @@ TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) {
TestClient client(SB_THREAT_TYPE_SAFE, url);
EXPECT_TRUE(GetQueuedChecks().empty());
v4_local_database_manager_->CheckBrowseUrl(url, &client);
- // The database is available so the check shouldn't get queued.
- EXPECT_TRUE(GetQueuedChecks().empty());
-
- ResetV4Database();
- v4_local_database_manager_->CheckBrowseUrl(url, &client);
// The database is unavailable so the check should get queued.
EXPECT_EQ(1ul, GetQueuedChecks().size());
- // The following function calls StartOnIOThread which should load the
- // database from disk and cause the queued check to be performed.
- StartLocalDatabaseManager();
+ // The following function waits for the DB to load.
+ WaitForTasksOnTaskRunner();
EXPECT_TRUE(GetQueuedChecks().empty());
ResetV4Database();
@@ -217,4 +429,296 @@ TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) {
EXPECT_TRUE(GetQueuedChecks().empty());
}
+// Verify that a window where checks cannot be cancelled is closed.
+TEST_F(V4LocalDatabaseManagerTest, CancelPending) {
+ // Setup to receive full-hash misses.
+ ScopedFakeGetHashProtocolManagerFactory pin;
+
+ // Reset the database manager so it picks up the replacement protocol manager.
+ ResetLocalDatabaseManager();
+ WaitForTasksOnTaskRunner();
+
+ // An URL and matching prefix.
+ const GURL url("http://example.com/a/");
+ const HashPrefix hash_prefix("eW\x1A\xF\xA9");
+
+ // Put a match in the db that will cause a protocol-manager request.
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), hash_prefix);
+ ReplaceV4Database(store_and_hash_prefixes);
+
+ // Test that a request flows through to the callback.
+ {
+ TestClient client(SB_THREAT_TYPE_SAFE, url);
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client));
+ EXPECT_FALSE(client.on_check_browse_url_result_called_);
+ WaitForTasksOnTaskRunner();
+ EXPECT_TRUE(client.on_check_browse_url_result_called_);
+ }
+
+ // Test that cancel prevents the callback from being called.
+ {
+ TestClient client(SB_THREAT_TYPE_SAFE, url);
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client));
+ v4_local_database_manager_->CancelCheck(&client);
+ EXPECT_FALSE(client.on_check_browse_url_result_called_);
+ WaitForTasksOnTaskRunner();
+ EXPECT_FALSE(client.on_check_browse_url_result_called_);
+ }
+}
+
+// When the database load flushes the queued requests, make sure that
+// CancelCheck() is not fatal in the client callback.
+TEST_F(V4LocalDatabaseManagerTest, CancelQueued) {
+ const GURL url("http://example.com/a/");
+
+ TestClient client1(SB_THREAT_TYPE_SAFE, url,
+ v4_local_database_manager_.get());
+ TestClient client2(SB_THREAT_TYPE_SAFE, url);
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client1));
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client2));
+ EXPECT_EQ(2ul, GetQueuedChecks().size());
+ EXPECT_FALSE(client1.on_check_browse_url_result_called_);
+ EXPECT_FALSE(client2.on_check_browse_url_result_called_);
+ WaitForTasksOnTaskRunner();
+ EXPECT_TRUE(client1.on_check_browse_url_result_called_);
+ EXPECT_TRUE(client2.on_check_browse_url_result_called_);
+}
+
+// This test is somewhat similar to TestCheckBrowseUrlWithFakeDbReturnsMatch but
+// it uses a fake V4LocalDatabaseManager to assert that PerformFullHashCheck is
+// called async.
+TEST_F(V4LocalDatabaseManagerTest, PerformFullHashCheckCalledAsync) {
+ SetupFakeManager();
+ net::TestURLFetcherFactory factory;
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetUrlMalwareId(),
+ HashPrefix("eW\x1A\xF\xA9"));
+ ReplaceV4Database(store_and_hash_prefixes);
+
+ // The fake database returns a matched hash prefix.
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(
+ GURL("http://example.com/a/"), nullptr));
+
+ EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+
+ // Wait for PerformFullHashCheck to complete.
+ WaitForTasksOnTaskRunner();
+
+ EXPECT_TRUE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+}
+
+TEST_F(V4LocalDatabaseManagerTest, UsingWeakPtrDropsCallback) {
+ SetupFakeManager();
+ net::TestURLFetcherFactory factory;
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetUrlMalwareId(),
+ HashPrefix("eW\x1A\xF\xA9"));
+ ReplaceV4Database(store_and_hash_prefixes);
+
+ // The fake database returns a matched hash prefix.
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(
+ GURL("http://example.com/a/"), nullptr));
+ v4_local_database_manager_->StopOnIOThread(true);
+
+ // Release the V4LocalDatabaseManager object right away before the callback
+ // gets called. When the callback gets called, without using a weak-ptr
+ // factory, this leads to a use after free. However, using the weak-ptr means
+ // that the callback is simply dropped.
+ v4_local_database_manager_ = nullptr;
+
+ // Wait for the tasks scheduled by StopOnIOThread to complete.
+ WaitForTasksOnTaskRunner();
+}
+
+TEST_F(V4LocalDatabaseManagerTest, TestMatchCsdWhitelistUrl) {
+ SetupFakeManager();
+ GURL good_url("http://safe.com");
+ GURL other_url("http://iffy.com");
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetUrlCsdWhitelistId(),
+ HashForUrl(good_url));
+
+ ReplaceV4Database(store_and_hash_prefixes, false /* not available */);
+ // No match, but since we never loaded the whitelist (not available),
+ // it defaults to true.
+ EXPECT_TRUE(v4_local_database_manager_->MatchCsdWhitelistUrl(good_url));
+
+ ReplaceV4Database(store_and_hash_prefixes, true /* available */);
+ // Not whitelisted.
+ EXPECT_FALSE(v4_local_database_manager_->MatchCsdWhitelistUrl(other_url));
+ // Whitelisted.
+ EXPECT_TRUE(v4_local_database_manager_->MatchCsdWhitelistUrl(good_url));
+
+ EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+}
+
+TEST_F(V4LocalDatabaseManagerTest, TestMatchDownloadWhitelistString) {
+ SetupFakeManager();
+ FullHash good_hash(crypto::SHA256HashString("Good .exe contents"));
+ FullHash other_hash(crypto::SHA256HashString("Other .exe contents"));
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetCertCsdDownloadWhitelistId(),
+ good_hash);
+
+ ReplaceV4Database(store_and_hash_prefixes, false /* not available */);
+ // Verify it defaults to false when DB is not available.
+ EXPECT_FALSE(
+ v4_local_database_manager_->MatchDownloadWhitelistString(good_hash));
+
+ ReplaceV4Database(store_and_hash_prefixes, true /* available */);
+ // Not whitelisted.
+ EXPECT_FALSE(
+ v4_local_database_manager_->MatchDownloadWhitelistString(other_hash));
+ // Whitelisted.
+ EXPECT_TRUE(
+ v4_local_database_manager_->MatchDownloadWhitelistString(good_hash));
+
+ EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+}
+
+TEST_F(V4LocalDatabaseManagerTest, TestMatchDownloadWhitelistUrl) {
+ SetupFakeManager();
+ GURL good_url("http://safe.com");
+ GURL other_url("http://iffy.com");
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetCertCsdDownloadWhitelistId(),
+ HashForUrl(good_url));
+
+ ReplaceV4Database(store_and_hash_prefixes, false /* not available */);
+ // Verify it defaults to false when DB is not available.
+ EXPECT_FALSE(v4_local_database_manager_->MatchDownloadWhitelistUrl(good_url));
+
+ ReplaceV4Database(store_and_hash_prefixes, true /* available */);
+ // Not whitelisted.
+ EXPECT_FALSE(
+ v4_local_database_manager_->MatchDownloadWhitelistUrl(other_url));
+ // Whitelisted.
+ EXPECT_TRUE(v4_local_database_manager_->MatchDownloadWhitelistUrl(good_url));
+
+ EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+}
+
+TEST_F(V4LocalDatabaseManagerTest, TestMatchMalwareIP) {
+ SetupFakeManager();
+
+ // >>> hashlib.sha1(socket.inet_pton(socket.AF_INET6,
+ // '::ffff:192.168.1.2')).digest() + chr(128)
+ // '\xb3\xe0z\xafAv#h\x9a\xcf<\xf3ee\x94\xda\xf6y\xb1\xad\x80'
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetIpMalwareId(),
+ FullHash("\xB3\xE0z\xAF"
+ "Av#h\x9A\xCF<\xF3"
+ "ee\x94\xDA\xF6y\xB1\xAD\x80"));
+ ReplaceV4Database(store_and_hash_prefixes);
+
+ EXPECT_FALSE(v4_local_database_manager_->MatchMalwareIP(""));
+ // Not blacklisted.
+ EXPECT_FALSE(v4_local_database_manager_->MatchMalwareIP("192.168.1.1"));
+ // Blacklisted.
+ EXPECT_TRUE(v4_local_database_manager_->MatchMalwareIP("192.168.1.2"));
+
+ EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+}
+
+TEST_F(V4LocalDatabaseManagerTest, TestMatchModuleWhitelist) {
+ SetupFakeManager();
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetChromeFilenameClientIncidentId(),
+ crypto::SHA256HashString("chrome.dll"));
+
+ ReplaceV4Database(store_and_hash_prefixes, false /* not available */);
+ // No match, but since we never loaded the whitelist (not available),
+ // it defaults to true.
+ EXPECT_TRUE(
+ v4_local_database_manager_->MatchModuleWhitelistString("badstuff.dll"));
+
+ ReplaceV4Database(store_and_hash_prefixes, true /* available */);
+ // Not whitelisted.
+ EXPECT_FALSE(
+ v4_local_database_manager_->MatchModuleWhitelistString("badstuff.dll"));
+ // Whitelisted.
+ EXPECT_TRUE(
+ v4_local_database_manager_->MatchModuleWhitelistString("chrome.dll"));
+
+ EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled(
+ v4_local_database_manager_));
+}
+
+// This verifies the fix for race in http://crbug.com/660293
+TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithSameClientAndCancel) {
+ ScopedFakeGetHashProtocolManagerFactory pin;
+ // Reset the database manager so it picks up the replacement protocol manager.
+ ResetLocalDatabaseManager();
+ WaitForTasksOnTaskRunner();
+
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetUrlMalwareId(),
+ HashPrefix("sÙ†\340\t\006_"));
+ ReplaceV4Database(store_and_hash_prefixes);
+
+ GURL first_url("http://example.com/a");
+ GURL second_url("http://example.com/");
+ TestClient client(SB_THREAT_TYPE_SAFE, first_url);
+ // The fake database returns a matched hash prefix.
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(first_url, &client));
+
+ // That check gets queued. Now, let's cancel the check. After this, we should
+ // not receive a call for |OnCheckBrowseUrlResult| with |first_url|.
+ v4_local_database_manager_->CancelCheck(&client);
+
+ // Now, re-use that client but for |second_url|.
+ client.expected_url = second_url;
+ EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(second_url, &client));
+
+ // Wait for PerformFullHashCheck to complete.
+ WaitForTasksOnTaskRunner();
+ // |on_check_browse_url_result_called_| is true only if OnCheckBrowseUrlResult
+ // gets called with the |url| equal to |expected_url|, which is |second_url|
+ // in
+ // this test.
+ EXPECT_TRUE(client.on_check_browse_url_result_called_);
+}
+
+TEST_F(V4LocalDatabaseManagerTest, TestCheckResourceUrl) {
+ // Setup to receive full-hash misses.
+ ScopedFakeGetHashProtocolManagerFactory pin;
+
+ // Reset the database manager so it picks up the replacement protocol manager.
+ ResetLocalDatabaseManager();
+ WaitForTasksOnTaskRunner();
+
+ // An URL and matching prefix.
+ const GURL url("http://example.com/a/");
+ const HashPrefix hash_prefix("eW\x1A\xF\xA9");
+
+ // Put a match in the db that will cause a protocol-manager request.
+ StoreAndHashPrefixes store_and_hash_prefixes;
+ store_and_hash_prefixes.emplace_back(GetChromeUrlClientIncidentId(),
+ hash_prefix);
+ ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */);
+
+ TestClient client(SB_THREAT_TYPE_SAFE, url);
+ EXPECT_FALSE(v4_local_database_manager_->CheckResourceUrl(url, &client));
+ EXPECT_FALSE(client.on_check_resource_url_result_called_);
+ WaitForTasksOnTaskRunner();
+ EXPECT_TRUE(client.on_check_resource_url_result_called_);
+}
+
+// TODO(nparker): Add tests for
+// CheckDownloadUrl()
+// CheckExtensionIDs()
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc b/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc
index c349b6b9d8f..dad0d7917cd 100644
--- a/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc
+++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc
@@ -5,12 +5,14 @@
#include "components/safe_browsing_db/v4_protocol_manager_util.h"
#include "base/base64.h"
-#include "base/metrics/sparse_histogram.h"
+#include "base/metrics/histogram_macros.h"
#include "base/rand_util.h"
+#include "base/sha1.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "crypto/sha2.h"
#include "net/base/escape.h"
+#include "net/base/ip_address.h"
#include "net/http/http_request_headers.h"
#include "url/url_util.h"
@@ -85,19 +87,55 @@ return LINUX_PLATFORM;
#endif
}
+const ListIdentifier GetCertCsdDownloadWhitelistId() {
+ return ListIdentifier(GetCurrentPlatformType(), CERT, CSD_DOWNLOAD_WHITELIST);
+}
+
+const ListIdentifier GetChromeExtMalwareId() {
+ return ListIdentifier(CHROME_PLATFORM, CHROME_EXTENSION, MALWARE_THREAT);
+}
+
const ListIdentifier GetChromeUrlApiId() {
return ListIdentifier(CHROME_PLATFORM, URL, API_ABUSE);
}
+const ListIdentifier GetChromeFilenameClientIncidentId() {
+ return ListIdentifier(CHROME_PLATFORM, FILENAME, CLIENT_INCIDENT);
+}
+
+const ListIdentifier GetChromeUrlClientIncidentId() {
+ return ListIdentifier(CHROME_PLATFORM, URL, CLIENT_INCIDENT);
+}
+
+const ListIdentifier GetIpMalwareId() {
+ return ListIdentifier(GetCurrentPlatformType(), IP_RANGE, MALWARE_THREAT);
+}
+
+const ListIdentifier GetUrlCsdDownloadWhitelistId() {
+ return ListIdentifier(GetCurrentPlatformType(), URL, CSD_DOWNLOAD_WHITELIST);
+}
+
+const ListIdentifier GetUrlCsdWhitelistId() {
+ return ListIdentifier(GetCurrentPlatformType(), URL, CSD_WHITELIST);
+}
+
const ListIdentifier GetUrlMalwareId() {
return ListIdentifier(GetCurrentPlatformType(), URL, MALWARE_THREAT);
}
+const ListIdentifier GetUrlMalBinId() {
+ return ListIdentifier(GetCurrentPlatformType(), URL, MALICIOUS_BINARY);
+}
+
const ListIdentifier GetUrlSocEngId() {
return ListIdentifier(GetCurrentPlatformType(), URL,
SOCIAL_ENGINEERING_PUBLIC);
}
+const ListIdentifier GetUrlUwsId() {
+ return ListIdentifier(GetCurrentPlatformType(), URL, UNWANTED_SOFTWARE);
+}
+
// The Safe Browsing V4 server URL prefix.
const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4";
@@ -159,7 +197,14 @@ ListIdentifier::ListIdentifier(const ListUpdateResponse& response)
response.threat_entry_type(),
response.threat_type()) {}
-V4ProtocolConfig::V4ProtocolConfig() : disable_auto_update(false) {}
+V4ProtocolConfig::V4ProtocolConfig(const std::string& client_name,
+ bool disable_auto_update,
+ const std::string& key_param,
+ const std::string& version)
+ : client_name(client_name),
+ disable_auto_update(disable_auto_update),
+ key_param(key_param),
+ version(version) {}
V4ProtocolConfig::V4ProtocolConfig(const V4ProtocolConfig& other) = default;
@@ -237,7 +282,7 @@ void V4ProtocolManagerUtil::UpdateHeaders(net::HttpRequestHeaders* headers) {
// static
void V4ProtocolManagerUtil::UrlToFullHashes(
const GURL& url,
- std::unordered_set<FullHash>* full_hashes) {
+ std::vector<FullHash>* full_hashes) {
std::string canon_host, canon_path, canon_query;
CanonicalizeUrl(url, &canon_host, &canon_path, &canon_query);
@@ -252,7 +297,7 @@ void V4ProtocolManagerUtil::UrlToFullHashes(
GeneratePathVariantsToCheck(canon_path, canon_query, &paths);
for (const std::string& host : hosts) {
for (const std::string& path : paths) {
- full_hashes->insert(crypto::SHA256HashString(host + path));
+ full_hashes->push_back(crypto::SHA256HashString(host + path));
}
}
}
@@ -505,4 +550,37 @@ void V4ProtocolManagerUtil::SetClientInfoFromConfig(
client_info->set_client_version(config.version);
}
+// static
+bool V4ProtocolManagerUtil::GetIPV6AddressFromString(
+ const std::string& ip_address,
+ net::IPAddress* address) {
+ DCHECK(address);
+ if (!address->AssignFromIPLiteral(ip_address))
+ return false;
+ if (address->IsIPv4())
+ *address = net::ConvertIPv4ToIPv4MappedIPv6(*address);
+ return address->IsIPv6();
+}
+
+// static
+bool V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash(
+ const std::string& ip_address,
+ FullHash* hashed_encoded_ip) {
+ net::IPAddress address;
+ if (!GetIPV6AddressFromString(ip_address, &address)) {
+ return false;
+ }
+ std::string packed_ip = net::IPAddressToPackedString(address);
+ if (packed_ip.empty()) {
+ return false;
+ }
+
+ const std::string hash = base::SHA1HashString(packed_ip);
+ DCHECK_EQ(20u, hash.size());
+ hashed_encoded_ip->resize(hash.size() + 1);
+ hashed_encoded_ip->replace(0, hash.size(), hash);
+ (*hashed_encoded_ip)[hash.size()] = static_cast<unsigned char>(128);
+ return true;
+}
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util.h b/chromium/components/safe_browsing_db/v4_protocol_manager_util.h
index 34e24125d7e..707b82da144 100644
--- a/chromium/components/safe_browsing_db/v4_protocol_manager_util.h
+++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util.h
@@ -20,6 +20,7 @@
namespace net {
class HttpRequestHeaders;
+class IPAddress;
} // namespace net
namespace safe_browsing {
@@ -49,23 +50,33 @@ struct V4ProtocolConfig {
// The safe browsing client name sent in each request.
std::string client_name;
- // Current product version sent in each request.
- std::string version;
+ // Disable auto-updates using a command line switch.
+ bool disable_auto_update;
// The Google API key.
std::string key_param;
- // Disable auto-updates using a command line switch?
- bool disable_auto_update;
+ // Current product version sent in each request.
+ std::string version;
- V4ProtocolConfig();
+ V4ProtocolConfig(const std::string& client_name,
+ bool disable_auto_update,
+ const std::string& key_param,
+ const std::string& version);
V4ProtocolConfig(const V4ProtocolConfig& other);
~V4ProtocolConfig();
+
+ private:
+ V4ProtocolConfig();
};
// Different types of threats that SafeBrowsing protects against. This is the
// type that's returned to the clients of SafeBrowsing in Chromium.
enum SBThreatType {
+ // This type can be used for lists that can be checked synchronously so a
+ // client callback isn't required, or for whitelists.
+ SB_THREAT_TYPE_UNUSED,
+
// No threat at all.
SB_THREAT_TYPE_SAFE,
@@ -130,9 +141,18 @@ struct ListIdentifier {
std::ostream& operator<<(std::ostream& os, const ListIdentifier& id);
PlatformType GetCurrentPlatformType();
+const ListIdentifier GetCertCsdDownloadWhitelistId();
+const ListIdentifier GetChromeExtMalwareId();
const ListIdentifier GetChromeUrlApiId();
+const ListIdentifier GetChromeFilenameClientIncidentId();
+const ListIdentifier GetChromeUrlClientIncidentId();
+const ListIdentifier GetIpMalwareId();
+const ListIdentifier GetUrlCsdDownloadWhitelistId();
+const ListIdentifier GetUrlCsdWhitelistId();
const ListIdentifier GetUrlMalwareId();
+const ListIdentifier GetUrlMalBinId();
const ListIdentifier GetUrlSocEngId();
+const ListIdentifier GetUrlUwsId();
// Represents the state of each store.
typedef base::hash_map<ListIdentifier, std::string> StoreStateMap;
@@ -140,7 +160,6 @@ typedef base::hash_map<ListIdentifier, std::string> StoreStateMap;
// Sever response, parsed in vector form.
typedef std::vector<std::unique_ptr<ListUpdateResponse>> ParsedServerResponse;
-// TODO(vakh): Consider using a std::pair for this.
// Holds the hash prefix and the store that it matched in.
struct StoreAndHashPrefix {
public:
@@ -248,7 +267,7 @@ class V4ProtocolManagerUtil {
// Generate the set of FullHashes to check for |url|.
static void UrlToFullHashes(const GURL& url,
- std::unordered_set<FullHash>* full_hashes);
+ std::vector<FullHash>* full_hashes);
static bool FullHashToHashPrefix(const FullHash& full_hash,
PrefixSize prefix_size,
@@ -263,6 +282,16 @@ class V4ProtocolManagerUtil {
static void SetClientInfoFromConfig(ClientInfo* client_info,
const V4ProtocolConfig& config);
+ static bool GetIPV6AddressFromString(const std::string& ip_address,
+ net::IPAddress* address);
+
+ // Converts a IPV4 or IPV6 address in |ip_address| to the SHA1 hash of the
+ // corresponding packed IPV6 address in |hashed_encoded_ip|, and adds an
+ // extra byte containing the value 128 at the end. This is done to match the
+ // server implementation for calculating the hash prefix of an IP address.
+ static bool IPAddressToEncodedIPV6Hash(const std::string& ip_address,
+ FullHash* hashed_encoded_ip);
+
private:
V4ProtocolManagerUtil(){};
FRIEND_TEST_ALL_PREFIXES(V4ProtocolManagerUtilTest, TestBackOffLogic);
diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc b/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc
index 20e808576bd..0680d3cc060 100644
--- a/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc
+++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc
@@ -9,6 +9,7 @@
#include "base/base64.h"
#include "base/strings/stringprintf.h"
#include "base/time/time.h"
+#include "components/safe_browsing_db/v4_test_util.h"
#include "net/base/escape.h"
#include "net/http/http_request_headers.h"
#include "testing/gtest/include/gtest/gtest.h"
@@ -18,10 +19,6 @@ using base::TimeDelta;
namespace {
-const char kClient[] = "unittest";
-const char kAppVer[] = "1.0";
-const char kKeyParam[] = "test_key_param";
-
bool VectorContains(const std::vector<std::string>& data,
const std::string& str) {
return std::find(data.begin(), data.end(), str) != data.end();
@@ -32,12 +29,6 @@ bool VectorContains(const std::vector<std::string>& data,
namespace safe_browsing {
class V4ProtocolManagerUtilTest : public testing::Test {
- protected:
- void PopulateV4ProtocolConfig(V4ProtocolConfig* config) {
- config->client_name = kClient;
- config->version = kAppVer;
- config->key_param = kKeyParam;
- }
};
TEST_F(V4ProtocolManagerUtilTest, TestBackOffLogic) {
@@ -115,13 +106,11 @@ TEST_F(V4ProtocolManagerUtilTest, TestBackOffLogic) {
}
TEST_F(V4ProtocolManagerUtilTest, TestGetRequestUrlAndUpdateHeaders) {
- V4ProtocolConfig config;
- PopulateV4ProtocolConfig(&config);
-
net::HttpRequestHeaders headers;
GURL gurl;
V4ProtocolManagerUtil::GetRequestUrlAndHeaders("request_base64", "someMethod",
- config, &gurl, &headers);
+ GetTestV4ProtocolConfig(),
+ &gurl, &headers);
std::string expectedUrl =
"https://safebrowsing.googleapis.com/v4/someMethod?"
"$req=request_base64&$ct=application/x-protobuf&key=test_key_param";
@@ -249,4 +238,35 @@ TEST_F(V4ProtocolManagerUtilTest, CanonicalizeUrl) {
}
}
+TEST_F(V4ProtocolManagerUtilTest, TestIPAddressToEncodedIPV6) {
+ // To verify the test values, here's the python code:
+ // >> import socket, hashlib, binascii
+ // >> hashlib.sha1(socket.inet_pton(socket.AF_INET6, input)).digest() +
+ // chr(128)
+ // For example:
+ // >>> hashlib.sha1(socket.inet_pton(socket.AF_INET6,
+ // '::ffff:192.168.1.1')).digest() + chr(128)
+ // 'X\xf8\xa1\x17I\xe6Pl\xfd\xdb\xbb\xa0\x0c\x02\x9d#\n|\xe7\xcd\x80'
+ std::vector<std::tuple<bool, std::string, std::string>> test_cases = {
+ std::make_tuple(false, "", ""),
+ std::make_tuple(
+ true, "192.168.1.1",
+ "X\xF8\xA1\x17I\xE6Pl\xFD\xDB\xBB\xA0\f\x2\x9D#\n|\xE7\xCD\x80"),
+ std::make_tuple(
+ true, "::",
+ "\xE1)\xF2|Q\x3\xBC\\\xC4K\xCD\xF0\xA1^\x16\rDPf\xFF\x80")};
+ for (size_t i = 0; i < test_cases.size(); i++) {
+ DVLOG(1) << "Running case: " << i;
+ bool success = std::get<0>(test_cases[i]);
+ const auto& input = std::get<1>(test_cases[i]);
+ const auto& expected_output = std::get<2>(test_cases[i]);
+ std::string encoded_ip;
+ ASSERT_EQ(success, V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash(
+ input, &encoded_ip));
+ if (success) {
+ ASSERT_EQ(expected_output, encoded_ip);
+ }
+ }
+}
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_store.cc b/chromium/components/safe_browsing_db/v4_store.cc
index d00573156ef..889c55c4b66 100644
--- a/chromium/components/safe_browsing_db/v4_store.cc
+++ b/chromium/components/safe_browsing_db/v4_store.cc
@@ -22,8 +22,28 @@ namespace safe_browsing {
namespace {
-const uint32_t kFileMagic = 0x600D71FE;
+// UMA related strings.
+// Part 1: Represent the overall operation being performed.
+const char kProcessFullUpdate[] = "SafeBrowsing.V4ProcessFullUpdate";
+const char kProcessPartialUpdate[] = "SafeBrowsing.V4ProcessPartialUpdate";
+const char kReadFromDisk[] = "SafeBrowsing.V4ReadFromDisk";
+// Part 2: Represent the sub-operation being performed as part of the larger
+// operation from part 1.
+const char kApplyUpdate[] = ".ApplyUpdate";
+const char kDecodeAdditions[] = ".DecodeAdditions";
+const char kDecodeRemovals[] = ".DecodeRemovals";
+const char kMergeUpdate[] = ".MergeUpdate";
+// Part 3: Represent the unit of value being measured and logged.
+const char kResult[] = ".Result";
+const char kTime[] = ".Time";
+// Part 4 (optional): Represent the name of the list for which the metric is
+// being logged. For instance, ".UrlSoceng".
+// UMA metric names for this file are generated by appending one value each,
+// in order, from parts [1, 2, and 3], or [1, 2, 3, and 4]. For example:
+// SafeBrowsing.V4ProcessPartialUpdate.ApplyUpdate.Result, or
+// SafeBrowsing.V4ProcessPartialUpdate.ApplyUpdate.Result.UrlSoceng
+const uint32_t kFileMagic = 0x600D71FE;
const uint32_t kFileVersion = 9;
std::string GetUmaSuffixForStore(const base::FilePath& file_path) {
@@ -31,26 +51,25 @@ std::string GetUmaSuffixForStore(const base::FilePath& file_path) {
".%" PRIsFP, file_path.BaseName().RemoveExtension().value().c_str());
}
-void RecordTimeWithAndWithoutStore(const std::string& metric,
- base::TimeDelta time,
- const base::FilePath& file_path) {
- std::string suffix = GetUmaSuffixForStore(file_path);
-
+void RecordTimeWithAndWithoutSuffix(const std::string& metric,
+ base::TimeDelta time,
+ const base::FilePath& file_path) {
// The histograms below are a modified expansion of the
// UMA_HISTOGRAM_LONG_TIMES macro adapted to allow for a dynamically suffixed
// histogram name.
// Note: The factory creates and owns the histogram.
const int kBucketCount = 100;
base::HistogramBase* histogram = base::Histogram::FactoryTimeGet(
- metric, base::TimeDelta::FromMilliseconds(1),
+ metric + kTime, base::TimeDelta::FromMilliseconds(1),
base::TimeDelta::FromMinutes(1), kBucketCount,
base::HistogramBase::kUmaTargetedHistogramFlag);
if (histogram) {
histogram->AddTime(time);
}
+ std::string suffix = GetUmaSuffixForStore(file_path);
base::HistogramBase* histogram_suffix = base::Histogram::FactoryTimeGet(
- metric + suffix, base::TimeDelta::FromMilliseconds(1),
+ metric + kTime + suffix, base::TimeDelta::FromMilliseconds(1),
base::TimeDelta::FromMinutes(1), kBucketCount,
base::HistogramBase::kUmaTargetedHistogramFlag);
if (histogram_suffix) {
@@ -58,74 +77,87 @@ void RecordTimeWithAndWithoutStore(const std::string& metric,
}
}
+void RecordEnumWithAndWithoutSuffix(const std::string& metric,
+ int32_t value,
+ int32_t maximum,
+ const base::FilePath& file_path) {
+ // The histograms below are an expansion of the UMA_HISTOGRAM_ENUMERATION
+ // macro adapted to allow for a dynamically suffixed histogram name.
+ // Note: The factory creates and owns the histogram.
+ base::HistogramBase* histogram = base::LinearHistogram::FactoryGet(
+ metric + kResult, 1, maximum, maximum + 1,
+ base::HistogramBase::kUmaTargetedHistogramFlag);
+ if (histogram) {
+ histogram->Add(value);
+ }
+
+ std::string suffix = GetUmaSuffixForStore(file_path);
+ base::HistogramBase* histogram_suffix = base::LinearHistogram::FactoryGet(
+ metric + kResult + suffix, 1, maximum, maximum + 1,
+ base::HistogramBase::kUmaTargetedHistogramFlag);
+ if (histogram_suffix) {
+ histogram_suffix->Add(value);
+ }
+}
+
void RecordAddUnlumpedHashesTime(base::TimeDelta time) {
- UMA_HISTOGRAM_LONG_TIMES("SafeBrowsing.V4AddUnlumpedHashesTime", time);
+ UMA_HISTOGRAM_LONG_TIMES("SafeBrowsing.V4AddUnlumpedHashes.Time", time);
}
-void RecordApplyUpdateResult(ApplyUpdateResult result) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4ApplyUpdateResult", result,
- APPLY_UPDATE_RESULT_MAX);
+void RecordApplyUpdateResult(const std::string& base_metric,
+ ApplyUpdateResult result,
+ const base::FilePath& file_path) {
+ RecordEnumWithAndWithoutSuffix(base_metric + kApplyUpdate, result,
+ APPLY_UPDATE_RESULT_MAX, file_path);
}
-void RecordApplyUpdateResultWhenReadingFromDisk(ApplyUpdateResult result) {
- UMA_HISTOGRAM_ENUMERATION(
- "SafeBrowsing.V4ApplyUpdateResultWhenReadingFromDisk", result,
- APPLY_UPDATE_RESULT_MAX);
+void RecordApplyUpdateTime(const std::string& base_metric,
+ base::TimeDelta time,
+ const base::FilePath& file_path) {
+ RecordTimeWithAndWithoutSuffix(base_metric + kApplyUpdate, time, file_path);
}
-void RecordDecodeAdditionsResult(V4DecodeResult result) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4DecodeAdditionsResult", result,
- DECODE_RESULT_MAX);
+void RecordDecodeAdditionsResult(const std::string& base_metric,
+ V4DecodeResult result,
+ const base::FilePath& file_path) {
+ RecordEnumWithAndWithoutSuffix(base_metric + kDecodeAdditions, result,
+ DECODE_RESULT_MAX, file_path);
}
-void RecordDecodeAdditionsTime(base::TimeDelta time,
+void RecordDecodeAdditionsTime(const std::string& base_metric,
+ base::TimeDelta time,
const base::FilePath& file_path) {
- RecordTimeWithAndWithoutStore("SafeBrowsing.V4DecodeAdditionsTime", time,
- file_path);
+ RecordTimeWithAndWithoutSuffix(base_metric + kDecodeAdditions, time,
+ file_path);
}
-void RecordDecodeRemovalsResult(V4DecodeResult result) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4DecodeRemovalsResult", result,
- DECODE_RESULT_MAX);
+void RecordDecodeRemovalsResult(const std::string& base_metric,
+ V4DecodeResult result,
+ const base::FilePath& file_path) {
+ RecordEnumWithAndWithoutSuffix(base_metric + kDecodeRemovals, result,
+ DECODE_RESULT_MAX, file_path);
}
-void RecordDecodeRemovalsTime(base::TimeDelta time,
+void RecordDecodeRemovalsTime(const std::string& base_metric,
+ base::TimeDelta time,
const base::FilePath& file_path) {
- RecordTimeWithAndWithoutStore("SafeBrowsing.V4DecodeRemovalsTime", time,
- file_path);
+ RecordTimeWithAndWithoutSuffix(base_metric + kDecodeRemovals, time,
+ file_path);
}
-void RecordMergeUpdateTime(base::TimeDelta time,
+void RecordMergeUpdateTime(const std::string& base_metric,
+ base::TimeDelta time,
const base::FilePath& file_path) {
- RecordTimeWithAndWithoutStore("SafeBrowsing.V4MergeUpdateTime", time,
- file_path);
-}
-
-void RecordProcessFullUpdateTime(base::TimeDelta time,
- const base::FilePath& file_path) {
- RecordTimeWithAndWithoutStore("SafeBrowsing.V4ProcessFullUpdateTime", time,
- file_path);
-}
-
-void RecordProcessPartialUpdateTime(base::TimeDelta time,
- const base::FilePath& file_path) {
- RecordTimeWithAndWithoutStore("SafeBrowsing.V4ProcessPartialUpdateTime", time,
- file_path);
-}
-
-void RecordReadFromDiskTime(base::TimeDelta time,
- const base::FilePath& file_path) {
- RecordTimeWithAndWithoutStore("SafeBrowsing.V4ReadFromDiskTime", time,
- file_path);
+ RecordTimeWithAndWithoutSuffix(base_metric + kMergeUpdate, time, file_path);
}
void RecordStoreReadResult(StoreReadResult result) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreReadResult", result,
+ UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreRead.Result", result,
STORE_READ_RESULT_MAX);
}
void RecordStoreWriteResult(StoreWriteResult result) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreWriteResult", result,
+ UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreWrite.Result", result,
STORE_WRITE_RESULT_MAX);
}
@@ -149,7 +181,7 @@ std::ostream& operator<<(std::ostream& os, const V4Store& store) {
V4Store* V4StoreFactory::CreateV4Store(
const scoped_refptr<base::SequencedTaskRunner>& task_runner,
const base::FilePath& store_path) {
- V4Store* new_store = new V4Store(task_runner, store_path);
+ V4Store* new_store = new V4Store(task_runner, store_path, 0);
new_store->Initialize();
return new_store;
}
@@ -159,14 +191,25 @@ void V4Store::Initialize() {
DCHECK(state_.empty());
StoreReadResult store_read_result = ReadFromDisk();
+ has_valid_data_ = (store_read_result == READ_SUCCESS);
RecordStoreReadResult(store_read_result);
}
-V4Store::V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
- const base::FilePath& store_path)
- : store_path_(store_path), task_runner_(task_runner) {}
+bool V4Store::HasValidData() const {
+ return has_valid_data_;
+}
-V4Store::~V4Store() {}
+V4Store::V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
+ const base::FilePath& store_path,
+ const int64_t old_file_size)
+ : file_size_(old_file_size),
+ has_valid_data_(false),
+ store_path_(store_path),
+ task_runner_(task_runner) {}
+
+V4Store::~V4Store() {
+ DCHECK(task_runner_->RunsTasksOnCurrentThread());
+}
std::string V4Store::DebugString() const {
std::string state_base64;
@@ -176,41 +219,54 @@ std::string V4Store::DebugString() const {
store_path_.value().c_str(), state_base64.c_str());
}
-bool V4Store::Reset() {
- // TODO(vakh): Implement skeleton.
+// static
+void V4Store::Destroy(std::unique_ptr<V4Store> v4_store) {
+ V4Store* v4_store_raw = v4_store.release();
+ if (v4_store_raw) {
+ v4_store_raw->task_runner_->DeleteSoon(FROM_HERE, v4_store_raw);
+ }
+}
+
+void V4Store::Reset() {
+ expected_checksum_.clear();
+ hash_prefix_map_.clear();
state_ = "";
- return true;
}
ApplyUpdateResult V4Store::ProcessPartialUpdateAndWriteToDisk(
+ const std::string& metric,
const HashPrefixMap& hash_prefix_map_old,
std::unique_ptr<ListUpdateResponse> response) {
DCHECK(response->has_response_type());
DCHECK_EQ(ListUpdateResponse::PARTIAL_UPDATE, response->response_type());
- TimeTicks before = TimeTicks::Now();
- ApplyUpdateResult result = ProcessUpdate(hash_prefix_map_old, response);
+ ApplyUpdateResult result = ProcessUpdate(
+ metric, hash_prefix_map_old, response, false /* delay_checksum check */);
if (result == APPLY_UPDATE_SUCCESS) {
- RecordProcessPartialUpdateTime(TimeTicks::Now() - before, store_path_);
- // TODO(vakh): Create a ListUpdateResponse containing RICE encoded
- // hash prefixes and response_type as FULL_UPDATE, and write that to disk.
+ Checksum checksum = response->checksum();
+ response.reset();
+ RecordStoreWriteResult(WriteToDisk(checksum));
}
return result;
}
ApplyUpdateResult V4Store::ProcessFullUpdateAndWriteToDisk(
+ const std::string& metric,
std::unique_ptr<ListUpdateResponse> response) {
- TimeTicks before = TimeTicks::Now();
- ApplyUpdateResult result = ProcessFullUpdate(response);
+ ApplyUpdateResult result =
+ ProcessFullUpdate(metric, response, false /* delay_checksum check */);
if (result == APPLY_UPDATE_SUCCESS) {
- RecordStoreWriteResult(WriteToDisk(std::move(response)));
- RecordProcessFullUpdateTime(TimeTicks::Now() - before, store_path_);
+ Checksum checksum = response->checksum();
+ response.reset();
+ RecordStoreWriteResult(WriteToDisk(checksum));
}
return result;
}
ApplyUpdateResult V4Store::ProcessFullUpdate(
- const std::unique_ptr<ListUpdateResponse>& response) {
+ const std::string& metric,
+ const std::unique_ptr<ListUpdateResponse>& response,
+ bool delay_checksum_check) {
DCHECK(response->has_response_type());
DCHECK_EQ(ListUpdateResponse::FULL_UPDATE, response->response_type());
// TODO(vakh): For a full update, we don't need to process the update in
@@ -218,12 +274,14 @@ ApplyUpdateResult V4Store::ProcessFullUpdate(
// checksum. It might save some CPU cycles to store the full update as-is and
// walk the list of hash prefixes in lexographical order only for checksum
// calculation.
- return ProcessUpdate(HashPrefixMap(), response);
+ return ProcessUpdate(metric, HashPrefixMap(), response, delay_checksum_check);
}
ApplyUpdateResult V4Store::ProcessUpdate(
+ const std::string& metric,
const HashPrefixMap& hash_prefix_map_old,
- const std::unique_ptr<ListUpdateResponse>& response) {
+ const std::unique_ptr<ListUpdateResponse>& response,
+ bool delay_checksum_check) {
const RepeatedField<int32>* raw_removals = nullptr;
RepeatedField<int32> rice_removals;
size_t removals_size = response->removals_size();
@@ -243,11 +301,11 @@ ApplyUpdateResult V4Store::ProcessUpdate(
rice_indices.num_entries(), rice_indices.encoded_data(),
&rice_removals);
- RecordDecodeRemovalsResult(decode_result);
+ RecordDecodeRemovalsResult(metric, decode_result, store_path_);
if (decode_result != DECODE_SUCCESS) {
return RICE_DECODING_FAILURE;
}
- RecordDecodeRemovalsTime(TimeTicks::Now() - before, store_path_);
+ RecordDecodeRemovalsTime(metric, TimeTicks::Now() - before, store_path_);
raw_removals = &rice_removals;
} else {
NOTREACHED() << "Unexpected compression_type type: " << compression_type;
@@ -256,8 +314,8 @@ ApplyUpdateResult V4Store::ProcessUpdate(
}
HashPrefixMap hash_prefix_map;
- ApplyUpdateResult apply_update_result =
- UpdateHashPrefixMapFromAdditions(response->additions(), &hash_prefix_map);
+ ApplyUpdateResult apply_update_result = UpdateHashPrefixMapFromAdditions(
+ metric, response->additions(), &hash_prefix_map);
if (apply_update_result != APPLY_UPDATE_SUCCESS) {
return apply_update_result;
}
@@ -268,12 +326,27 @@ ApplyUpdateResult V4Store::ProcessUpdate(
}
TimeTicks before = TimeTicks::Now();
- apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map,
- raw_removals, expected_checksum);
- if (apply_update_result != APPLY_UPDATE_SUCCESS) {
- return apply_update_result;
+ if (delay_checksum_check) {
+ DCHECK(hash_prefix_map_old.empty());
+ DCHECK(!raw_removals);
+ // We delay the checksum check at startup to be able to load the DB
+ // quickly. In this case, the |hash_prefix_map_old| should be empty, so just
+ // copy over the |hash_prefix_map|.
+ hash_prefix_map_ = hash_prefix_map;
+
+ // Calculate the checksum asynchronously later and if it doesn't match,
+ // reset the store.
+ expected_checksum_ = expected_checksum;
+
+ apply_update_result = APPLY_UPDATE_SUCCESS;
+ } else {
+ apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map,
+ raw_removals, expected_checksum);
+ if (apply_update_result != APPLY_UPDATE_SUCCESS) {
+ return apply_update_result;
+ }
}
- RecordMergeUpdateTime(TimeTicks::Now() - before, store_path_);
+ RecordMergeUpdateTime(metric, TimeTicks::Now() - before, store_path_);
state_ = response->new_client_state();
return APPLY_UPDATE_SUCCESS;
@@ -284,14 +357,18 @@ void V4Store::ApplyUpdate(
const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner,
UpdatedStoreReadyCallback callback) {
std::unique_ptr<V4Store> new_store(
- new V4Store(this->task_runner_, this->store_path_));
+ new V4Store(task_runner_, store_path_, file_size_));
ApplyUpdateResult apply_update_result;
+ std::string metric;
+ TimeTicks before = TimeTicks::Now();
if (response->response_type() == ListUpdateResponse::PARTIAL_UPDATE) {
+ metric = kProcessPartialUpdate;
apply_update_result = new_store->ProcessPartialUpdateAndWriteToDisk(
- hash_prefix_map_, std::move(response));
+ metric, hash_prefix_map_, std::move(response));
} else if (response->response_type() == ListUpdateResponse::FULL_UPDATE) {
+ metric = kProcessFullUpdate;
apply_update_result =
- new_store->ProcessFullUpdateAndWriteToDisk(std::move(response));
+ new_store->ProcessFullUpdateAndWriteToDisk(metric, std::move(response));
} else {
apply_update_result = UNEXPECTED_RESPONSE_TYPE_FAILURE;
NOTREACHED() << "Failure: Unexpected response type: "
@@ -299,21 +376,26 @@ void V4Store::ApplyUpdate(
}
if (apply_update_result == APPLY_UPDATE_SUCCESS) {
- // new_store is done updating, pass it to the callback.
- callback_task_runner->PostTask(
- FROM_HERE, base::Bind(callback, base::Passed(&new_store)));
+ new_store->has_valid_data_ = true;
+ RecordApplyUpdateTime(metric, TimeTicks::Now() - before, store_path_);
} else {
- DVLOG(1) << "Failure: ApplyUpdate: reason: " << apply_update_result
- << "; store: " << *this;
- // new_store failed updating. Pass a nullptr to the callback.
- callback_task_runner->PostTask(FROM_HERE, base::Bind(callback, nullptr));
+ new_store.reset();
+ DLOG(WARNING) << "Failure: ApplyUpdate: reason: " << apply_update_result
+ << "; store: " << *this;
}
- RecordApplyUpdateResult(apply_update_result);
+ RecordApplyUpdateResult(metric, apply_update_result, store_path_);
+
+ // Posting the task should be the last thing to do in this function.
+ // Otherwise, the posted task can end up running in parallel. If that
+ // happens, the old store will get destoyed and can lead to use-after-free in
+ // this function.
+ callback_task_runner->PostTask(
+ FROM_HERE, base::Bind(callback, base::Passed(&new_store)));
}
-// static
ApplyUpdateResult V4Store::UpdateHashPrefixMapFromAdditions(
+ const std::string& metric,
const RepeatedPtrField<ThreatEntrySet>& additions,
HashPrefixMap* additions_map) {
for (const auto& addition : additions) {
@@ -335,11 +417,12 @@ ApplyUpdateResult V4Store::UpdateHashPrefixMapFromAdditions(
V4DecodeResult decode_result = V4RiceDecoder::DecodePrefixes(
rice_hashes.first_value(), rice_hashes.rice_parameter(),
rice_hashes.num_entries(), rice_hashes.encoded_data(), &raw_hashes);
- RecordDecodeAdditionsResult(decode_result);
+ RecordDecodeAdditionsResult(metric, decode_result, store_path_);
if (decode_result != DECODE_SUCCESS) {
return RICE_DECODING_FAILURE;
} else {
- RecordDecodeAdditionsTime(TimeTicks::Now() - before, store_path_);
+ RecordDecodeAdditionsTime(metric, TimeTicks::Now() - before,
+ store_path_);
char* raw_hashes_start = reinterpret_cast<char*>(raw_hashes.data());
size_t raw_hashes_size = sizeof(uint32_t) * raw_hashes.size();
@@ -450,7 +533,15 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
const HashPrefixMap& additions_map,
const RepeatedField<int32>* raw_removals,
const std::string& expected_checksum) {
+ DCHECK(task_runner_->RunsTasksOnCurrentThread());
DCHECK(hash_prefix_map_.empty());
+
+ bool calculate_checksum = !expected_checksum.empty();
+ if (calculate_checksum &&
+ (expected_checksum.size() != crypto::kSHA256Length)) {
+ return CHECKSUM_MISMATCH_FAILURE;
+ }
+
hash_prefix_map_.clear();
ReserveSpaceInPrefixMap(old_prefixes_map, &hash_prefix_map_);
ReserveSpaceInPrefixMap(additions_map, &hash_prefix_map_);
@@ -472,7 +563,6 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
// At least one of the maps still has elements that need to be merged into the
// new store.
- bool calculate_checksum = !expected_checksum.empty();
std::unique_ptr<crypto::SecureHash> checksum_ctx(
crypto::SecureHash::Create(crypto::SecureHash::SHA256));
@@ -502,7 +592,7 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
next_smallest_prefix_size = next_smallest_prefix_old.size();
// Update the iterator map, which means that we have merged one hash
- // prefix of size |next_size_for_old| from the old store.
+ // prefix of size |next_smallest_prefix_size| from the old store.
old_iterator_map[next_smallest_prefix_size] += next_smallest_prefix_size;
if (!raw_removals || removals_iter == raw_removals->end() ||
@@ -511,7 +601,7 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
hash_prefix_map_[next_smallest_prefix_size] += next_smallest_prefix_old;
if (calculate_checksum) {
- checksum_ctx->Update(base::string_as_array(&next_smallest_prefix_old),
+ checksum_ctx->Update(next_smallest_prefix_old.data(),
next_smallest_prefix_size);
}
} else {
@@ -532,9 +622,8 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
next_smallest_prefix_additions;
if (calculate_checksum) {
- checksum_ctx->Update(
- base::string_as_array(&next_smallest_prefix_additions),
- next_smallest_prefix_size);
+ checksum_ctx->Update(next_smallest_prefix_additions.data(),
+ next_smallest_prefix_size);
}
// Update the iterator map, which means that we have merged one hash
@@ -554,15 +643,21 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
}
if (calculate_checksum) {
- std::string checksum(crypto::kSHA256Length, 0);
- checksum_ctx->Finish(base::string_as_array(&checksum), checksum.size());
- if (checksum != expected_checksum) {
- std::string checksum_base64, expected_checksum_base64;
- base::Base64Encode(checksum, &checksum_base64);
- base::Base64Encode(expected_checksum, &expected_checksum_base64);
- DVLOG(1) << "Failure: Checksum mismatch: calculated: " << checksum_base64
- << " expected: " << expected_checksum_base64;
- return CHECKSUM_MISMATCH_FAILURE;
+ char checksum[crypto::kSHA256Length];
+ checksum_ctx->Finish(checksum, sizeof(checksum));
+ for (size_t i = 0; i < crypto::kSHA256Length; i++) {
+ if (checksum[i] != expected_checksum[i]) {
+#if DCHECK_IS_ON()
+ std::string checksum_b64, expected_checksum_b64;
+ base::Base64Encode(base::StringPiece(checksum, arraysize(checksum)),
+ &checksum_b64);
+ base::Base64Encode(expected_checksum, &expected_checksum_b64);
+ DVLOG(1) << "Failure: Checksum mismatch: calculated: " << checksum_b64
+ << "; expected: " << expected_checksum_b64
+ << "; store: " << *this;
+#endif
+ return CHECKSUM_MISMATCH_FAILURE;
+ }
}
}
@@ -572,20 +667,26 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
StoreReadResult V4Store::ReadFromDisk() {
DCHECK(task_runner_->RunsTasksOnCurrentThread());
+ V4StoreFileFormat file_format;
+ int64_t file_size;
TimeTicks before = TimeTicks::Now();
- std::string contents;
- bool read_success = base::ReadFileToString(store_path_, &contents);
- if (!read_success) {
- return FILE_UNREADABLE_FAILURE;
- }
+ {
+ // A temporary scope to make sure that |contents| get destroyed as soon as
+ // we are doing using it.
+ std::string contents;
+ bool read_success = base::ReadFileToString(store_path_, &contents);
+ if (!read_success) {
+ return FILE_UNREADABLE_FAILURE;
+ }
- if (contents.empty()) {
- return FILE_EMPTY_FAILURE;
- }
+ if (contents.empty()) {
+ return FILE_EMPTY_FAILURE;
+ }
- V4StoreFileFormat file_format;
- if (!file_format.ParseFromString(contents)) {
- return PROTO_PARSING_FAILURE;
+ if (!file_format.ParseFromString(contents)) {
+ return PROTO_PARSING_FAILURE;
+ }
+ file_size = static_cast<int64_t>(contents.size());
}
if (file_format.magic_number() != kFileMagic) {
@@ -604,39 +705,42 @@ StoreReadResult V4Store::ReadFromDisk() {
std::unique_ptr<ListUpdateResponse> response(new ListUpdateResponse);
response->Swap(file_format.mutable_list_update_response());
- ApplyUpdateResult apply_update_result = ProcessFullUpdate(response);
- RecordApplyUpdateResultWhenReadingFromDisk(apply_update_result);
+ ApplyUpdateResult apply_update_result = ProcessFullUpdate(
+ kReadFromDisk, response, true /* delay_checksum check */);
+ RecordApplyUpdateResult(kReadFromDisk, apply_update_result, store_path_);
if (apply_update_result != APPLY_UPDATE_SUCCESS) {
hash_prefix_map_.clear();
return HASH_PREFIX_MAP_GENERATION_FAILURE;
}
- RecordReadFromDiskTime(TimeTicks::Now() - before, store_path_);
+ RecordApplyUpdateTime(kReadFromDisk, TimeTicks::Now() - before, store_path_);
+
+ // Update |file_size_| now because we parsed the file correctly.
+ file_size_ = file_size;
return READ_SUCCESS;
}
-StoreWriteResult V4Store::WriteToDisk(
- std::unique_ptr<ListUpdateResponse> response) const {
- // Do not write partial updates to the disk.
- // After merging the updates, the ListUpdateResponse passed to this method
- // should be a FULL_UPDATE.
- if (!response->has_response_type() ||
- response->response_type() != ListUpdateResponse::FULL_UPDATE) {
- DVLOG(1) << "Failure: response->has_response_type(): "
- << response->has_response_type()
- << " : response->response_type(): " << response->response_type();
- return INVALID_RESPONSE_TYPE_FAILURE;
+StoreWriteResult V4Store::WriteToDisk(const Checksum& checksum) {
+ V4StoreFileFormat file_format;
+ ListUpdateResponse* lur = file_format.mutable_list_update_response();
+ *(lur->mutable_checksum()) = checksum;
+ lur->set_new_client_state(state_);
+ lur->set_response_type(ListUpdateResponse::FULL_UPDATE);
+ for (auto map_iter : hash_prefix_map_) {
+ ThreatEntrySet* additions = lur->add_additions();
+ // TODO(vakh): Write RICE encoded hash prefixes on disk. Not doing so
+ // currently since it takes a long time to decode them on startup, which
+ // blocks resource load. See: http://crbug.com/654819
+ additions->set_compression_type(RAW);
+ additions->mutable_raw_hashes()->set_prefix_size(map_iter.first);
+ additions->mutable_raw_hashes()->set_raw_hashes(map_iter.second);
}
// Attempt writing to a temporary file first and at the end, swap the files.
const base::FilePath new_filename = TemporaryFileForFilename(store_path_);
- V4StoreFileFormat file_format;
file_format.set_magic_number(kFileMagic);
file_format.set_version_number(kFileVersion);
- ListUpdateResponse* response_to_write =
- file_format.mutable_list_update_response();
- response_to_write->Swap(response.get());
std::string file_format_string;
file_format.SerializeToString(&file_format_string);
size_t written = base::WriteFile(new_filename, file_format_string.data(),
@@ -647,6 +751,9 @@ StoreWriteResult V4Store::WriteToDisk(
return UNABLE_TO_RENAME_FAILURE;
}
+ // Update |file_size_| now because we wrote the file correctly.
+ file_size_ = static_cast<int64_t>(written);
+
return WRITE_SUCCESS;
}
@@ -654,7 +761,7 @@ HashPrefix V4Store::GetMatchingHashPrefix(const FullHash& full_hash) {
// It should never be the case that more than one hash prefixes match a given
// full hash. However, if that happens, this method returns any one of them.
// It does not guarantee which one of those will be returned.
- DCHECK_EQ(32u, full_hash.size());
+ DCHECK(full_hash.size() == 32u || full_hash.size() == 21u);
for (const auto& pair : hash_prefix_map_) {
const PrefixSize& prefix_size = pair.first;
const HashPrefixes& hash_prefixes = pair.second;
@@ -690,4 +797,71 @@ bool V4Store::HashPrefixMatches(const HashPrefix& hash_prefix,
}
}
+bool V4Store::VerifyChecksum() {
+ DCHECK(task_runner_->RunsTasksOnCurrentThread());
+
+ if (expected_checksum_.empty()) {
+ // Nothing to check here folks!
+ // TODO(vakh): Do not allow empty checksums.
+ return true;
+ }
+
+ IteratorMap iterator_map;
+ HashPrefix next_smallest_prefix;
+ InitializeIteratorMap(hash_prefix_map_, &iterator_map);
+ bool has_unmerged = GetNextSmallestUnmergedPrefix(
+ hash_prefix_map_, iterator_map, &next_smallest_prefix);
+
+ std::unique_ptr<crypto::SecureHash> checksum_ctx(
+ crypto::SecureHash::Create(crypto::SecureHash::SHA256));
+ while (has_unmerged) {
+ PrefixSize next_smallest_prefix_size;
+ next_smallest_prefix_size = next_smallest_prefix.size();
+
+ // Update the iterator map, which means that we have read one hash
+ // prefix of size |next_smallest_prefix_size| from hash_prefix_map_.
+ iterator_map[next_smallest_prefix_size] += next_smallest_prefix_size;
+
+ checksum_ctx->Update(next_smallest_prefix.data(),
+ next_smallest_prefix_size);
+
+ // Find the next smallest unmerged element in the map.
+ has_unmerged = GetNextSmallestUnmergedPrefix(hash_prefix_map_, iterator_map,
+ &next_smallest_prefix);
+ }
+
+ char checksum[crypto::kSHA256Length];
+ checksum_ctx->Finish(checksum, sizeof(checksum));
+ for (size_t i = 0; i < crypto::kSHA256Length; i++) {
+ if (checksum[i] != expected_checksum_[i]) {
+ RecordApplyUpdateResult(kReadFromDisk, CHECKSUM_MISMATCH_FAILURE,
+ store_path_);
+#if DCHECK_IS_ON()
+ std::string checksum_b64, expected_checksum_b64;
+ base::Base64Encode(base::StringPiece(checksum, arraysize(checksum)),
+ &checksum_b64);
+ base::Base64Encode(expected_checksum_, &expected_checksum_b64);
+ DVLOG(1) << "Failure: Checksum mismatch: calculated: " << checksum_b64
+ << "; expected: " << expected_checksum_b64
+ << "; store: " << *this;
+#endif
+ return false;
+ }
+ }
+ return true;
+}
+
+int64_t V4Store::RecordAndReturnFileSize(const std::string& base_metric) {
+ std::string suffix = GetUmaSuffixForStore(store_path_);
+ // Histogram properties as in UMA_HISTOGRAM_COUNTS macro.
+ base::HistogramBase* histogram = base::Histogram::FactoryGet(
+ base_metric + suffix, 1, 1000000, 50,
+ base::HistogramBase::kUmaTargetedHistogramFlag);
+ if (histogram) {
+ const int64_t file_size_kilobytes = file_size_ / 1024;
+ histogram->Add(file_size_kilobytes);
+ }
+ return file_size_;
+}
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_store.h b/chromium/components/safe_browsing_db/v4_store.h
index f0172808c8f..0deb76a268c 100644
--- a/chromium/components/safe_browsing_db/v4_store.h
+++ b/chromium/components/safe_browsing_db/v4_store.h
@@ -15,7 +15,7 @@ namespace safe_browsing {
class V4Store;
-typedef base::Callback<void(std::unique_ptr<V4Store>)>
+typedef base::Callback<void(std::unique_ptr<V4Store> new_store)>
UpdatedStoreReadyCallback;
// The sorted list of hash prefixes.
@@ -158,8 +158,14 @@ class V4Store {
// The |task_runner| is used to ensure that the operations in this file are
// performed on the correct thread. |store_path| specifies the location on
// disk for this file. The constructor doesn't read the store file from disk.
+ // If the store is being created to apply an update to the old store, then
+ // |old_file_size| is the size of the existing file on disk for this store;
+ // 0 otherwise. This is needed so that we can correctly report the size of
+ // store file on disk, even if writing the new file fails after successfully
+ // applying an update.
V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
- const base::FilePath& store_path);
+ const base::FilePath& store_path,
+ const int64_t old_file_size = 0);
virtual ~V4Store();
const std::string& state() const { return state_; }
@@ -167,8 +173,12 @@ class V4Store {
const base::FilePath& store_path() const { return store_path_; }
void ApplyUpdate(std::unique_ptr<ListUpdateResponse> response,
- const scoped_refptr<base::SingleThreadTaskRunner>&,
- UpdatedStoreReadyCallback);
+ const scoped_refptr<base::SingleThreadTaskRunner>& runner,
+ UpdatedStoreReadyCallback callback);
+
+ // Records (in kilobytes) and returns the size of the file on disk for this
+ // store using |base_metric| as prefix and the filename as suffix.
+ int64_t RecordAndReturnFileSize(const std::string& base_metric);
// If a hash prefix in this store matches |full_hash|, returns that hash
// prefix; otherwise returns an empty hash prefix.
@@ -176,12 +186,28 @@ class V4Store {
std::string DebugString() const;
+ // Schedules the destruction of the V4Store object pointed to by |v4_store|,
+ // on the task runner.
+ static void Destroy(std::unique_ptr<V4Store> v4_store);
+
// Reads the store file from disk and populates the in-memory representation
// of the hash prefixes.
void Initialize();
- // Reset internal state and delete the backing file.
- virtual bool Reset();
+ // True if this store has valid contents, either from a successful read
+ // from disk or a full update. This does not mean the checksum was verified.
+ virtual bool HasValidData() const;
+
+ // Reset internal state.
+ void Reset();
+
+ // Scheduled after reading the store file from disk on startup. When run, it
+ // ensures that the checksum of the hash prefixes in lexicographical sorted
+ // order matches the expected value in |expected_checksum_|. Returns true if
+ // it matches; false otherwise. Checksum verification can take a long time,
+ // so it is performed outside of the hotpath of loading SafeBrowsing database,
+ // which blocks resource loads.
+ bool VerifyChecksum();
private:
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestReadFromEmptyFile);
@@ -241,11 +267,15 @@ class V4Store {
TestHashPrefixExistsInMapWithDifferentSizes);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest,
TestHashPrefixDoesNotExistInMapWithDifferentSizes);
+ FRIEND_TEST_ALL_PREFIXES(V4StoreTest, GetMatchingHashPrefixSize32Or21);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest,
TestAdditionsWithRiceEncodingFailsWithInvalidInput);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestAdditionsWithRiceEncodingSucceeds);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestMergeUpdatesFailsChecksum);
+ FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestChecksumErrorOnStartup);
+ FRIEND_TEST_ALL_PREFIXES(V4StoreTest, WriteToDiskFails);
+ FRIEND_TEST_ALL_PREFIXES(V4StoreTest, FullUpdateFailsChecksumSynchronously);
friend class V4StoreTest;
// If |prefix_size| is within expected range, and |raw_hashes_length| is a
@@ -293,57 +323,84 @@ class V4Store {
// Merges the prefix map from the old store (|old_hash_prefix_map|) and the
// update (additions_map) to populate the prefix map for the current store.
// The indices in the |raw_removals| list, which may be NULL, are not merged.
- // The SHA256 checksum of the final list of hash prefixes, in lexographically
- // sorted order, must match |expected_checksum| (if it's not empty).
- ApplyUpdateResult MergeUpdate(const HashPrefixMap& old_hash_prefix_map,
- const HashPrefixMap& additions_map,
- const ::google::protobuf::RepeatedField<
- ::google::protobuf::int32>* raw_removals,
- const std::string& expected_checksum);
+ // The SHA256 checksum of the final list of hash prefixes, in
+ // lexicographically sorted order, must match |expected_checksum| (if it's not
+ // empty).
+ ApplyUpdateResult MergeUpdate(
+ const HashPrefixMap& old_hash_prefix_map,
+ const HashPrefixMap& additions_map,
+ const ::google::protobuf::RepeatedField<::google::protobuf::int32>*
+ raw_removals,
+ const std::string& expected_checksum);
// Processes the FULL_UPDATE |response| from the server, and writes the
// merged V4Store to disk. If processing the |response| succeeds, it returns
- // APPLY_UPDATE_SUCCESS.
+ // APPLY_UPDATE_SUCCESS. The UMA metrics for all interesting sub-operations
+ // use the prefix |metric|.
// This method is only called when we receive a FULL_UPDATE from the server.
ApplyUpdateResult ProcessFullUpdateAndWriteToDisk(
+ const std::string& metric,
std::unique_ptr<ListUpdateResponse> response);
// Processes a FULL_UPDATE |response| and updates the V4Store. If processing
// the |response| succeeds, it returns APPLY_UPDATE_SUCCESS.
// This method is called when we receive a FULL_UPDATE from the server, and
- // when we read a store file from disk on startup.
+ // when we read a store file from disk on startup. The UMA metrics for all
+ // interesting sub-operations use the prefix |metric|. Delays the checksum
+ // check if |delay_checksum_check| is true.
ApplyUpdateResult ProcessFullUpdate(
- const std::unique_ptr<ListUpdateResponse>& response);
+ const std::string& metric,
+ const std::unique_ptr<ListUpdateResponse>& response,
+ bool delay_checksum_check);
// Merges the hash prefixes in |hash_prefix_map_old| and |response|, updates
// the |hash_prefix_map_| and |state_| in the V4Store, and writes the merged
// store to disk. If processing succeeds, it returns APPLY_UPDATE_SUCCESS.
// This method is only called when we receive a PARTIAL_UPDATE from the
- // server.
+ // server. The UMA metrics for all interesting sub-operations use the prefix
+ // |metric|.
ApplyUpdateResult ProcessPartialUpdateAndWriteToDisk(
+ const std::string& metric,
const HashPrefixMap& hash_prefix_map_old,
std::unique_ptr<ListUpdateResponse> response);
// Merges the hash prefixes in |hash_prefix_map_old| and |response|, and
// updates the |hash_prefix_map_| and |state_| in the V4Store. If processing
- // succeeds, it returns APPLY_UPDATE_SUCCESS.
+ // succeeds, it returns APPLY_UPDATE_SUCCESS. The UMA metrics for all
+ // interesting sub-operations use the prefix |metric|. Delays the checksum
+ // check if |delay_checksum_check| is true.
ApplyUpdateResult ProcessUpdate(
+ const std::string& metric,
const HashPrefixMap& hash_prefix_map_old,
- const std::unique_ptr<ListUpdateResponse>& response);
+ const std::unique_ptr<ListUpdateResponse>& response,
+ bool delay_checksum_check);
// Reads the state of the store from the file on disk and returns the reason
// for the failure or reports success.
StoreReadResult ReadFromDisk();
// Updates the |additions_map| with the additions received in the partial
- // update from the server.
+ // update from the server. The UMA metrics for all interesting sub-operations
+ // use the prefix |metric|.
ApplyUpdateResult UpdateHashPrefixMapFromAdditions(
+ const std::string& metric,
const ::google::protobuf::RepeatedPtrField<ThreatEntrySet>& additions,
HashPrefixMap* additions_map);
- // Writes the FULL_UPDATE |response| to disk as a V4StoreFileFormat proto.
- StoreWriteResult WriteToDisk(
- std::unique_ptr<ListUpdateResponse> response) const;
+ // Writes the hash_prefix_map_ to disk as a V4StoreFileFormat proto.
+ // |checksum| is used to set the |checksum| field in the final proto.
+ StoreWriteResult WriteToDisk(const Checksum& checksum);
+
+ // The checksum value as read from the disk, until it is verified. Once
+ // verified, it is cleared.
+ std::string expected_checksum_;
+
+ // The size of the file on disk for this store.
+ int64_t file_size_;
+
+ // True if the file was successfully read+parsed or was populated from
+ // a full update.
+ bool has_valid_data_;
// The state of the store as returned by the PVer4 server in the last applied
// update response.
diff --git a/chromium/components/safe_browsing_db/v4_store_unittest.cc b/chromium/components/safe_browsing_db/v4_store_unittest.cc
index 90c6525fa88..2d2e3d76720 100644
--- a/chromium/components/safe_browsing_db/v4_store_unittest.cc
+++ b/chromium/components/safe_browsing_db/v4_store_unittest.cc
@@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#include "base/base64.h"
#include "base/bind.h"
#include "base/files/file_util.h"
#include "base/files/scoped_temp_dir.h"
@@ -56,25 +57,35 @@ class V4StoreTest : public PlatformTest {
file_format_string.size());
}
- void UpdatedStoreReadyAfterRiceRemovals(bool* called_back,
- std::unique_ptr<V4Store> new_store) {
+ void UpdatedStoreReady(bool* called_back,
+ bool expect_store,
+ std::unique_ptr<V4Store> store) {
*called_back = true;
- EXPECT_EQ(2u, new_store->hash_prefix_map_.size());
- EXPECT_EQ("22222", new_store->hash_prefix_map_[5]);
- EXPECT_EQ("abcd", new_store->hash_prefix_map_[4]);
+ if (expect_store) {
+ ASSERT_TRUE(store);
+ EXPECT_EQ(2u, store->hash_prefix_map_.size());
+ EXPECT_EQ("22222", store->hash_prefix_map_[5]);
+ EXPECT_EQ("abcd", store->hash_prefix_map_[4]);
+ } else {
+ ASSERT_FALSE(store);
+ }
+
+ updated_store_ = std::move(store);
}
base::ScopedTempDir temp_dir_;
base::FilePath store_path_;
scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
content::TestBrowserThreadBundle thread_bundle_;
+ std::unique_ptr<V4Store> updated_store_;
};
TEST_F(V4StoreTest, TestReadFromEmptyFile) {
base::CloseFile(base::OpenFile(store_path_, "wb+"));
- EXPECT_EQ(FILE_EMPTY_FAILURE,
- V4Store(task_runner_, store_path_).ReadFromDisk());
+ V4Store store(task_runner_, store_path_);
+ EXPECT_EQ(FILE_EMPTY_FAILURE, store.ReadFromDisk());
+ EXPECT_FALSE(store.HasValidData());
}
TEST_F(V4StoreTest, TestReadFromAbsentFile) {
@@ -125,36 +136,11 @@ TEST_F(V4StoreTest, TestReadFromNoHashPrefixesFile) {
list_update_response.set_platform_type(LINUX_PLATFORM);
list_update_response.set_response_type(ListUpdateResponse::FULL_UPDATE);
WriteFileFormatProtoToFile(0x600D71FE, 9, &list_update_response);
- EXPECT_EQ(READ_SUCCESS, V4Store(task_runner_, store_path_).ReadFromDisk());
-}
-
-TEST_F(V4StoreTest, TestWriteNoResponseType) {
- EXPECT_EQ(INVALID_RESPONSE_TYPE_FAILURE,
- V4Store(task_runner_, store_path_)
- .WriteToDisk(base::WrapUnique(new ListUpdateResponse)));
-}
-
-TEST_F(V4StoreTest, TestWritePartialResponseType) {
- std::unique_ptr<ListUpdateResponse> list_update_response(
- new ListUpdateResponse);
- list_update_response->set_response_type(ListUpdateResponse::PARTIAL_UPDATE);
- EXPECT_EQ(INVALID_RESPONSE_TYPE_FAILURE,
- V4Store(task_runner_, store_path_)
- .WriteToDisk(std::move(list_update_response)));
-}
-
-TEST_F(V4StoreTest, TestWriteFullResponseType) {
- std::unique_ptr<ListUpdateResponse> list_update_response(
- new ListUpdateResponse);
- list_update_response->set_response_type(ListUpdateResponse::FULL_UPDATE);
- list_update_response->set_new_client_state("test_client_state");
- EXPECT_EQ(WRITE_SUCCESS, V4Store(task_runner_, store_path_)
- .WriteToDisk(std::move(list_update_response)));
-
- V4Store read_store(task_runner_, store_path_);
- EXPECT_EQ(READ_SUCCESS, read_store.ReadFromDisk());
- EXPECT_EQ("test_client_state", read_store.state_);
- EXPECT_TRUE(read_store.hash_prefix_map_.empty());
+ V4Store store(task_runner_, store_path_);
+ EXPECT_EQ(READ_SUCCESS, store.ReadFromDisk());
+ EXPECT_TRUE(store.hash_prefix_map_.empty());
+ EXPECT_EQ(14, store.file_size_);
+ EXPECT_FALSE(store.HasValidData());
}
TEST_F(V4StoreTest, TestAddUnlumpedHashesWithInvalidAddition) {
@@ -568,22 +554,13 @@ TEST_F(V4StoreTest, TestMergeUpdatesRemovesMultipleAcrossDifferentSizes) {
}
TEST_F(V4StoreTest, TestReadFullResponseWithValidHashPrefixMap) {
- std::unique_ptr<ListUpdateResponse> lur(new ListUpdateResponse);
- lur->set_response_type(ListUpdateResponse::FULL_UPDATE);
- lur->set_new_client_state("test_client_state");
- lur->set_platform_type(WINDOWS_PLATFORM);
- lur->set_threat_entry_type(URL);
- lur->set_threat_type(MALWARE_THREAT);
- ThreatEntrySet* additions = lur->add_additions();
- additions->set_compression_type(RAW);
- additions->mutable_raw_hashes()->set_prefix_size(5);
- additions->mutable_raw_hashes()->set_raw_hashes("00000abcde");
- additions = lur->add_additions();
- additions->set_compression_type(RAW);
- additions->mutable_raw_hashes()->set_prefix_size(4);
- additions->mutable_raw_hashes()->set_raw_hashes("00000abc");
- EXPECT_EQ(WRITE_SUCCESS,
- V4Store(task_runner_, store_path_).WriteToDisk(std::move(lur)));
+ V4Store write_store(task_runner_, store_path_);
+ write_store.hash_prefix_map_[4] = "00000abc";
+ write_store.hash_prefix_map_[5] = "00000abcde";
+ write_store.state_ = "test_client_state";
+ EXPECT_FALSE(base::PathExists(write_store.store_path_));
+ EXPECT_EQ(WRITE_SUCCESS, write_store.WriteToDisk(Checksum()));
+ EXPECT_TRUE(base::PathExists(write_store.store_path_));
V4Store read_store (task_runner_, store_path_);
EXPECT_EQ(READ_SUCCESS, read_store.ReadFromDisk());
@@ -591,6 +568,7 @@ TEST_F(V4StoreTest, TestReadFullResponseWithValidHashPrefixMap) {
ASSERT_EQ(2u, read_store.hash_prefix_map_.size());
EXPECT_EQ("00000abc", read_store.hash_prefix_map_[4]);
EXPECT_EQ("00000abcde", read_store.hash_prefix_map_[5]);
+ EXPECT_EQ(71, read_store.file_size_);
}
// This tests fails to read the prefix map from the disk because the file on
@@ -598,23 +576,18 @@ TEST_F(V4StoreTest, TestReadFullResponseWithValidHashPrefixMap) {
// size is 5 so the parser isn't able to split the hash prefixes list
// completely.
TEST_F(V4StoreTest, TestReadFullResponseWithInvalidHashPrefixMap) {
- std::unique_ptr<ListUpdateResponse> lur(new ListUpdateResponse);
- lur->set_response_type(ListUpdateResponse::FULL_UPDATE);
- lur->set_new_client_state("test_client_state");
- lur->set_platform_type(WINDOWS_PLATFORM);
- lur->set_threat_entry_type(URL);
- lur->set_threat_type(MALWARE_THREAT);
- ThreatEntrySet* additions = lur->add_additions();
- additions->set_compression_type(RAW);
- additions->mutable_raw_hashes()->set_prefix_size(5);
- additions->mutable_raw_hashes()->set_raw_hashes("abcdef");
- EXPECT_EQ(WRITE_SUCCESS,
- V4Store(task_runner_, store_path_).WriteToDisk(std::move(lur)));
+ V4Store write_store(task_runner_, store_path_);
+ write_store.hash_prefix_map_[5] = "abcdef";
+ write_store.state_ = "test_client_state";
+ EXPECT_FALSE(base::PathExists(write_store.store_path_));
+ EXPECT_EQ(WRITE_SUCCESS, write_store.WriteToDisk(Checksum()));
+ EXPECT_TRUE(base::PathExists(write_store.store_path_));
V4Store read_store(task_runner_, store_path_);
EXPECT_EQ(HASH_PREFIX_MAP_GENERATION_FAILURE, read_store.ReadFromDisk());
EXPECT_TRUE(read_store.state_.empty());
EXPECT_TRUE(read_store.hash_prefix_map_.empty());
+ EXPECT_EQ(0, read_store.file_size_);
}
TEST_F(V4StoreTest, TestHashPrefixExistsAtTheBeginning) {
@@ -700,6 +673,22 @@ TEST_F(V4StoreTest, TestHashPrefixDoesNotExistInMapWithDifferentSizes) {
EXPECT_TRUE(store.GetMatchingHashPrefix(full_hash).empty());
}
+TEST_F(V4StoreTest, GetMatchingHashPrefixSize32Or21) {
+ HashPrefix prefix = "0123";
+ V4Store store(task_runner_, store_path_);
+ store.hash_prefix_map_[4] = prefix;
+
+ FullHash full_hash_21 = "0123456789ABCDEF01234";
+ EXPECT_EQ(prefix, store.GetMatchingHashPrefix(full_hash_21));
+ FullHash full_hash_32 = "0123456789ABCDEF0123456789ABCDEF";
+ EXPECT_EQ(prefix, store.GetMatchingHashPrefix(full_hash_32));
+#if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON)
+ // This hits a DCHECK so it is release mode only.
+ FullHash full_hash_22 = "0123456789ABCDEF012345";
+ EXPECT_EQ(prefix, store.GetMatchingHashPrefix(full_hash_22));
+#endif
+}
+
#if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON)
// This test hits a NOTREACHED so it is a release mode only test.
TEST_F(V4StoreTest, TestAdditionsWithRiceEncodingFailsWithInvalidInput) {
@@ -710,7 +699,8 @@ TEST_F(V4StoreTest, TestAdditionsWithRiceEncodingFailsWithInvalidInput) {
HashPrefixMap additions_map;
EXPECT_EQ(RICE_DECODING_FAILURE,
V4Store(task_runner_, store_path_)
- .UpdateHashPrefixMapFromAdditions(additions, &additions_map));
+ .UpdateHashPrefixMapFromAdditions("V4Metric", additions,
+ &additions_map));
}
#endif
@@ -731,7 +721,8 @@ TEST_F(V4StoreTest, TestAdditionsWithRiceEncodingSucceeds) {
HashPrefixMap additions_map;
EXPECT_EQ(APPLY_UPDATE_SUCCESS,
V4Store(task_runner_, store_path_)
- .UpdateHashPrefixMapFromAdditions(additions, &additions_map));
+ .UpdateHashPrefixMapFromAdditions("V4Metric", additions,
+ &additions_map));
EXPECT_EQ(1u, additions_map.size());
EXPECT_EQ(std::string("\x5\0\0\0\fL\x93\xADV\x7F\xF6o\xCEo1\x81", 16),
additions_map[4]);
@@ -754,6 +745,7 @@ TEST_F(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds) {
EXPECT_EQ(APPLY_UPDATE_SUCCESS,
store.MergeUpdate(prefix_map_old, prefix_map_additions, nullptr,
expected_checksum));
+ EXPECT_FALSE(store.HasValidData()); // Never actually read from disk.
// At this point, the store map looks like this:
// 4: 1111abcdefgh
@@ -773,14 +765,20 @@ TEST_F(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds) {
bool called_back = false;
UpdatedStoreReadyCallback store_ready_callback =
- base::Bind(&V4StoreTest::UpdatedStoreReadyAfterRiceRemovals,
- base::Unretained(this), &called_back);
+ base::Bind(&V4StoreTest::UpdatedStoreReady, base::Unretained(this),
+ &called_back, true /* expect_store */);
+ EXPECT_FALSE(base::PathExists(store.store_path_));
store.ApplyUpdate(std::move(lur), task_runner_, store_ready_callback);
+ EXPECT_TRUE(base::PathExists(store.store_path_));
+
task_runner_->RunPendingTasks();
base::RunLoop().RunUntilIdle();
// This ensures that the callback was called.
EXPECT_TRUE(called_back);
+ // ApplyUpdate was successful, so we have valid data.
+ ASSERT_TRUE(updated_store_);
+ EXPECT_TRUE(updated_store_->HasValidData());
}
TEST_F(V4StoreTest, TestMergeUpdatesFailsChecksum) {
@@ -800,4 +798,86 @@ TEST_F(V4StoreTest, TestMergeUpdatesFailsChecksum) {
.MergeUpdate(prefix_map_old, HashPrefixMap(), nullptr, "aawc"));
}
+TEST_F(V4StoreTest, TestChecksumErrorOnStartup) {
+ // First the case of checksum not matching after reading from disk.
+ ListUpdateResponse list_update_response;
+ list_update_response.set_new_client_state("test_client_state");
+ list_update_response.set_platform_type(LINUX_PLATFORM);
+ list_update_response.set_response_type(ListUpdateResponse::FULL_UPDATE);
+ list_update_response.mutable_checksum()->set_sha256(
+ std::string(crypto::kSHA256Length, 0));
+ WriteFileFormatProtoToFile(0x600D71FE, 9, &list_update_response);
+ V4Store store(task_runner_, store_path_);
+ EXPECT_TRUE(store.expected_checksum_.empty());
+ EXPECT_EQ(READ_SUCCESS, store.ReadFromDisk());
+ EXPECT_TRUE(!store.expected_checksum_.empty());
+ EXPECT_EQ(69, store.file_size_);
+ EXPECT_EQ("test_client_state", store.state());
+
+ EXPECT_FALSE(store.VerifyChecksum());
+
+ // Now the case of checksum matching after reading from disk.
+ // Proof of checksum mismatch using python:
+ // >>> import hashlib
+ // >>> m = hashlib.sha256()
+ // >>> m.update("abcde")
+ // >>> import base64
+ // >>> encoded = base64.b64encode(m.digest())
+ // >>> encoded
+ // 'NrvlDtloQdEEQ7y2cNZVTwo0t2G+Z+ycSorSwMRMpCw='
+ std::string expected_checksum;
+ base::Base64Decode("NrvlDtloQdEEQ7y2cNZVTwo0t2G+Z+ycSorSwMRMpCw=",
+ &expected_checksum);
+ ThreatEntrySet* additions = list_update_response.add_additions();
+ additions->set_compression_type(RAW);
+ additions->mutable_raw_hashes()->set_prefix_size(5);
+ additions->mutable_raw_hashes()->set_raw_hashes("abcde");
+ list_update_response.mutable_checksum()->set_sha256(expected_checksum);
+ WriteFileFormatProtoToFile(0x600D71FE, 9, &list_update_response);
+ V4Store another_store(task_runner_, store_path_);
+ EXPECT_TRUE(another_store.expected_checksum_.empty());
+
+ EXPECT_EQ(READ_SUCCESS, another_store.ReadFromDisk());
+ EXPECT_TRUE(!another_store.expected_checksum_.empty());
+ EXPECT_EQ("test_client_state", another_store.state());
+ EXPECT_EQ(69, store.file_size_);
+
+ EXPECT_TRUE(another_store.VerifyChecksum());
+}
+
+TEST_F(V4StoreTest, WriteToDiskFails) {
+ // Pass the directory name as file name so that when the code tries to rename
+ // the temp store file to |store_path_| it fails.
+ EXPECT_EQ(UNABLE_TO_RENAME_FAILURE,
+ V4Store(task_runner_, temp_dir_.GetPath()).WriteToDisk(Checksum()));
+}
+
+TEST_F(V4StoreTest, FullUpdateFailsChecksumSynchronously) {
+ V4Store store(task_runner_, store_path_);
+ bool called_back = false;
+ UpdatedStoreReadyCallback store_ready_callback =
+ base::Bind(&V4StoreTest::UpdatedStoreReady, base::Unretained(this),
+ &called_back, false /* expect_store */);
+ EXPECT_FALSE(base::PathExists(store.store_path_));
+ EXPECT_FALSE(store.HasValidData()); // Never actually read from disk.
+
+ // Now create a response with invalid checksum.
+ std::unique_ptr<ListUpdateResponse> lur(new ListUpdateResponse);
+ lur->set_response_type(ListUpdateResponse::FULL_UPDATE);
+ lur->mutable_checksum()->set_sha256(std::string(crypto::kSHA256Length, 0));
+ store.ApplyUpdate(std::move(lur), task_runner_, store_ready_callback);
+ // The update should fail synchronously and not create a store file.
+ EXPECT_FALSE(base::PathExists(store.store_path_));
+
+ // Run everything on the task runner to ensure there are no pending tasks.
+ task_runner_->RunPendingTasks();
+ base::RunLoop().RunUntilIdle();
+
+ // This ensures that the callback was called.
+ EXPECT_TRUE(called_back);
+ // Ensure that the file is still not created.
+ EXPECT_FALSE(base::PathExists(store.store_path_));
+ EXPECT_FALSE(updated_store_);
+}
+
} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_test_util.cc b/chromium/components/safe_browsing_db/v4_test_util.cc
new file mode 100644
index 00000000000..a37279569e2
--- /dev/null
+++ b/chromium/components/safe_browsing_db/v4_test_util.cc
@@ -0,0 +1,30 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/safe_browsing_db/util.h"
+#include "components/safe_browsing_db/v4_test_util.h"
+
+namespace safe_browsing {
+
+namespace {
+
+const char kClient[] = "unittest";
+const char kAppVer[] = "1.0";
+const char kKeyParam[] = "test_key_param";
+
+} // namespace
+
+V4ProtocolConfig GetTestV4ProtocolConfig(bool disable_auto_update) {
+ return V4ProtocolConfig(kClient, disable_auto_update, kKeyParam, kAppVer);
+}
+
+std::ostream& operator<<(std::ostream& os, const ThreatMetadata& meta) {
+ os << "{threat_pattern_type=" << static_cast<int>(meta.threat_pattern_type)
+ << ", api_permissions=[";
+ for (auto p : meta.api_permissions)
+ os << p << ",";
+ return os << "], population_id=" << meta.population_id << "}";
+}
+
+} // namespace safe_browsing
diff --git a/chromium/components/safe_browsing_db/v4_test_util.h b/chromium/components/safe_browsing_db/v4_test_util.h
new file mode 100644
index 00000000000..d21170cecf7
--- /dev/null
+++ b/chromium/components/safe_browsing_db/v4_test_util.h
@@ -0,0 +1,23 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_SAFE_BROWSING_DB_V4_TEST_UTIL_H_
+#define COMPONENTS_SAFE_BROWSING_DB_V4_TEST_UTIL_H_
+
+// Contains methods useful for tests.
+
+#include <ostream>
+
+namespace safe_browsing {
+
+struct ThreatMetadata;
+struct V4ProtocolConfig;
+
+V4ProtocolConfig GetTestV4ProtocolConfig(bool disable_auto_update = false);
+
+std::ostream& operator<<(std::ostream& os, const ThreatMetadata& meta);
+
+} // namespace safe_browsing
+
+#endif // COMPONENTS_SAFE_BROWSING_DB_V4_TEST_UTIL_H_
diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc b/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc
index ae43b2e3b40..47736aa73bf 100644
--- a/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc
+++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc
@@ -50,13 +50,13 @@ enum ParseResultType {
// Record parsing errors of an update result.
void RecordParseUpdateResult(ParseResultType result_type) {
- UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.ParseV4UpdateResult", result_type,
+ UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4Update.Parse.Result", result_type,
PARSE_RESULT_TYPE_MAX);
}
void RecordUpdateResult(safe_browsing::V4OperationResult result) {
UMA_HISTOGRAM_ENUMERATION(
- "SafeBrowsing.V4UpdateResult", result,
+ "SafeBrowsing.V4Update.Result", result,
safe_browsing::V4OperationResult::OPERATION_RESULT_MAX);
}
@@ -70,6 +70,9 @@ static const int kV4TimerStartIntervalSecMin = 60;
// Maximum time, in seconds, from start up before we must issue an update query.
static const int kV4TimerStartIntervalSecMax = 300;
+// Maximum time, in seconds, to wait for a response to an update request.
+static const int kV4TimerUpdateWaitSecMax = 30;
+
// The default V4UpdateProtocolManagerFactory.
class V4UpdateProtocolManagerFactoryImpl
: public V4UpdateProtocolManagerFactory {
@@ -142,8 +145,6 @@ void V4UpdateProtocolManager::ScheduleNextUpdate(
void V4UpdateProtocolManager::ScheduleNextUpdateWithBackoff(bool back_off) {
DCHECK(CalledOnValidThread());
- // TODO(vakh): Set disable_auto_update correctly using the command line
- // switch.
if (config_.disable_auto_update) {
DCHECK(!IsUpdateScheduled());
return;
@@ -250,7 +251,6 @@ bool V4UpdateProtocolManager::ParseUpdateResponse(
base::TimeDelta::FromSeconds(minimum_wait_duration_seconds);
}
- // TODO(vakh): Do something useful with this response.
for (ListUpdateResponse& list_update_response :
*response.mutable_list_update_responses()) {
if (!list_update_response.has_platform_type()) {
@@ -295,7 +295,17 @@ void V4UpdateProtocolManager::IssueUpdateRequest() {
request_->SetLoadFlags(net::LOAD_DISABLE_CACHE);
request_->SetRequestContext(request_context_getter_.get());
request_->Start();
- // TODO(vakh): Handle request timeout.
+
+ // Begin the update request timeout.
+ timeout_timer_.Start(FROM_HERE,
+ TimeDelta::FromSeconds(kV4TimerUpdateWaitSecMax), this,
+ &V4UpdateProtocolManager::HandleTimeout);
+}
+
+void V4UpdateProtocolManager::HandleTimeout() {
+ UMA_HISTOGRAM_BOOLEAN("SafeBrowsing.V4Update.TimedOut", true);
+ request_.reset();
+ ScheduleNextUpdateWithBackoff(false);
}
// net::URLFetcherDelegate implementation ----------------------------------
@@ -305,10 +315,13 @@ void V4UpdateProtocolManager::OnURLFetchComplete(
const net::URLFetcher* source) {
DCHECK(CalledOnValidThread());
+ timeout_timer_.Stop();
+
int response_code = source->GetResponseCode();
net::URLRequestStatus status = source->GetStatus();
V4ProtocolManagerUtil::RecordHttpResponseOrErrorCode(
- "SafeBrowsing.V4UpdateHttpResponseOrErrorCode", status, response_code);
+ "SafeBrowsing.V4Update.Network.Result", status, response_code);
+ UMA_HISTOGRAM_BOOLEAN("SafeBrowsing.V4Update.TimedOut", false);
last_response_time_ = Time::Now();
@@ -325,7 +338,7 @@ void V4UpdateProtocolManager::OnURLFetchComplete(
}
request_.reset();
- UMA_HISTOGRAM_COUNTS("SafeBrowsing.V4UpdateResponseSizeKB",
+ UMA_HISTOGRAM_COUNTS("SafeBrowsing.V4Update.ResponseSizeKB",
data.size() / 1024);
// The caller should update its state now, based on parsed_server_response.
diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager.h b/chromium/components/safe_browsing_db/v4_update_protocol_manager.h
index 7da1667c66f..a3b4cd538e0 100644
--- a/chromium/components/safe_browsing_db/v4_update_protocol_manager.h
+++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager.h
@@ -87,6 +87,9 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate,
TestGetUpdatesWithOneBackoff);
FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest,
TestBase64EncodingUsesUrlEncoding);
+ FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, TestDisableAutoUpdates);
+ FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest,
+ TestGetUpdatesHasTimeout);
friend class V4UpdateProtocolManagerFactoryImpl;
// Fills a FetchThreatListUpdatesRequest protocol buffer for a request.
@@ -111,6 +114,10 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate,
// Resets the update error counter and multiplier.
void ResetUpdateErrors();
+ // Called when update request times out. Cancels the existing request and
+ // re-sends the update request.
+ void HandleTimeout();
+
// Updates internal update and backoff state for each update response error,
// assuming that the current time is |now|.
void HandleUpdateError(const base::Time& now);
@@ -172,6 +179,10 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate,
base::Time last_response_time_;
+ // Used to interrupt and re-schedule update requests that take too long to
+ // complete.
+ base::OneShotTimer timeout_timer_;
+
DISALLOW_COPY_AND_ASSIGN(V4UpdateProtocolManager);
};
diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc
index e8596e68e89..cdef7382501 100644
--- a/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc
+++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc
@@ -15,6 +15,7 @@
#include "base/time/time.h"
#include "components/safe_browsing_db/safebrowsing.pb.h"
#include "components/safe_browsing_db/util.h"
+#include "components/safe_browsing_db/v4_test_util.h"
#include "net/base/escape.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
@@ -25,14 +26,6 @@
using base::Time;
using base::TimeDelta;
-namespace {
-
-const char kClient[] = "unittest";
-const char kAppVer[] = "1.0";
-const char kKeyParam[] = "test_key_param";
-
-} // namespace
-
namespace safe_browsing {
class V4UpdateProtocolManagerTest : public PlatformTest {
@@ -65,19 +58,11 @@ class V4UpdateProtocolManagerTest : public PlatformTest {
}
}
- V4ProtocolConfig GetProtocolConfig() {
- V4ProtocolConfig config;
- config.client_name = kClient;
- config.version = kAppVer;
- config.key_param = kKeyParam;
- config.disable_auto_update = false;
- return config;
- }
-
std::unique_ptr<V4UpdateProtocolManager> CreateProtocolManager(
- const std::vector<ListUpdateResponse>& expected_lurs) {
+ const std::vector<ListUpdateResponse>& expected_lurs,
+ bool disable_auto_update = false) {
return V4UpdateProtocolManager::Create(
- NULL, GetProtocolConfig(),
+ NULL, GetTestV4ProtocolConfig(disable_auto_update),
base::Bind(&V4UpdateProtocolManagerTest::ValidateGetUpdatesResults,
base::Unretained(this), expected_lurs));
}
@@ -163,8 +148,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesErrorHandlingNetwork) {
EXPECT_FALSE(pm->IsUpdateScheduled());
- runner->RunPendingTasks();
-
net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
DCHECK(fetcher);
// Failed request status should result in error.
@@ -197,8 +180,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesErrorHandlingResponseCode) {
EXPECT_FALSE(pm->IsUpdateScheduled());
- runner->RunPendingTasks();
-
net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
DCHECK(fetcher);
fetcher->set_status(net::URLRequestStatus());
@@ -233,8 +214,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesNoError) {
EXPECT_FALSE(pm->IsUpdateScheduled());
- runner->RunPendingTasks();
-
net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
DCHECK(fetcher);
fetcher->set_status(net::URLRequestStatus());
@@ -268,8 +247,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesWithOneBackoff) {
EXPECT_FALSE(pm->IsUpdateScheduled());
- runner->RunPendingTasks();
-
net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
DCHECK(fetcher);
fetcher->set_status(net::URLRequestStatus());
@@ -285,6 +262,7 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesWithOneBackoff) {
// Retry, now no backoff.
expect_callback_to_be_called_ = true;
+ // Call RunPendingTasks to ensure that the request is sent after backoff.
runner->RunPendingTasks();
fetcher = factory.GetFetcherByID(1);
@@ -320,4 +298,64 @@ TEST_F(V4UpdateProtocolManagerTest, TestBase64EncodingUsesUrlEncoding) {
// the '-' case is sufficient to prove that we are using URL encoding.
}
+TEST_F(V4UpdateProtocolManagerTest, TestDisableAutoUpdates) {
+ scoped_refptr<base::TestSimpleTaskRunner> runner(
+ new base::TestSimpleTaskRunner());
+ base::ThreadTaskRunnerHandle runner_handler(runner);
+ net::TestURLFetcherFactory factory;
+ std::unique_ptr<V4UpdateProtocolManager> pm(CreateProtocolManager(
+ std::vector<ListUpdateResponse>(), true /* disable_auto_update */));
+
+ // Initial state. No errors.
+ pm->ScheduleNextUpdate(std::move(store_state_map_));
+ EXPECT_FALSE(pm->IsUpdateScheduled());
+
+ runner->RunPendingTasks();
+ EXPECT_FALSE(pm->IsUpdateScheduled());
+
+ net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
+ DCHECK(!fetcher);
+}
+
+TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesHasTimeout) {
+ scoped_refptr<base::TestSimpleTaskRunner> runner(
+ new base::TestSimpleTaskRunner());
+ base::ThreadTaskRunnerHandle runner_handler(runner);
+ net::TestURLFetcherFactory factory;
+ std::vector<ListUpdateResponse> expected_lurs;
+ SetupExpectedListUpdateResponse(&expected_lurs);
+ std::unique_ptr<V4UpdateProtocolManager> pm(
+ CreateProtocolManager(expected_lurs));
+ runner->ClearPendingTasks();
+
+ // Initial state. No errors.
+ EXPECT_EQ(0ul, pm->update_error_count_);
+ EXPECT_EQ(1ul, pm->update_back_off_mult_);
+ expect_callback_to_be_called_ = true;
+ pm->store_state_map_ = std::move(store_state_map_);
+ pm->IssueUpdateRequest();
+
+ net::TestURLFetcher* timeout_fetcher = factory.GetFetcherByID(0);
+ DCHECK(timeout_fetcher);
+ // Don't set anything on the fetcher. Let it time out.
+ runner->RunPendingTasks();
+
+ net::TestURLFetcher* fetcher = factory.GetFetcherByID(1);
+ DCHECK(!fetcher);
+ // Now wait for the next request to be scheduled.
+ runner->RunPendingTasks();
+
+ // There should be another fetcher now.
+ fetcher = factory.GetFetcherByID(1);
+ DCHECK(fetcher);
+ fetcher->set_status(net::URLRequestStatus());
+ fetcher->set_response_code(net::HTTP_OK);
+ fetcher->SetResponseString(GetExpectedV4UpdateResponse(expected_lurs));
+ fetcher->delegate()->OnURLFetchComplete(fetcher);
+
+ // No error, back off multiplier is unchanged.
+ EXPECT_EQ(0ul, pm->update_error_count_);
+ EXPECT_EQ(1ul, pm->update_back_off_mult_);
+}
+
} // namespace safe_browsing