diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-03-08 10:28:10 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-03-20 13:40:30 +0000 |
commit | e733310db58160074f574c429d48f8308c0afe17 (patch) | |
tree | f8aef4b7e62a69928dbcf880620eece20f98c6df /chromium/components/safe_browsing_db | |
parent | 2f583e4aec1ae3a86fa047829c96b310dc12ecdf (diff) | |
download | qtwebengine-chromium-e733310db58160074f574c429d48f8308c0afe17.tar.gz |
BASELINE: Update Chromium to 56.0.2924.122
Change-Id: I4e04de8f47e47e501c46ed934c76a431c6337ced
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/safe_browsing_db')
35 files changed, 3071 insertions, 660 deletions
diff --git a/chromium/components/safe_browsing_db/BUILD.gn b/chromium/components/safe_browsing_db/BUILD.gn index e14da2e5b0c..e7e788ceec9 100644 --- a/chromium/components/safe_browsing_db/BUILD.gn +++ b/chromium/components/safe_browsing_db/BUILD.gn @@ -31,6 +31,7 @@ group("safe_browsing_db_shared") { ":database_manager", ":hit_report", ":prefix_set", + ":safe_browsing_prefs", ":safebrowsing_proto", ":util", ] @@ -78,6 +79,7 @@ static_library("hit_report") { "hit_report.h", ] deps = [ + ":safe_browsing_prefs", ":util", "//components/metrics", "//url", @@ -136,6 +138,17 @@ static_library("safe_browsing_api_handler_util") { ] } +static_library("safe_browsing_prefs") { + sources = [ + "safe_browsing_prefs.cc", + "safe_browsing_prefs.h", + ] + deps = [ + "//base:base", + "//components/prefs", + ] +} + static_library("test_database_manager") { sources = [ "test_database_manager.cc", @@ -276,6 +289,18 @@ source_set("v4_store") { ] } +static_library("v4_test_util") { + testonly = true + sources = [ + "v4_test_util.cc", + "v4_test_util.h", + ] + deps = [ + ":util", + ":v4_protocol_manager_util", + ] +} + static_library("v4_update_protocol_manager") { sources = [ "v4_update_protocol_manager.cc", @@ -318,6 +343,7 @@ source_set("v4_get_hash_protocol_manager_unittest") { ":v4_database", ":v4_get_hash_protocol_manager", ":v4_local_database_manager", + ":v4_test_util", "//base", "//base/test:test_support", "//content/test:test_support", @@ -335,6 +361,7 @@ source_set("v4_local_database_manager_unittest") { deps = [ ":v4_database", ":v4_local_database_manager", + ":v4_test_util", "//base", "//base/test:test_support", "//content/test:test_support", @@ -344,11 +371,30 @@ source_set("v4_local_database_manager_unittest") { ] } +source_set("v4_update_protocol_manager_unittest") { + testonly = true + sources = [ + "v4_update_protocol_manager_unittest.cc", + ] + deps = [ + ":safebrowsing_proto", + ":util", + ":v4_test_util", + ":v4_update_protocol_manager", + "//base", + "//base/test:test_support", + "//net", + "//net:test_support", + "//testing/gtest", + ] +} + source_set("unit_tests") { testonly = true sources = [ "database_manager_unittest.cc", "prefix_set_unittest.cc", + "safe_browsing_prefs_unittest.cc", "util_unittest.cc", "v4_database_unittest.cc", "v4_get_hash_protocol_manager_unittest.cc", @@ -361,6 +407,7 @@ source_set("unit_tests") { deps = [ ":database_manager", ":prefix_set", + ":safe_browsing_prefs", ":safebrowsing_proto", ":test_database_manager", ":util", @@ -371,8 +418,10 @@ source_set("unit_tests") { ":v4_rice", ":v4_store", ":v4_store_proto", + ":v4_test_util", ":v4_update_protocol_manager", "//base", + "//components/prefs:test_support", "//content/test:test_support", "//crypto", "//net", @@ -398,6 +447,7 @@ source_set("unit_tests_mobile") { ":safe_browsing_api_handler", ":safe_browsing_api_handler_util", ":util", + ":v4_test_util", "//base", "//components/variations", "//testing/gtest", diff --git a/chromium/components/safe_browsing_db/DEPS b/chromium/components/safe_browsing_db/DEPS index 9d835a18bfe..91ea0ff090f 100644 --- a/chromium/components/safe_browsing_db/DEPS +++ b/chromium/components/safe_browsing_db/DEPS @@ -1,5 +1,6 @@ include_rules = [ "+components/data_use_measurement/core", + "+components/prefs", "+components/variations", "+content/public/browser", "+content/public/common", diff --git a/chromium/components/safe_browsing_db/PRESUBMIT.py b/chromium/components/safe_browsing_db/PRESUBMIT.py new file mode 100644 index 00000000000..9a395c9243f --- /dev/null +++ b/chromium/components/safe_browsing_db/PRESUBMIT.py @@ -0,0 +1,12 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Presubmit script for components/safe_browsing_db + +See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts +for more details about the presubmit API built into depot_tools. +""" + +def CheckChangeOnUpload(input_api, output_api): + return input_api.canned_checks.CheckPatchFormatted(input_api, output_api) diff --git a/chromium/components/safe_browsing_db/database_manager_unittest.cc b/chromium/components/safe_browsing_db/database_manager_unittest.cc index 21338b86012..5180687b6f3 100644 --- a/chromium/components/safe_browsing_db/database_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/database_manager_unittest.cc @@ -20,6 +20,7 @@ #include "base/threading/thread_task_runner_handle.h" #include "components/safe_browsing_db/test_database_manager.h" #include "components/safe_browsing_db/v4_protocol_manager_util.h" +#include "components/safe_browsing_db/v4_test_util.h" #include "content/public/browser/browser_thread.h" #include "content/public/test/test_browser_thread_bundle.h" #include "crypto/sha2.h" @@ -63,7 +64,7 @@ class SafeBrowsingDatabaseManagerTest : public testing::Test { protected: void SetUp() override { db_manager_ = new TestSafeBrowsingDatabaseManager(); - db_manager_->StartOnIOThread(NULL, V4ProtocolConfig()); + db_manager_->StartOnIOThread(NULL, GetTestV4ProtocolConfig()); } void TearDown() override { diff --git a/chromium/components/safe_browsing_db/hit_report.h b/chromium/components/safe_browsing_db/hit_report.h index 158ec983d69..afa567637f1 100644 --- a/chromium/components/safe_browsing_db/hit_report.h +++ b/chromium/components/safe_browsing_db/hit_report.h @@ -7,6 +7,7 @@ #ifndef COMPONENTS_SAFE_BROWSING_DB_HIT_REPORT_H_ #define COMPONENTS_SAFE_BROWSING_DB_HIT_REPORT_H_ +#include "components/safe_browsing_db/safe_browsing_prefs.h" #include "components/safe_browsing_db/util.h" #include "url/gurl.h" @@ -16,9 +17,9 @@ namespace safe_browsing { enum class ThreatSource { UNKNOWN, DATA_SAVER, // From the Data Reduction service. - LOCAL_PVER3, // From LocalSafeBrowingDatabaseManager, protocol v3 - LOCAL_PVER4, // From LocalSafeBrowingDatabaseManager, protocol v4 - REMOTE, // From RemoteSafeBrowingDatabaseManager + LOCAL_PVER3, // From LocalSafeBrowsingDatabaseManager, protocol v3 + LOCAL_PVER4, // From V4LocalDatabaseManager, protocol v4 + REMOTE, // From RemoteSafeBrowsingDatabaseManager CLIENT_SIDE_DETECTION, // From ClientSideDetectionHost }; @@ -41,7 +42,7 @@ struct HitReport { // Opaque string used for tracking Pver4-based experiments std::string population_id; - bool is_extended_reporting; + ExtendedReportingLevel extended_reporting_level; bool is_metrics_reporting_active; std::string post_data; diff --git a/chromium/components/safe_browsing_db/prefix_set.cc b/chromium/components/safe_browsing_db/prefix_set.cc index 824bbd4f742..0701b8e25bc 100644 --- a/chromium/components/safe_browsing_db/prefix_set.cc +++ b/chromium/components/safe_browsing_db/prefix_set.cc @@ -15,7 +15,7 @@ #include "base/md5.h" #include "base/memory/ptr_util.h" #include "base/metrics/histogram.h" -#include "base/metrics/sparse_histogram.h" +#include "base/metrics/histogram_macros.h" namespace safe_browsing { diff --git a/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc b/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc index 8e91b098842..fbf07be6136 100644 --- a/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc +++ b/chromium/components/safe_browsing_db/safe_browsing_api_handler_unittest.cc @@ -4,8 +4,8 @@ #include "components/safe_browsing_db/metadata.pb.h" #include "components/safe_browsing_db/safe_browsing_api_handler_util.h" -#include "components/safe_browsing_db/testing_util.h" #include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_test_util.h" #include "testing/gtest/include/gtest/gtest.h" namespace safe_browsing { diff --git a/chromium/components/safe_browsing_db/safe_browsing_prefs.cc b/chromium/components/safe_browsing_db/safe_browsing_prefs.cc new file mode 100644 index 00000000000..c5a110aabdb --- /dev/null +++ b/chromium/components/safe_browsing_db/safe_browsing_prefs.cc @@ -0,0 +1,247 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/command_line.h" +#include "base/metrics/histogram_macros.h" +#include "components/prefs/pref_service.h" +#include "components/safe_browsing_db/safe_browsing_prefs.h" + +namespace { + +// Switch value which force the ScoutGroupSelected pref to true. +const char kForceScoutGroupValueTrue[] = "true"; +// Switch value which force the ScoutGroupSelected pref to false. +const char kForceScoutGroupValueFalse[] = "false"; + +// Name of the Scout Transition UMA metric. +const char kScoutTransitionMetricName[] = "SafeBrowsing.Pref.Scout.Transition"; + +// Reasons that a state transition for Scout was performed. +// These values are written to logs. New enum values can be added, but +// existing enums must never be renumbered or deleted and reused. +enum ScoutTransitionReason { + // Flag forced Scout Group to true + FORCE_SCOUT_FLAG_TRUE = 0, + // Flag forced Scout Group to false + FORCE_SCOUT_FLAG_FALSE = 1, + // User in OnlyShowScout group, enters Scout Group + ONLY_SHOW_SCOUT_OPT_IN = 2, + // User in CanShowScout group, enters Scout Group immediately + CAN_SHOW_SCOUT_OPT_IN_SCOUT_GROUP_ON = 3, + // User in CanShowScout group, waiting for interstitial to enter Scout Group + CAN_SHOW_SCOUT_OPT_IN_WAIT_FOR_INTERSTITIAL = 4, + // User in CanShowScout group saw the first interstitial and entered the Scout + // Group + CAN_SHOW_SCOUT_OPT_IN_SAW_FIRST_INTERSTITIAL = 5, + // User in Control group + CONTROL = 6, + // Rollback: SBER2 on on implies SBER1 can turn on + ROLLBACK_SBER2_IMPLIES_SBER1 = 7, + // Rollback: SBER2 off so SBER1 must be turned off + ROLLBACK_NO_SBER2_SET_SBER1_FALSE = 8, + // Rollback: SBER2 absent so SBER1 must be cleared + ROLLBACK_NO_SBER2_CLEAR_SBER1 = 9, + // New reasons must be added BEFORE MAX_REASONS + MAX_REASONS +}; +} // namespace + +namespace prefs { +const char kSafeBrowsingExtendedReportingEnabled[] = + "safebrowsing.extended_reporting_enabled"; +const char kSafeBrowsingScoutReportingEnabled[] = + "safebrowsing.scout_reporting_enabled"; +const char kSafeBrowsingScoutGroupSelected[] = + "safebrowsing.scout_group_selected"; +} // namespace prefs + +namespace safe_browsing { + +const char kSwitchForceScoutGroup[] = "force-scout-group"; + +const base::Feature kCanShowScoutOptIn{"CanShowScoutOptIn", + base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kOnlyShowScoutOptIn{"OnlyShowScoutOptIn", + base::FEATURE_DISABLED_BY_DEFAULT}; + +std::string ChooseOptInTextPreference( + const PrefService& prefs, + const std::string& extended_reporting_pref, + const std::string& scout_pref) { + return IsScout(prefs) ? scout_pref : extended_reporting_pref; +} + +int ChooseOptInTextResource(const PrefService& prefs, + int extended_reporting_resource, + int scout_resource) { + return IsScout(prefs) ? scout_resource : extended_reporting_resource; +} + +bool ExtendedReportingPrefExists(const PrefService& prefs) { + return prefs.HasPrefPath(GetExtendedReportingPrefName(prefs)); +} + +ExtendedReportingLevel GetExtendedReportingLevel(const PrefService& prefs) { + if (!IsExtendedReportingEnabled(prefs)) { + return SBER_LEVEL_OFF; + } else { + return IsScout(prefs) ? SBER_LEVEL_SCOUT : SBER_LEVEL_LEGACY; + } +} + +const char* GetExtendedReportingPrefName(const PrefService& prefs) { + // The Scout pref is active if either of the experiment features are on, and + // ScoutGroupSelected is on as well. + if ((base::FeatureList::IsEnabled(kCanShowScoutOptIn) || + base::FeatureList::IsEnabled(kOnlyShowScoutOptIn)) && + prefs.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) { + return prefs::kSafeBrowsingScoutReportingEnabled; + } + + // ..otherwise, either no experiment is on (ie: the Control group) or + // ScoutGroupSelected is off. So we use the SBER pref instead. + return prefs::kSafeBrowsingExtendedReportingEnabled; +} + +void InitializeSafeBrowsingPrefs(PrefService* prefs) { + // First handle forcing kSafeBrowsingScoutGroupSelected pref via cmd-line. + if (base::CommandLine::ForCurrentProcess()->HasSwitch( + kSwitchForceScoutGroup)) { + std::string switch_value = + base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( + kSwitchForceScoutGroup); + if (switch_value == kForceScoutGroupValueTrue) { + prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + FORCE_SCOUT_FLAG_TRUE, MAX_REASONS); + } else if (switch_value == kForceScoutGroupValueFalse) { + prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, false); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + FORCE_SCOUT_FLAG_FALSE, MAX_REASONS); + } + + // If the switch is used, don't process the experiment state. + return; + } + + // Handle the three possible experiment states. + if (base::FeatureList::IsEnabled(kOnlyShowScoutOptIn)) { + // OnlyShowScoutOptIn immediately turns on ScoutGroupSelected pref. + prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + ONLY_SHOW_SCOUT_OPT_IN, MAX_REASONS); + } else if (base::FeatureList::IsEnabled(kCanShowScoutOptIn)) { + // CanShowScoutOptIn will only turn on ScoutGroupSelected pref if the legacy + // SBER pref is false. Otherwise the legacy SBER pref will stay on and + // continue to be used until the next security incident, at which point + // the Scout pref will become the active one. + if (!prefs->GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)) { + prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + CAN_SHOW_SCOUT_OPT_IN_SCOUT_GROUP_ON, + MAX_REASONS); + } else { + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + CAN_SHOW_SCOUT_OPT_IN_WAIT_FOR_INTERSTITIAL, + MAX_REASONS); + } + } else { + // Both experiment features are off, so this is the Control group. We must + // handle the possibility that the user was previously in an experiment + // group (above) that was reverted. We want to restore the user to a + // reasonable state based on the ScoutGroup and ScoutReporting preferences. + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, CONTROL, MAX_REASONS); + if (prefs->GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled)) { + // User opted-in to Scout which is broader than legacy Extended Reporting. + // Opt them in to Extended Reporting. + prefs->SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, true); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + ROLLBACK_SBER2_IMPLIES_SBER1, MAX_REASONS); + } else if (prefs->GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) { + // User was in the Scout Group (ie: seeing the Scout opt-in text) but did + // NOT opt-in to Scout. Assume this was a conscious choice and remove + // their legacy Extended Reporting opt-in as well. The user will have a + // chance to evaluate their choice next time they see the opt-in text. + + // We make the Extended Reporting pref mimic the state of the Scout + // Reporting pref. So we either Clear it or set it to False. + if (prefs->HasPrefPath(prefs::kSafeBrowsingScoutReportingEnabled)) { + // Scout Reporting pref was explicitly set to false, so set the SBER + // pref to false. + prefs->SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, false); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + ROLLBACK_NO_SBER2_SET_SBER1_FALSE, + MAX_REASONS); + } else { + // Scout Reporting pref is unset, so clear the SBER pref. + prefs->ClearPref(prefs::kSafeBrowsingExtendedReportingEnabled); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + ROLLBACK_NO_SBER2_CLEAR_SBER1, MAX_REASONS); + } + } + + // Also clear both the Scout settings to start over from a clean state and + // avoid the above logic from triggering on next restart. + prefs->ClearPref(prefs::kSafeBrowsingScoutGroupSelected); + prefs->ClearPref(prefs::kSafeBrowsingScoutReportingEnabled); + } +} + +bool IsExtendedReportingEnabled(const PrefService& prefs) { + return prefs.GetBoolean(GetExtendedReportingPrefName(prefs)); +} + +bool IsScout(const PrefService& prefs) { + return GetExtendedReportingPrefName(prefs) == + prefs::kSafeBrowsingScoutReportingEnabled; +} + +void RecordExtendedReportingMetrics(const PrefService& prefs) { + // This metric tracks the extended browsing opt-in based on whichever setting + // the user is currently seeing. It tells us whether extended reporting is + // happening for this user. + UMA_HISTOGRAM_BOOLEAN("SafeBrowsing.Pref.Extended", + IsExtendedReportingEnabled(prefs)); + + // These metrics track the Scout transition. + if (prefs.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) { + // Users in the Scout group: currently seeing the Scout opt-in. + UMA_HISTOGRAM_BOOLEAN( + "SafeBrowsing.Pref.Scout.ScoutGroup.SBER1Pref", + prefs.GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)); + UMA_HISTOGRAM_BOOLEAN( + "SafeBrowsing.Pref.Scout.ScoutGroup.SBER2Pref", + prefs.GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled)); + } else { + // Users not in the Scout group: currently seeing the SBER opt-in. + UMA_HISTOGRAM_BOOLEAN( + "SafeBrowsing.Pref.Scout.NoScoutGroup.SBER1Pref", + prefs.GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)); + // The following metric is a corner case. User was previously in the + // Scout group and was able to opt-in to the Scout pref, but was since + // removed from the Scout group (eg: by rolling back a Scout experiment). + UMA_HISTOGRAM_BOOLEAN( + "SafeBrowsing.Pref.Scout.NoScoutGroup.SBER2Pref", + prefs.GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled)); + } +} + +void SetExtendedReportingPref(PrefService* prefs, bool value) { + prefs->SetBoolean(GetExtendedReportingPrefName(*prefs), value); +} + +void UpdatePrefsBeforeSecurityInterstitial(PrefService* prefs) { + // Move the user into the Scout Group if the CanShowScoutOptIn experiment is + // enabled and they're not in the group already. + if (base::FeatureList::IsEnabled(kCanShowScoutOptIn) && + !prefs->GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)) { + prefs->SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true); + UMA_HISTOGRAM_ENUMERATION(kScoutTransitionMetricName, + CAN_SHOW_SCOUT_OPT_IN_SAW_FIRST_INTERSTITIAL, + MAX_REASONS); + } +} + +} // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/safe_browsing_prefs.h b/chromium/components/safe_browsing_db/safe_browsing_prefs.h new file mode 100644 index 00000000000..c8856b40c1f --- /dev/null +++ b/chromium/components/safe_browsing_db/safe_browsing_prefs.h @@ -0,0 +1,113 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// Safe Browsing preferences and some basic utility functions for using them. + +#ifndef COMPONENTS_SAFE_BROWSING_DB_SAFE_BROWSING_PREFS_H_ +#define COMPONENTS_SAFE_BROWSING_DB_SAFE_BROWSING_PREFS_H_ + +#include "base/feature_list.h" + +class PrefService; + +namespace prefs { +// Boolean that tell us whether Safe Browsing extended reporting is enabled. +extern const char kSafeBrowsingExtendedReportingEnabled[]; + +// Boolean indicating whether Safe Browsing Scout reporting is enabled, which +// collects data for malware detection. +extern const char kSafeBrowsingScoutReportingEnabled[]; + +// Boolean indicating whether the Scout reporting workflow is enabled. This +// affects which of SafeBrowsingExtendedReporting or SafeBrowsingScoutReporting +// is used. +extern const char kSafeBrowsingScoutGroupSelected[]; +} + +namespace safe_browsing { + +// Command-line switch for changing the scout_group_selected preference. Should +// be set to either 'true' or 'false'. Primarily for testing purposes. +// TODO: this is temporary (crbug.com/662944) +extern const char kSwitchForceScoutGroup[]; + +// When this feature is enabled, the Scout opt-in text will be displayed as of +// the next security incident. Until then, the legacy SBER text will appear. +// TODO: this is temporary (crbug.com/662944) +extern const base::Feature kCanShowScoutOptIn; + +// When this feature is enabled, the Scout opt-in text will immediately be +// displayed everywhere. +// TODO: this is temporary (crbug.com/662944) +extern const base::Feature kOnlyShowScoutOptIn; + +// Enumerates the level of Safe Browsing Extended Reporting that is currently +// available. +enum ExtendedReportingLevel { + // Extended reporting is off. + SBER_LEVEL_OFF = 0, + // The Legacy level of extended reporting is available, reporting happens in + // response to security incidents. + SBER_LEVEL_LEGACY = 1, + // The Scout level of extended reporting is available, some data can be + // collected to actively detect dangerous apps and sites. + SBER_LEVEL_SCOUT = 2, +}; + +// Determines which opt-in text should be used based on the currently active +// preference. Will return either |extended_reporting_pref| if the legacy +// Extended Reporting pref is active, or |scout_pref| if the Scout pref is +// active. Used for Android. +std::string ChooseOptInTextPreference( + const PrefService& prefs, + const std::string& extended_reporting_pref, + const std::string& scout_pref); + +// Determines which opt-in text should be used based on the currently active +// preference. Will return either |extended_reporting_resource| if the legacy +// Extended Reporting pref is active, or |scout_resource| if the Scout pref is +// active. +int ChooseOptInTextResource(const PrefService& prefs, + int extended_reporting_resource, + int scout_resource); + +// Returns whether the currently active Safe Browsing Extended Reporting +// preference exists (eg: has been set before). +bool ExtendedReportingPrefExists(const PrefService& prefs); + +// Returns the level of reporting available for the current user. +ExtendedReportingLevel GetExtendedReportingLevel(const PrefService& prefs); + +// Returns the name of the Safe Browsing Extended Reporting pref that is +// currently in effect. The specific pref in-use may change through experiments. +const char* GetExtendedReportingPrefName(const PrefService& prefs); + +// Initializes Safe Browsing preferences based on data such as experiment state, +// command line flags, etc. +// TODO: this is temporary (crbug.com/662944) +void InitializeSafeBrowsingPrefs(PrefService* prefs); + +// Returns whether Safe Browsing Extended Reporting is currently enabled. +// This should be used to decide if any of the reporting preferences are set, +// regardless of which specific one is set. +bool IsExtendedReportingEnabled(const PrefService& prefs); + +// Returns whether the currently-active Extended Reporting pref is Scout. +bool IsScout(const PrefService& prefs); + +// Updates UMA metrics about Safe Browsing Extended Reporting states. +void RecordExtendedReportingMetrics(const PrefService& prefs); + +// Sets the currently active Safe Browsing Extended Reporting preference to the +// specified value. +void SetExtendedReportingPref(PrefService* prefs, bool value); + +// Called to indicate that a security interstitial is about to be shown to the +// user. This may trigger the user to begin seeing the Scout opt-in text +// depending on their experiment state. +void UpdatePrefsBeforeSecurityInterstitial(PrefService* prefs); + +} // namespace safe_browsing + +#endif // COMPONENTS_SAFE_BROWSING_DB_SAFE_BROWSING_PREFS_H_ diff --git a/chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc b/chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc new file mode 100644 index 00000000000..af38c78edf8 --- /dev/null +++ b/chromium/components/safe_browsing_db/safe_browsing_prefs_unittest.cc @@ -0,0 +1,457 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <string> +#include <vector> + +#include "base/command_line.h" +#include "base/strings/string_util.h" +#include "base/test/scoped_feature_list.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/testing_pref_service.h" +#include "components/safe_browsing_db/safe_browsing_prefs.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace safe_browsing { + +class SafeBrowsingPrefsTest : public ::testing::Test { + protected: + void SetUp() override { + prefs_.registry()->RegisterBooleanPref( + prefs::kSafeBrowsingExtendedReportingEnabled, false); + prefs_.registry()->RegisterBooleanPref( + prefs::kSafeBrowsingScoutReportingEnabled, false); + prefs_.registry()->RegisterBooleanPref( + prefs::kSafeBrowsingScoutGroupSelected, false); + + ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/false); + } + + void ResetPrefs(bool sber_reporting, bool scout_reporting, bool scout_group) { + prefs_.SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, + sber_reporting); + prefs_.SetBoolean(prefs::kSafeBrowsingScoutReportingEnabled, + scout_reporting); + prefs_.SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, scout_group); + } + + void ResetExperiments(bool can_show_scout, bool only_show_scout) { + std::vector<std::string> enabled_features; + std::vector<std::string> disabled_features; + + auto* target_vector = + can_show_scout ? &enabled_features : &disabled_features; + target_vector->push_back(kCanShowScoutOptIn.name); + + target_vector = only_show_scout ? &enabled_features : &disabled_features; + target_vector->push_back(kOnlyShowScoutOptIn.name); + + feature_list_.reset(new base::test::ScopedFeatureList); + feature_list_->InitFromCommandLine( + base::JoinString(enabled_features, ","), + base::JoinString(disabled_features, ",")); + } + + std::string GetActivePref() { return GetExtendedReportingPrefName(prefs_); } + + // Convenience method for explicitly setting up all combinations of prefs and + // experiments. + void TestGetPrefName(bool sber_reporting, + bool scout_reporting, + bool scout_group, + bool can_show_scout, + bool only_show_scout, + const std::string& expected_pref) { + ResetPrefs(sber_reporting, scout_reporting, scout_group); + ResetExperiments(can_show_scout, only_show_scout); + EXPECT_EQ(expected_pref, GetActivePref()) + << "sber=" << sber_reporting << " scout=" << scout_reporting + << " scout_group=" << scout_group + << " can_show_scout=" << can_show_scout + << " only_show_scout=" << only_show_scout; + } + + void InitPrefs() { InitializeSafeBrowsingPrefs(&prefs_); } + + bool IsScoutGroupSelected() { + return prefs_.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected); + } + + void ExpectPrefs(bool sber_reporting, + bool scout_reporting, + bool scout_group) { + LOG(INFO) << "Pref values: sber=" << sber_reporting + << " scout=" << scout_reporting << " scout_group=" << scout_group; + EXPECT_EQ(sber_reporting, + prefs_.GetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled)); + EXPECT_EQ(scout_reporting, + prefs_.GetBoolean(prefs::kSafeBrowsingScoutReportingEnabled)); + EXPECT_EQ(scout_group, + prefs_.GetBoolean(prefs::kSafeBrowsingScoutGroupSelected)); + } + + void ExpectPrefsExist(bool sber_reporting, + bool scout_reporting, + bool scout_group) { + LOG(INFO) << "Prefs exist: sber=" << sber_reporting + << " scout=" << scout_reporting << " scout_group=" << scout_group; + EXPECT_EQ(sber_reporting, + prefs_.HasPrefPath(prefs::kSafeBrowsingExtendedReportingEnabled)); + EXPECT_EQ(scout_reporting, + prefs_.HasPrefPath(prefs::kSafeBrowsingScoutReportingEnabled)); + EXPECT_EQ(scout_group, + prefs_.HasPrefPath(prefs::kSafeBrowsingScoutGroupSelected)); + } + TestingPrefServiceSimple prefs_; + + private: + std::unique_ptr<base::test::ScopedFeatureList> feature_list_; +}; + +// This test ensures that we correctly select between SBER and Scout as the +// active preference in a number of common scenarios. +TEST_F(SafeBrowsingPrefsTest, GetExtendedReportingPrefName_Common) { + const std::string& sber = prefs::kSafeBrowsingExtendedReportingEnabled; + const std::string& scout = prefs::kSafeBrowsingScoutReportingEnabled; + + // By default (all prefs and experiment features disabled), SBER pref is used. + TestGetPrefName(false, false, false, false, false, sber); + + // Changing any prefs (including ScoutGroupSelected) keeps SBER as the active + // pref because the experiment remains in the Control group. + TestGetPrefName(/*sber=*/true, false, false, false, false, sber); + TestGetPrefName(false, /*scout=*/true, false, false, false, sber); + TestGetPrefName(false, false, /*scout_group=*/true, false, false, sber); + + // Being in either experiment group with ScoutGroup selected makes Scout the + // active pref. + TestGetPrefName(false, false, /*scout_group=*/true, /*can_show_scout=*/true, + false, scout); + TestGetPrefName(false, false, /*scout_group=*/true, false, + /*only_show_scout=*/true, scout); + + // When ScoutGroup is not selected then SBER remains the active pref, + // regardless which experiment is enabled. + TestGetPrefName(false, false, false, /*can_show_scout=*/true, false, sber); + TestGetPrefName(false, false, false, false, /*only_show_scout=*/true, sber); +} + +// Here we exhaustively check all combinations of pref and experiment states. +// This should help catch regressions. +TEST_F(SafeBrowsingPrefsTest, GetExtendedReportingPrefName_Exhaustive) { + const std::string& sber = prefs::kSafeBrowsingExtendedReportingEnabled; + const std::string& scout = prefs::kSafeBrowsingScoutReportingEnabled; + TestGetPrefName(false, false, false, false, false, sber); + TestGetPrefName(false, false, false, false, true, sber); + TestGetPrefName(false, false, false, true, false, sber); + TestGetPrefName(false, false, false, true, true, sber); + TestGetPrefName(false, false, true, false, false, sber); + TestGetPrefName(false, false, true, false, true, scout); + TestGetPrefName(false, false, true, true, false, scout); + TestGetPrefName(false, false, true, true, true, scout); + TestGetPrefName(false, true, false, false, false, sber); + TestGetPrefName(false, true, false, false, true, sber); + TestGetPrefName(false, true, false, true, false, sber); + TestGetPrefName(false, true, false, true, true, sber); + TestGetPrefName(false, true, true, false, false, sber); + TestGetPrefName(false, true, true, false, true, scout); + TestGetPrefName(false, true, true, true, false, scout); + TestGetPrefName(false, true, true, true, true, scout); + TestGetPrefName(true, false, false, false, false, sber); + TestGetPrefName(true, false, false, false, true, sber); + TestGetPrefName(true, false, false, true, false, sber); + TestGetPrefName(true, false, false, true, true, sber); + TestGetPrefName(true, false, true, false, false, sber); + TestGetPrefName(true, false, true, false, true, scout); + TestGetPrefName(true, false, true, true, false, scout); + TestGetPrefName(true, false, true, true, true, scout); + TestGetPrefName(true, true, false, false, false, sber); + TestGetPrefName(true, true, false, false, true, sber); + TestGetPrefName(true, true, false, true, false, sber); + TestGetPrefName(true, true, false, true, true, sber); + TestGetPrefName(true, true, true, false, false, sber); + TestGetPrefName(true, true, true, false, true, scout); + TestGetPrefName(true, true, true, true, false, scout); + TestGetPrefName(true, true, true, true, true, scout); +} + +// Basic test that command-line flags can force the ScoutGroupSelected pref on +// or off. +TEST_F(SafeBrowsingPrefsTest, InitPrefs_ForceScoutGroupOnOff) { + // By default ScoutGroupSelected is off. + EXPECT_FALSE(IsScoutGroupSelected()); + + // Command-line flag can force it on during initialization. + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + kSwitchForceScoutGroup, "true"); + InitPrefs(); + EXPECT_TRUE(IsScoutGroupSelected()); + + // ScoutGroup remains on if switches are cleared, but only if an experiment + // is active (since being in the Control group automatically clears the + // Scout prefs). + base::CommandLine::StringVector empty; + base::CommandLine::ForCurrentProcess()->InitFromArgv(empty); + ResetExperiments(/*can_show_scout=*/true, /*only_show_scout=*/false); + EXPECT_TRUE(IsScoutGroupSelected()); + + // Nonsense values are ignored and ScoutGroup is unchanged. + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + kSwitchForceScoutGroup, "foo"); + InitPrefs(); + EXPECT_TRUE(IsScoutGroupSelected()); + + // ScoutGroup can also be forced off during initialization. + base::CommandLine::ForCurrentProcess()->InitFromArgv(empty); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + kSwitchForceScoutGroup, "false"); + InitPrefs(); + EXPECT_FALSE(IsScoutGroupSelected()); +} + +// Test all combinations of prefs during initialization when neither experiment +// is on (ie: control group). In all cases the Scout prefs should be cleared, +// and the SBER pref may get switched. +TEST_F(SafeBrowsingPrefsTest, InitPrefs_Control) { + // Turn both experiments off. + ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/false); + + // Default case (everything off) - no change on init. + ResetPrefs(false, false, false); + InitPrefs(); + ExpectPrefs(false, false, false); + // SBER pref exists because it was set to false above. + ExpectPrefsExist(true, false, false); + + // ScoutGroup on - SBER cleared since Scout opt-in was shown but Scout pref + // was not chosen. Scout prefs cleared. + ResetPrefs(false, false, true); + InitPrefs(); + ExpectPrefs(false, false, false); + ExpectPrefsExist(true, false, false); + + // ScoutReporting on without ScoutGroup - SBER turns on since user opted-in to + // broader Scout reporting, we can continue collecting the SBER subset. Scout + // prefs cleared. + ResetPrefs(false, true, false); + InitPrefs(); + ExpectPrefs(true, false, false); + ExpectPrefsExist(true, false, false); + + // ScoutReporting and ScoutGroup on - SBER turns on since user opted-in to + // broader Scout reporting, we can continue collecting the SBER subset. Scout + // prefs cleared. + ResetPrefs(false, true, true); + InitPrefs(); + ExpectPrefs(true, false, false); + ExpectPrefsExist(true, false, false); + + // SBER on - no change on init since ScoutGroup is off implying that user + // never saw Scout opt-in text. Scout prefs remain cleared. + ResetPrefs(true, false, false); + InitPrefs(); + ExpectPrefs(true, false, false); + ExpectPrefsExist(true, false, false); + + // SBER and ScoutGroup on - SBER cleared. User previously opted-in to SBER + // and they saw Scout opt-in text (ie. ScoutGroup on), but chose not to opt-in + // to Scout reporting. We want them to re-evaluate their choice of SBER since + // the lack of Scout opt-in was a conscious choice. Scout cleared. + ResetPrefs(true, false, true); + InitPrefs(); + ExpectPrefs(false, false, false); + ExpectPrefsExist(true, false, false); + + // SBER and ScoutReporting on. User has opted-in to broader level of reporting + // so SBER stays on. Scout prefs cleared. + ResetPrefs(true, true, false); + InitPrefs(); + ExpectPrefs(true, false, false); + ExpectPrefsExist(true, false, false); + + // Everything on. User has opted-in to broader level of reporting so SBER + // stays on. Scout prefs cleared. + ResetPrefs(true, true, true); + InitPrefs(); + ExpectPrefs(true, false, false); + ExpectPrefs(true, false, false); +} + +// Tests a unique case where the Extended Reporting pref will be Cleared instead +// of set to False in order to mimic the state of the Scout reporting pref. +// This happens when a user is in the OnlyShowScoutOptIn experiment but never +// encounters a security issue so never sees the Scout opt-in. This user then +// returns to the Control group having never seen the Scout opt-in, so their +// Scout Reporting pref is un-set. We want to return their SBER pref to the +// unset state as well. +TEST_F(SafeBrowsingPrefsTest, InitPrefs_Control_SberPrefCleared) { + // Turn both experiments off. + ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/false); + + // Set the user's old SBER pref to on to be explicit. + prefs_.SetBoolean(prefs::kSafeBrowsingExtendedReportingEnabled, true); + // User is in the OnlyShowScoutOptIn experiment so they go directly to the + // Scout Group. + prefs_.SetBoolean(prefs::kSafeBrowsingScoutGroupSelected, true); + // But they never see a security popup or change the setting manually so the + // Scout pref remains unset. + prefs_.ClearPref(prefs::kSafeBrowsingScoutReportingEnabled); + + InitPrefs(); + + // All pref values should be false and unset. + ExpectPrefs(false, false, false); + ExpectPrefsExist(false, false, false); +} + +// Test all combinations of prefs during initialization when the CanShowScout +// experiment is on. +TEST_F(SafeBrowsingPrefsTest, InitPrefs_CanShowScout) { + // Turn the CanShowScout experiment on. + ResetExperiments(/*can_show_scout=*/true, /*only_show_scout=*/false); + + // Default case (everything off) - ScoutGroup turns on because SBER is off. + ResetPrefs(false, false, false); + InitPrefs(); + ExpectPrefs(false, false, true); + + // ScoutGroup on - no change on init since ScoutGroup is already on. + ResetPrefs(false, false, true); + InitPrefs(); + ExpectPrefs(false, false, true); + + // ScoutReporting on without ScoutGroup - ScoutGroup turns on because SBER is + // off. + ResetPrefs(false, true, false); + InitPrefs(); + ExpectPrefs(false, true, true); + + // ScoutReporting and ScoutGroup on - no change on init since ScoutGroup is + // already on. + ResetPrefs(false, true, true); + InitPrefs(); + ExpectPrefs(false, true, true); + + // SBER on - no change on init. Will wait for first security incident before + // turning on ScoutGroup and displaying the Scout opt-in. + ResetPrefs(true, false, false); + InitPrefs(); + ExpectPrefs(true, false, false); + + // SBER and ScoutGroup on - no change on init since ScoutGroup is already on. + ResetPrefs(true, false, true); + InitPrefs(); + ExpectPrefs(true, false, true); + + // SBER and ScoutReporting on - no change on init because SBER is on. + // ScoutGroup will turn on on next security incident. + ResetPrefs(true, true, false); + InitPrefs(); + ExpectPrefs(true, true, false); + + // Everything on - no change on init since ScoutGroup is already on. + ResetPrefs(true, true, true); + InitPrefs(); + ExpectPrefs(true, true, true); +} + +// Test all combinations of prefs during initialization when the OnlyShowScout +// experiment is on. +TEST_F(SafeBrowsingPrefsTest, InitPrefs_OnlyShowScout) { + // Turn the OnlyShowScout experiment on. + ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/true); + + // Default case (everything off) - ScoutGroup turns on. + ResetPrefs(false, false, false); + InitPrefs(); + ExpectPrefs(false, false, true); + + // ScoutGroup on - no change on init since ScoutGroup is already on. + ResetPrefs(false, false, true); + InitPrefs(); + ExpectPrefs(false, false, true); + + // ScoutReporting on without ScoutGroup - ScoutGroup turns on. + ResetPrefs(false, true, false); + InitPrefs(); + ExpectPrefs(false, true, true); + + // ScoutReporting and ScoutGroup on - no change on init since ScoutGroup is + // already on. + ResetPrefs(false, true, true); + InitPrefs(); + ExpectPrefs(false, true, true); + + // SBER on - ScoutGroup turns on immediately, not waiting for first security + // incident. + ResetPrefs(true, false, false); + InitPrefs(); + ExpectPrefs(true, false, true); + + // SBER and ScoutGroup on - no change on init since ScoutGroup is already on. + ResetPrefs(true, false, true); + InitPrefs(); + ExpectPrefs(true, false, true); + + // SBER and ScoutReporting on - ScoutGroup turns on immediately, not waiting + // for first security incident. + ResetPrefs(true, true, false); + InitPrefs(); + ExpectPrefs(true, true, true); + + // Everything on - no change on init since ScoutGroup is already on. + ResetPrefs(true, true, true); + InitPrefs(); + ExpectPrefs(true, true, true); +} + +TEST_F(SafeBrowsingPrefsTest, ChooseOptInText) { + const int kSberResource = 100; + const int kScoutResource = 500; + // By default, SBER opt-in is used + EXPECT_EQ(kSberResource, + ChooseOptInTextResource(prefs_, kSberResource, kScoutResource)); + + // Enabling Scout switches to the Scout opt-in text. + ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/true); + ResetPrefs(/*sber=*/false, /*scout=*/false, /*scout_group=*/true); + EXPECT_EQ(kScoutResource, + ChooseOptInTextResource(prefs_, kSberResource, kScoutResource)); +} + +TEST_F(SafeBrowsingPrefsTest, GetSafeBrowsingExtendedReportingLevel) { + // By Default, SBER is off + EXPECT_EQ(SBER_LEVEL_OFF, GetExtendedReportingLevel(prefs_)); + + // Opt-in to Legacy SBER gives Legacy reporting leve. + ResetPrefs(/*sber=*/true, /*scout_reporting=*/false, /*scout_group=*/false); + EXPECT_EQ(SBER_LEVEL_LEGACY, GetExtendedReportingLevel(prefs_)); + + // The value of the Scout pref doesn't change the reporting level if the user + // is outside of the Scout Group and/or no experiment is running. + // No scout group. + ResetPrefs(/*sber=*/true, /*scout_reporting=*/true, /*scout_group=*/false); + EXPECT_EQ(SBER_LEVEL_LEGACY, GetExtendedReportingLevel(prefs_)); + // Scout group but no experiment. + ResetPrefs(/*sber=*/true, /*scout_reporting=*/true, /*scout_group=*/true); + EXPECT_EQ(SBER_LEVEL_LEGACY, GetExtendedReportingLevel(prefs_)); + + // Remaining in the Scout Group and adding an experiment will switch to the + // Scout pref to determine reporting level. + ResetExperiments(/*can_show_scout=*/false, /*only_show_scout=*/true); + // Both reporting prefs off, so reporting is off. + ResetPrefs(/*sber=*/false, /*scout_reporting=*/false, /*scout_group=*/true); + EXPECT_EQ(SBER_LEVEL_OFF, GetExtendedReportingLevel(prefs_)); + // Legacy pref on when we're using Scout - reporting remains off. + ResetPrefs(/*sber=*/true, /*scout_reporting=*/false, /*scout_group=*/true); + EXPECT_EQ(SBER_LEVEL_OFF, GetExtendedReportingLevel(prefs_)); + // Turning on Scout gives us Scout level reporting + ResetPrefs(/*sber=*/false, /*scout_reporting=*/true, /*scout_group=*/true); + EXPECT_EQ(SBER_LEVEL_SCOUT, GetExtendedReportingLevel(prefs_)); + // .. and the legacy pref doesn't affect this. + ResetPrefs(/*sber=*/true, /*scout_reporting=*/true, /*scout_group=*/true); + EXPECT_EQ(SBER_LEVEL_SCOUT, GetExtendedReportingLevel(prefs_)); +} + +} // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/safebrowsing.proto b/chromium/components/safe_browsing_db/safebrowsing.proto index c9c24deb953..fb5785d28d7 100644 --- a/chromium/components/safe_browsing_db/safebrowsing.proto +++ b/chromium/components/safe_browsing_db/safebrowsing.proto @@ -267,6 +267,18 @@ enum ThreatType { // API abuse threat type. API_ABUSE = 6; + + // Malicious binary threat type. + MALICIOUS_BINARY = 7; + + // Client side detection whitelist threat type. + CSD_WHITELIST = 8; + + // Client side download detection whitelist threat type. + CSD_DOWNLOAD_WHITELIST = 9; + + // Client incident threat type. + CLIENT_INCIDENT = 10; } // Types of platforms. @@ -354,6 +366,15 @@ enum ThreatEntryType { // An IP range. IP_RANGE = 3; + + // Chrome extension. + CHROME_EXTENSION = 4; + + // Filename. + FILENAME = 5; + + // CERT. + CERT = 6; } // A set of threats that should be added or removed from a client's local diff --git a/chromium/components/safe_browsing_db/testing_util.h b/chromium/components/safe_browsing_db/testing_util.h deleted file mode 100644 index 8447162681c..00000000000 --- a/chromium/components/safe_browsing_db/testing_util.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// Utilities to be used in tests of safe_browsing_db/ component. - -#ifndef COMPONENTS_SAFE_BROWSING_DB_TESTING_UTIL_H_ -#define COMPONENTS_SAFE_BROWSING_DB_TESTING_UTIL_H_ - -#include "components/safe_browsing_db/util.h" - -#include <ostream> - -namespace safe_browsing { - -inline std::ostream& operator<<(std::ostream& os, const ThreatMetadata& meta) { - os << "{threat_pattern_type=" << static_cast<int>(meta.threat_pattern_type) - << ", api_permissions=["; - for (auto p : meta.api_permissions) - os << p << ","; - return os << "], population_id=" << meta.population_id << "}"; -} - -} // namespace safe_browsing - -#endif // COMPONENTS_SAFE_BROWSING_DB_TESTING_UTIL_H_ diff --git a/chromium/components/safe_browsing_db/util.cc b/chromium/components/safe_browsing_db/util.cc index 3f34fe5fb09..0f38dee61bd 100644 --- a/chromium/components/safe_browsing_db/util.cc +++ b/chromium/components/safe_browsing_db/util.cc @@ -134,6 +134,7 @@ bool GetListName(ListType list_id, std::string* list) { break; case MODULEWHITELIST: *list = kModuleWhitelist; + break; case RESOURCEBLACKLIST: *list = kResourceBlacklist; break; diff --git a/chromium/components/safe_browsing_db/v4_database.cc b/chromium/components/safe_browsing_db/v4_database.cc index 6e14e5f099d..5edeb3e9422 100644 --- a/chromium/components/safe_browsing_db/v4_database.cc +++ b/chromium/components/safe_browsing_db/v4_database.cc @@ -8,14 +8,22 @@ #include "base/debug/leak_annotations.h" #include "base/files/file_util.h" #include "base/memory/ptr_util.h" +#include "base/metrics/histogram_macros.h" #include "base/threading/thread_task_runner_handle.h" #include "components/safe_browsing_db/v4_database.h" #include "content/public/browser/browser_thread.h" using content::BrowserThread; +using base::TimeTicks; namespace safe_browsing { +namespace { + +const char kV4DatabaseSizeMetric[] = "SafeBrowsing.V4Database.Size"; + +} // namespace + // static V4StoreFactory* V4Database::factory_ = NULL; @@ -31,9 +39,9 @@ void V4Database::Create( const scoped_refptr<base::SingleThreadTaskRunner> callback_task_runner = base::ThreadTaskRunnerHandle::Get(); db_task_runner->PostTask( - FROM_HERE, - base::Bind(&V4Database::CreateOnTaskRunner, db_task_runner, base_path, - list_infos, callback_task_runner, new_db_callback)); + FROM_HERE, base::Bind(&V4Database::CreateOnTaskRunner, db_task_runner, + base_path, list_infos, callback_task_runner, + new_db_callback, TimeTicks::Now())); } // static @@ -42,7 +50,8 @@ void V4Database::CreateOnTaskRunner( const base::FilePath& base_path, const ListInfos& list_infos, const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, - NewDatabaseReadyCallback new_db_callback) { + NewDatabaseReadyCallback new_db_callback, + const TimeTicks create_start_time) { DCHECK(db_task_runner->RunsTasksOnCurrentThread()); if (!factory_) { @@ -69,9 +78,12 @@ void V4Database::CreateOnTaskRunner( new V4Database(db_task_runner, std::move(store_map))); // Database is done loading, pass it to the new_db_callback on the caller's - // thread. + // thread. This would unblock resource loads. callback_task_runner->PostTask( FROM_HERE, base::Bind(new_db_callback, base::Passed(&v4_database))); + + UMA_HISTOGRAM_TIMES("SafeBrowsing.V4DatabaseOpen.Time", + TimeTicks::Now() - create_start_time); } V4Database::V4Database( @@ -81,11 +93,11 @@ V4Database::V4Database( store_map_(std::move(store_map)), pending_store_updates_(0) { DCHECK(db_task_runner->RunsTasksOnCurrentThread()); - // TODO(vakh): Implement skeleton } // static void V4Database::Destroy(std::unique_ptr<V4Database> v4_database) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); V4Database* v4_database_raw = v4_database.release(); if (v4_database_raw) { v4_database_raw->db_task_runner_->DeleteSoon(FROM_HERE, v4_database_raw); @@ -142,7 +154,9 @@ void V4Database::UpdatedStoreReady(ListIdentifier identifier, DCHECK_CURRENTLY_ON(BrowserThread::IO); DCHECK(pending_store_updates_); if (new_store) { - (*store_map_)[identifier] = std::move(new_store); + (*store_map_)[identifier].swap(new_store); + // |new_store| now is the store that needs to be destroyed on task runner. + V4Store::Destroy(std::move(new_store)); } pending_store_updates_--; @@ -152,17 +166,6 @@ void V4Database::UpdatedStoreReady(ListIdentifier identifier, } } -bool V4Database::ResetDatabase() { - DCHECK(db_task_runner_->RunsTasksOnCurrentThread()); - bool reset_success = true; - for (const auto& store_map_iter : *store_map_) { - if (!store_map_iter.second->Reset()) { - reset_success = false; - } - } - return reset_success; -} - std::unique_ptr<StoreStateMap> V4Database::GetStoreStateMap() { std::unique_ptr<StoreStateMap> store_state_map = base::MakeUnique<StoreStateMap>(); @@ -172,6 +175,19 @@ std::unique_ptr<StoreStateMap> V4Database::GetStoreStateMap() { return store_state_map; } +bool V4Database::AreStoresAvailable( + const StoresToCheck& stores_to_check) const { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + for (const ListIdentifier& identifier : stores_to_check) { + const auto& store_pair = store_map_->find(identifier); + if (store_pair == store_map_->end()) + return false; // Store not in our list + if (!store_pair->second->HasValidData()) + return false; // Store never properly populated. + } + return true; +} + void V4Database::GetStoresMatchingFullHash( const FullHash& full_hash, const StoresToCheck& stores_to_check, @@ -189,6 +205,51 @@ void V4Database::GetStoresMatchingFullHash( } } +void V4Database::ResetStores( + const std::vector<ListIdentifier>& stores_to_reset) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + for (const ListIdentifier& identifier : stores_to_reset) { + store_map_->at(identifier)->Reset(); + } +} + +void V4Database::VerifyChecksum( + DatabaseReadyForUpdatesCallback db_ready_for_updates_callback) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + // TODO(vakh): Consider using PostTaskAndReply instead. + const scoped_refptr<base::SingleThreadTaskRunner> callback_task_runner = + base::ThreadTaskRunnerHandle::Get(); + db_task_runner_->PostTask( + FROM_HERE, base::Bind(&V4Database::VerifyChecksumOnTaskRunner, + base::Unretained(this), callback_task_runner, + db_ready_for_updates_callback)); +} + +void V4Database::VerifyChecksumOnTaskRunner( + const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, + DatabaseReadyForUpdatesCallback db_ready_for_updates_callback) { + std::vector<ListIdentifier> stores_to_reset; + for (const auto& store_map_iter : *store_map_) { + if (!store_map_iter.second->VerifyChecksum()) { + stores_to_reset.push_back(store_map_iter.first); + } + } + + callback_task_runner->PostTask( + FROM_HERE, base::Bind(db_ready_for_updates_callback, stores_to_reset)); +} + +void V4Database::RecordFileSizeHistograms() { + int64_t db_size = 0; + for (const auto& store_map_iter : *store_map_) { + const int64_t size = + store_map_iter.second->RecordAndReturnFileSize(kV4DatabaseSizeMetric); + db_size += size; + } + const int64_t db_size_kilobytes = static_cast<int64_t>(db_size / 1024); + UMA_HISTOGRAM_COUNTS(kV4DatabaseSizeMetric, db_size_kilobytes); +} + ListInfo::ListInfo(const bool fetch_updates, const std::string& filename, const ListIdentifier& list_id, diff --git a/chromium/components/safe_browsing_db/v4_database.h b/chromium/components/safe_browsing_db/v4_database.h index bddba598850..19fee93a53b 100644 --- a/chromium/components/safe_browsing_db/v4_database.h +++ b/chromium/components/safe_browsing_db/v4_database.h @@ -17,9 +17,17 @@ namespace safe_browsing { class V4Database; +// Scheduled when the database has been read from disk and is ready to process +// resource reputation requests. typedef base::Callback<void(std::unique_ptr<V4Database>)> NewDatabaseReadyCallback; +// Scheduled when the checksum for all the stores in the database has been +// verified to match the expected value. Stores for which the checksum did not +// match are passed as the argument and need to be reset. +typedef base::Callback<void(const std::vector<ListIdentifier>&)> + DatabaseReadyForUpdatesCallback; + // This callback is scheduled once the database has finished processing the // update requests for all stores and is ready to process the next set of update // requests. @@ -108,6 +116,13 @@ class V4Database { // Returns the current state of each of the stores being managed. std::unique_ptr<StoreStateMap> GetStoreStateMap(); + // Check if all the selected stores are available and populated. + // Returns false if any of |stores_to_check| don't have valid data. + // A store may be unavailble if either it hasn't yet gotten a proper + // full-update (just after install, or corrupted/missing file), or if it's + // not supported in this build (i.e. Chromium). + virtual bool AreStoresAvailable(const StoresToCheck& stores_to_check) const; + // Searches for a hash prefix matching the |full_hash| in stores in the // database, filtered by |stores_to_check|, and returns the identifier of the // store along with the matching hash prefix in |matched_hash_prefix_map|. @@ -116,8 +131,21 @@ class V4Database { const StoresToCheck& stores_to_check, StoreAndHashPrefixes* matched_store_and_full_hashes); - // Deletes the current database and creates a new one. - virtual bool ResetDatabase(); + // Resets the stores in |stores_to_reset| to an empty state. This is done if + // the checksum doesn't match the expected value. + void ResetStores(const std::vector<ListIdentifier>& stores_to_reset); + + // Schedules verification of the checksum of each store read from disk on task + // runner. If the checksum doesn't match, that store is passed to the + // |db_ready_for_updates_callback|. At the end, + // |db_ready_for_updates_callback| is scheduled (on the same thread as it was + // called) to indicate that the database updates can now be scheduled. + void VerifyChecksum( + DatabaseReadyForUpdatesCallback db_ready_for_updates_callback); + + // Records the size of each of the stores managed by this database, along + // with the combined size of all the stores. + void RecordFileSizeHistograms(); protected: V4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner, @@ -147,7 +175,8 @@ class V4Database { const base::FilePath& base_path, const ListInfos& list_infos, const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, - NewDatabaseReadyCallback callback); + NewDatabaseReadyCallback callback, + const base::TimeTicks create_start_time); // Callback called when a new store has been created and is ready to be used. // This method updates the store_map_ to point to the new store, which causes @@ -155,6 +184,11 @@ class V4Database { void UpdatedStoreReady(ListIdentifier identifier, std::unique_ptr<V4Store> store); + // See |VerifyChecksum|. + void VerifyChecksumOnTaskRunner( + const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, + DatabaseReadyForUpdatesCallback db_ready_for_updates_callback); + const scoped_refptr<base::SequencedTaskRunner> db_task_runner_; // Map of ListIdentifier to the V4Store. diff --git a/chromium/components/safe_browsing_db/v4_database_unittest.cc b/chromium/components/safe_browsing_db/v4_database_unittest.cc index 2ce05ee3ec1..a9133d74f2b 100644 --- a/chromium/components/safe_browsing_db/v4_database_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_database_unittest.cc @@ -19,50 +19,42 @@ class FakeV4Store : public V4Store { public: FakeV4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner, const base::FilePath& store_path, - const bool reset_succeeds, const bool hash_prefix_matches) : V4Store( task_runner, base::FilePath(store_path.value() + FILE_PATH_LITERAL(".fake"))), - hash_prefix_should_match_(hash_prefix_matches), - reset_succeeds_(reset_succeeds) {} - - bool Reset() override { return reset_succeeds_; } + hash_prefix_should_match_(hash_prefix_matches) {} HashPrefix GetMatchingHashPrefix(const FullHash& full_hash) override { return hash_prefix_should_match_ ? full_hash : HashPrefix(); } + bool HasValidData() const override { return true; } + void set_hash_prefix_matches(bool hash_prefix_matches) { hash_prefix_should_match_ = hash_prefix_matches; } private: bool hash_prefix_should_match_; - bool reset_succeeds_; }; -// This factory creates a "fake" store. It allows the caller to specify that the -// first store should fail on Reset() i.e. return false. All subsequent stores -// always return true. This is used to test the Reset() method in V4Database. +// This factory creates a "fake" store. It allows the caller to specify whether +// the store has a hash prefix matching a full hash. This is used to test the +// |GetStoresMatchingFullHash()| method in |V4Database|. class FakeV4StoreFactory : public V4StoreFactory { public: - FakeV4StoreFactory(bool next_store_reset_fails, bool hash_prefix_matches) - : hash_prefix_should_match_(hash_prefix_matches), - next_store_reset_fails_(next_store_reset_fails) {} + FakeV4StoreFactory(bool hash_prefix_matches) + : hash_prefix_should_match_(hash_prefix_matches) {} V4Store* CreateV4Store( const scoped_refptr<base::SequencedTaskRunner>& task_runner, const base::FilePath& store_path) override { - bool reset_succeeds = !next_store_reset_fails_; - next_store_reset_fails_ = false; - return new FakeV4Store(task_runner, store_path, reset_succeeds, - hash_prefix_should_match_); + return new FakeV4Store(task_runner, store_path, hash_prefix_should_match_); } private: bool hash_prefix_should_match_; - bool next_store_reset_fails_; }; class V4DatabaseTest : public PlatformTest { @@ -81,6 +73,7 @@ class V4DatabaseTest : public PlatformTest { created_but_not_called_back_ = false; created_and_called_back_ = false; + verify_checksum_called_back_ = false; callback_db_updated_ = base::Bind(&V4DatabaseTest::DatabaseUpdated, base::Unretained(this)); @@ -97,10 +90,8 @@ class V4DatabaseTest : public PlatformTest { PlatformTest::TearDown(); } - void RegisterFactory(bool fails_first_reset, - bool hash_prefix_matches = true) { - factory_.reset( - new FakeV4StoreFactory(fails_first_reset, hash_prefix_matches)); + void RegisterFactory(bool hash_prefix_matches = true) { + factory_.reset(new FakeV4StoreFactory(hash_prefix_matches)); V4Database::RegisterStoreFactoryForTest(factory_.get()); } @@ -137,8 +128,6 @@ class V4DatabaseTest : public PlatformTest { EXPECT_EQ(expected_store_path, store->store_path()); } - EXPECT_EQ(expected_resets_successfully_, v4_database->ResetDatabase()); - EXPECT_FALSE(created_and_called_back_); created_and_called_back_ = true; @@ -195,6 +184,16 @@ class V4DatabaseTest : public PlatformTest { } } + void VerifyChecksumCallback(const std::vector<ListIdentifier>& stores) { + EXPECT_FALSE(verify_checksum_called_back_); + verify_checksum_called_back_ = true; + } + + void WaitForTasksOnTaskRunner() { + task_runner_->RunPendingTasks(); + base::RunLoop().RunUntilIdle(); + } + scoped_refptr<base::TestSimpleTaskRunner> task_runner_; std::unique_ptr<V4Database> v4_database_; base::FilePath database_dirname_; @@ -202,10 +201,10 @@ class V4DatabaseTest : public PlatformTest { content::TestBrowserThreadBundle thread_bundle_; bool created_but_not_called_back_; bool created_and_called_back_; + bool verify_checksum_called_back_; ListInfos list_infos_; std::vector<ListIdentifier> expected_identifiers_; std::vector<base::FilePath> expected_store_paths_; - bool expected_resets_successfully_; std::unique_ptr<FakeV4StoreFactory> factory_; DatabaseUpdatedCallback callback_db_updated_; NewDatabaseReadyCallback callback_db_ready_; @@ -216,42 +215,23 @@ class V4DatabaseTest : public PlatformTest { // Test to set up the database with fake stores. TEST_F(V4DatabaseTest, TestSetupDatabaseWithFakeStores) { - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_); + RegisterFactory(); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - - base::RunLoop().RunUntilIdle(); - EXPECT_EQ(true, created_and_called_back_); -} - -// Test to set up the database with fake stores that fail to reset. -TEST_F(V4DatabaseTest, TestSetupDatabaseWithFakeStoresFailsReset) { - expected_resets_successfully_ = false; - RegisterFactory(!expected_resets_successfully_); - - V4Database::Create(task_runner_, database_dirname_, list_infos_, - callback_db_ready_); - created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); EXPECT_EQ(true, created_and_called_back_); } // Test to check database updates as expected. TEST_F(V4DatabaseTest, TestApplyUpdateWithNewStates) { - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_); + RegisterFactory(); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); // The database has now been created. Time to try to update it. EXPECT_TRUE(v4_database_); @@ -267,22 +247,23 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithNewStates) { CreateFakeServerResponse(expected_store_state_map_, true), callback_db_updated_); - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + // Wait for the ApplyUpdate callback to get called. + WaitForTasksOnTaskRunner(); VerifyExpectedStoresState(true); + + // Wait for the old stores to get destroyed on task runner. + WaitForTasksOnTaskRunner(); } // Test to ensure no state updates leads to no store updates. TEST_F(V4DatabaseTest, TestApplyUpdateWithNoNewState) { - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_); + RegisterFactory(); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); // The database has now been created. Time to try to update it. EXPECT_TRUE(v4_database_); @@ -298,22 +279,19 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithNoNewState) { CreateFakeServerResponse(expected_store_state_map_, true), callback_db_updated_); - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); VerifyExpectedStoresState(false); } // Test to ensure no updates leads to no store updates. TEST_F(V4DatabaseTest, TestApplyUpdateWithEmptyUpdate) { - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_); + RegisterFactory(); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); // The database has now been created. Time to try to update it. EXPECT_TRUE(v4_database_); @@ -330,22 +308,19 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithEmptyUpdate) { v4_database_->ApplyUpdate(std::move(parsed_server_response), callback_db_updated_); - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); VerifyExpectedStoresState(false); } // Test to ensure invalid update leads to no store changes. TEST_F(V4DatabaseTest, TestApplyUpdateWithInvalidUpdate) { - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_); + RegisterFactory(); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); // The database has now been created. Time to try to update it. EXPECT_TRUE(v4_database_); @@ -360,8 +335,7 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithInvalidUpdate) { v4_database_->ApplyUpdate( CreateFakeServerResponse(expected_store_state_map_, false), callback_db_updated_); - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); VerifyExpectedStoresState(false); } @@ -369,15 +343,12 @@ TEST_F(V4DatabaseTest, TestApplyUpdateWithInvalidUpdate) { // Test to ensure the case that all stores match a given full hash. TEST_F(V4DatabaseTest, TestAllStoresMatchFullHash) { bool hash_prefix_matches = true; - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_, hash_prefix_matches); + RegisterFactory(hash_prefix_matches); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); EXPECT_EQ(true, created_and_called_back_); StoresToCheck stores_to_check({linux_malware_id_, win_malware_id_}); @@ -395,15 +366,12 @@ TEST_F(V4DatabaseTest, TestAllStoresMatchFullHash) { // Test to ensure the case that no stores match a given full hash. TEST_F(V4DatabaseTest, TestNoStoreMatchesFullHash) { bool hash_prefix_matches = false; - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_, hash_prefix_matches); + RegisterFactory(hash_prefix_matches); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); EXPECT_EQ(true, created_and_called_back_); StoreAndHashPrefixes store_and_hash_prefixes; @@ -417,15 +385,12 @@ TEST_F(V4DatabaseTest, TestNoStoreMatchesFullHash) { TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHash) { // Setup stores to not match the full hash. bool hash_prefix_matches = false; - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_, hash_prefix_matches); + RegisterFactory(hash_prefix_matches); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); EXPECT_EQ(true, created_and_called_back_); // Set the store corresponding to linux_malware_id_ to match the full hash. @@ -447,15 +412,12 @@ TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHash) { TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHashBecauseOfStoresToMatch) { // Setup all stores to match the full hash. bool hash_prefix_matches = true; - expected_resets_successfully_ = true; - RegisterFactory(!expected_resets_successfully_, hash_prefix_matches); + RegisterFactory(hash_prefix_matches); V4Database::Create(task_runner_, database_dirname_, list_infos_, callback_db_ready_); created_but_not_called_back_ = true; - task_runner_->RunPendingTasks(); - - base::RunLoop().RunUntilIdle(); + WaitForTasksOnTaskRunner(); EXPECT_EQ(true, created_and_called_back_); // Don't add win_malware_id_ to the StoresToCheck. @@ -467,4 +429,51 @@ TEST_F(V4DatabaseTest, TestSomeStoresMatchFullHashBecauseOfStoresToMatch) { EXPECT_FALSE(store_and_hash_prefixes.begin()->hash_prefix.empty()); } +TEST_F(V4DatabaseTest, VerifyChecksumCalledAsync) { + bool hash_prefix_matches = true; + RegisterFactory(hash_prefix_matches); + + V4Database::Create(task_runner_, database_dirname_, list_infos_, + callback_db_ready_); + created_but_not_called_back_ = true; + WaitForTasksOnTaskRunner(); + EXPECT_EQ(true, created_and_called_back_); + + // verify_checksum_called_back_ set to false in the constructor. + EXPECT_FALSE(verify_checksum_called_back_); + // Now call VerifyChecksum and pass the callback that sets + // verify_checksum_called_back_ to true. + v4_database_->VerifyChecksum(base::Bind( + &V4DatabaseTest::VerifyChecksumCallback, base::Unretained(this))); + // verify_checksum_called_back_ should still be false since the checksum + // verification is async. + EXPECT_FALSE(verify_checksum_called_back_); + WaitForTasksOnTaskRunner(); + EXPECT_TRUE(verify_checksum_called_back_); +} + +// Test that we can properly check for unsupported stores +TEST_F(V4DatabaseTest, TestStoresAvailable) { + bool hash_prefix_matches = false; + RegisterFactory(hash_prefix_matches); + + V4Database::Create(task_runner_, database_dirname_, list_infos_, + callback_db_ready_); + created_but_not_called_back_ = true; + WaitForTasksOnTaskRunner(); + EXPECT_EQ(true, created_and_called_back_); + + // Doesn't exist in out list + const ListIdentifier bogus_id(LINUX_PLATFORM, CHROME_EXTENSION, + CSD_WHITELIST); + + EXPECT_TRUE(v4_database_->AreStoresAvailable( + StoresToCheck({linux_malware_id_, win_malware_id_}))); + + EXPECT_FALSE(v4_database_->AreStoresAvailable( + StoresToCheck({linux_malware_id_, bogus_id}))); + + EXPECT_FALSE(v4_database_->AreStoresAvailable(StoresToCheck({bogus_id}))); +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_feature_list.cc b/chromium/components/safe_browsing_db/v4_feature_list.cc index a8435edf051..323ad4db74f 100644 --- a/chromium/components/safe_browsing_db/v4_feature_list.cc +++ b/chromium/components/safe_browsing_db/v4_feature_list.cc @@ -14,17 +14,27 @@ const base::Feature kLocalDatabaseManagerEnabled{ "SafeBrowsingV4LocalDatabaseManagerEnabled", base::FEATURE_DISABLED_BY_DEFAULT}; -const base::Feature kParallelCheckEnabled{"SafeBrowingV4ParallelCheckEnabled", - base::FEATURE_DISABLED_BY_DEFAULT}; +const base::Feature kV4HybridEnabled{"SafeBrowsingV4HybridEnabled", + base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kV4OnlyEnabled{"SafeBrowsingV4OnlyEnabled", + base::FEATURE_DISABLED_BY_DEFAULT}; + } // namespace bool IsLocalDatabaseManagerEnabled() { - return IsParallelCheckEnabled() || - base::FeatureList::IsEnabled(kLocalDatabaseManagerEnabled); + return base::FeatureList::IsEnabled(kLocalDatabaseManagerEnabled) || + IsV4HybridEnabled() || IsV4OnlyEnabled(); +} + +bool IsV4HybridEnabled() { + return base::FeatureList::IsEnabled(kV4HybridEnabled); } -bool IsParallelCheckEnabled() { - return base::FeatureList::IsEnabled(kParallelCheckEnabled); +bool IsV4OnlyEnabled() { + // TODO(vakh): Enable this only when all the lists can be synced from the + // server. See http://b/33182208 + return base::FeatureList::IsEnabled(kV4OnlyEnabled); } } // namespace V4FeatureList diff --git a/chromium/components/safe_browsing_db/v4_feature_list.h b/chromium/components/safe_browsing_db/v4_feature_list.h index b30554143a1..f549a9556ab 100644 --- a/chromium/components/safe_browsing_db/v4_feature_list.h +++ b/chromium/components/safe_browsing_db/v4_feature_list.h @@ -11,11 +11,18 @@ namespace safe_browsing { // through Finch. namespace V4FeatureList { -// Is the Pver4 database manager enabled? +// Is the PVer4 database manager enabled? Should be true if either of those +// below are true. bool IsLocalDatabaseManagerEnabled(); -// Is the Pver4 database manager doing resource checks in paralled with PVer3? -bool IsParallelCheckEnabled(); +// Is the PVer4 database being checked for resource reputation? If this returns +// true, use PVer4 database for CheckBrowseUrl, otherwise use PVer3. +bool IsV4HybridEnabled(); + +// Is only the PVer4 database being checked for resource reputation? If this +// returns true, use PVer4 database for all SafeBrowsing operations, and don't +// update the PVer3 database at all. This is the launch configuration. +bool IsV4OnlyEnabled(); } // namespace V4FeatureList diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc index 96bafa951e3..f0dd8cb1db6 100644 --- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc +++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc @@ -10,6 +10,7 @@ #include "base/macros.h" #include "base/memory/ptr_util.h" #include "base/metrics/histogram_macros.h" +#include "base/stl_util.h" #include "base/timer/timer.h" #include "content/public/browser/browser_thread.h" #include "net/base/load_flags.h" @@ -27,7 +28,7 @@ namespace { // Record a GetHash result. void RecordGetHashResult(safe_browsing::V4OperationResult result) { UMA_HISTOGRAM_ENUMERATION( - "SafeBrowsing.GetV4HashResult", result, + "SafeBrowsing.V4GetHash.Result", result, safe_browsing::V4OperationResult::OPERATION_RESULT_MAX); } @@ -67,7 +68,7 @@ enum ParseResultType { // Record parsing errors of a GetHash result. void RecordParseGetHashResult(ParseResultType result_type) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.ParseV4HashResult", result_type, + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHash.Parse.Result", result_type, PARSE_RESULT_TYPE_MAX); } @@ -90,8 +91,8 @@ enum V4FullHashCacheResultType { // Record a full hash cache hit result. void RecordV4FullHashCacheResult(V4FullHashCacheResultType result_type) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4FullHashCacheResult", result_type, - FULL_HASH_CACHE_RESULT_MAX); + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHash.CacheHit.Result", + result_type, FULL_HASH_CACHE_RESULT_MAX); } // Enumerate GetHash hits/misses for histogramming purposes. DO NOT CHANGE THE @@ -113,7 +114,7 @@ enum V4GetHashCheckResultType { // Record a GetHash hit result. void RecordV4GetHashCheckResult(V4GetHashCheckResultType result_type) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHashCheckResult", result_type, + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4GetHash.Check.Result", result_type, GET_HASH_CHECK_RESULT_MAX); } @@ -131,9 +132,6 @@ const char kPhishing[] = "PHISHING"; namespace safe_browsing { -const char kUmaV4HashResponseMetricName[] = - "SafeBrowsing.GetV4HashHttpResponseOrErrorCode"; - // The default V4GetHashProtocolManagerFactory. class V4GetHashProtocolManagerFactoryImpl : public V4GetHashProtocolManagerFactory { @@ -171,12 +169,14 @@ FullHashCallbackInfo::FullHashCallbackInfo( std::unique_ptr<net::URLFetcher> fetcher, const FullHashToStoreAndHashPrefixesMap& full_hash_to_store_and_hash_prefixes, - const FullHashCallback& callback) + const FullHashCallback& callback, + const base::Time& network_start_time) : cached_full_hash_infos(cached_full_hash_infos), callback(callback), fetcher(std::move(fetcher)), full_hash_to_store_and_hash_prefixes( full_hash_to_store_and_hash_prefixes), + network_start_time(network_start_time), prefixes_requested(prefixes_requested) {} FullHashCallbackInfo::~FullHashCallbackInfo() {} @@ -239,11 +239,18 @@ V4GetHashProtocolManager::V4GetHashProtocolManager( url_fetcher_id_(0), clock_(new base::DefaultClock()) { DCHECK(!stores_to_check.empty()); + std::set<PlatformType> platform_types; + std::set<ThreatEntryType> threat_entry_types; + std::set<ThreatType> threat_types; for (const ListIdentifier& store : stores_to_check) { - platform_types_.insert(store.platform_type()); - threat_entry_types_.insert(store.threat_entry_type()); - threat_types_.insert(store.threat_type()); + platform_types.insert(store.platform_type()); + threat_entry_types.insert(store.threat_entry_type()); + threat_types.insert(store.threat_type()); } + platform_types_.assign(platform_types.begin(), platform_types.end()); + threat_entry_types_.assign(threat_entry_types.begin(), + threat_entry_types.end()); + threat_types_.assign(threat_types.begin(), threat_types.end()); } V4GetHashProtocolManager::~V4GetHashProtocolManager() {} @@ -298,7 +305,7 @@ void V4GetHashProtocolManager::GetFullHashes( net::URLFetcher* fetcher = owned_fetcher.get(); pending_hash_requests_[fetcher].reset(new FullHashCallbackInfo( cached_full_hash_infos, prefixes_to_request, std::move(owned_fetcher), - full_hash_to_store_and_hash_prefixes, callback)); + full_hash_to_store_and_hash_prefixes, callback, clock_->Now())); fetcher->SetExtraRequestHeaders(headers.ToString()); fetcher->SetLoadFlags(net::LOAD_DISABLE_CACHE); @@ -312,7 +319,7 @@ void V4GetHashProtocolManager::GetFullHashesWithApis( DCHECK_CURRENTLY_ON(BrowserThread::IO); DCHECK(url.SchemeIs(url::kHttpScheme) || url.SchemeIs(url::kHttpsScheme)); - std::unordered_set<FullHash> full_hashes; + std::vector<FullHash> full_hashes; V4ProtocolManagerUtil::UrlToFullHashes(url.GetOrigin(), &full_hashes); FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; @@ -470,12 +477,13 @@ void V4GetHashProtocolManager::HandleGetHashError(const Time& now) { void V4GetHashProtocolManager::OnFullHashForApi( const ThreatMetadataForApiCallback& api_callback, - const std::unordered_set<FullHash>& full_hashes, + const std::vector<FullHash>& full_hashes, const std::vector<FullHashInfo>& full_hash_infos) { ThreatMetadata md; for (const FullHashInfo& full_hash_info : full_hash_infos) { DCHECK_EQ(GetChromeUrlApiId(), full_hash_info.list_id); - DCHECK(full_hashes.find(full_hash_info.full_hash) != full_hashes.end()); + DCHECK(std::find(full_hashes.begin(), full_hashes.end(), + full_hash_info.full_hash) != full_hashes.end()); md.api_permissions.insert(full_hash_info.metadata.api_permissions.begin(), full_hash_info.metadata.api_permissions.end()); } @@ -529,6 +537,15 @@ bool V4GetHashProtocolManager::ParseHashResponse( ListIdentifier list_id(match.platform_type(), match.threat_entry_type(), match.threat_type()); + if (!base::ContainsValue(platform_types_, list_id.platform_type()) || + !base::ContainsValue(threat_entry_types_, + list_id.threat_entry_type()) || + !base::ContainsValue(threat_types_, list_id.threat_type())) { + // The server may send a ThreatMatch response for lists that we didn't ask + // for so ignore those ThreatMatch responses. + continue; + } + base::Time positive_expiry; if (match.has_cache_duration()) { // Seconds resolution is good enough so we ignore the nanos field. @@ -539,35 +556,33 @@ bool V4GetHashProtocolManager::ParseHashResponse( } FullHashInfo full_hash_info(match.threat().hash(), list_id, positive_expiry); - if (!ParseMetadata(match, &full_hash_info.metadata)) { - return false; - } - + ParseMetadata(match, &full_hash_info.metadata); full_hash_infos->push_back(full_hash_info); } return true; } -bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match, +// static +void V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match, ThreatMetadata* metadata) { // Different threat types will handle the metadata differently. if (match.threat_type() == API_ABUSE) { if (!match.has_platform_type() || match.platform_type() != CHROME_PLATFORM) { RecordParseGetHashResult(UNEXPECTED_PLATFORM_TYPE_ERROR); - return false; + return; } if (!match.has_threat_entry_metadata()) { RecordParseGetHashResult(NO_METADATA_ERROR); - return false; + return; } // For API Abuse, store a list of the returned permissions. for (const ThreatEntryMetadata::MetadataEntry& m : match.threat_entry_metadata().entries()) { if (m.key() != kPermission) { RecordParseGetHashResult(UNEXPECTED_METADATA_VALUE_ERROR); - return false; + return; } metadata->api_permissions.insert(m.value()); } @@ -575,7 +590,6 @@ bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match, match.threat_type() == POTENTIALLY_HARMFUL_APPLICATION) { for (const ThreatEntryMetadata::MetadataEntry& m : match.threat_entry_metadata().entries()) { - // TODO: Need to confirm the below key/value pairs with CSD backend. if (m.key() == kPhaPatternType || m.key() == kMalwarePatternType) { if (m.value() == kLanding) { metadata->threat_pattern_type = ThreatPatternType::MALWARE_LANDING; @@ -586,7 +600,7 @@ bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match, break; } else { RecordParseGetHashResult(UNEXPECTED_METADATA_VALUE_ERROR); - return false; + return; } } } @@ -607,16 +621,14 @@ bool V4GetHashProtocolManager::ParseMetadata(const ThreatMatch& match, break; } else { RecordParseGetHashResult(UNEXPECTED_METADATA_VALUE_ERROR); - return false; + return; } } } - } else { + } else if (match.has_threat_entry_metadata() && + match.threat_entry_metadata().entries_size() > 1) { RecordParseGetHashResult(UNEXPECTED_THREAT_TYPE_ERROR); - return false; } - - return true; } void V4GetHashProtocolManager::ResetGetHashErrors() { @@ -698,7 +710,7 @@ void V4GetHashProtocolManager::OnURLFetchComplete( int response_code = source->GetResponseCode(); net::URLRequestStatus status = source->GetStatus(); V4ProtocolManagerUtil::RecordHttpResponseOrErrorCode( - kUmaV4HashResponseMetricName, status, response_code); + "SafeBrowsing.V4GetHash.Network.Result", status, response_code); std::vector<FullHashInfo> full_hash_infos; Time negative_cache_expire; @@ -726,6 +738,8 @@ void V4GetHashProtocolManager::OnURLFetchComplete( } const std::unique_ptr<FullHashCallbackInfo>& fhci = it->second; + UMA_HISTOGRAM_LONG_TIMES("SafeBrowsing.V4GetHash.Network.Time", + clock_->Now() - fhci->network_start_time); UpdateCache(fhci->prefixes_requested, full_hash_infos, negative_cache_expire); MergeResults(fhci->full_hash_to_store_and_hash_prefixes, full_hash_infos, &fhci->cached_full_hash_infos); diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h index 720e09684cb..ffd4ea89056 100644 --- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h +++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h @@ -106,7 +106,8 @@ struct FullHashCallbackInfo { std::unique_ptr<net::URLFetcher> fetcher, const FullHashToStoreAndHashPrefixesMap& full_hash_to_store_and_hash_prefixes, - const FullHashCallback& callback); + const FullHashCallback& callback, + const base::Time& network_start_time); ~FullHashCallbackInfo(); // The FullHashInfo objects retrieved from cache. These are merged with the @@ -125,6 +126,10 @@ struct FullHashCallbackInfo { // which to look for a full hash match. FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; + // Used to measure how long did it take to fetch the full hash response from + // the server. + base::Time network_start_time; + // The prefixes that were requested from the server. std::vector<HashPrefix> prefixes_requested; }; @@ -254,7 +259,7 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate, // permission API metadata for full hashes in those |full_hash_infos| that // have a full hash in |full_hashes|. void OnFullHashForApi(const ThreatMetadataForApiCallback& api_callback, - const std::unordered_set<FullHash>& full_hashes, + const std::vector<FullHash>& full_hashes, const std::vector<FullHashInfo>& full_hash_infos); // Parses a FindFullHashesResponse protocol buffer and fills the results in @@ -266,10 +271,10 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate, std::vector<FullHashInfo>* full_hash_infos, base::Time* negative_cache_expire); - // Parses the store specific |metadata| information from |match|. Returns - // true if the metadata information was parsed correctly and was consistent - // with what's expected from that corresponding store; false otherwise. - bool ParseMetadata(const ThreatMatch& match, ThreatMetadata* metadata); + // Parses the store specific |metadata| information from |match|. Logs errors + // to UMA if the metadata information was not parsed correctly or was + // inconsistent with what's expected from that corresponding store. + static void ParseMetadata(const ThreatMatch& match, ThreatMetadata* metadata); // Resets the gethash error counter and multiplier. void ResetGetHashErrors(); @@ -330,9 +335,9 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate, // The following sets represent the combination of lists that we would always // request from the server, irrespective of which list we found the hash // prefix match in. - std::unordered_set<PlatformType> platform_types_; - std::unordered_set<ThreatEntryType> threat_entry_types_; - std::unordered_set<ThreatType> threat_types_; + std::vector<PlatformType> platform_types_; + std::vector<ThreatEntryType> threat_entry_types_; + std::vector<ThreatType> threat_types_; DISALLOW_COPY_AND_ASSIGN(V4GetHashProtocolManager); }; diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc index fa37e0622f3..85a42c16c65 100644 --- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc @@ -14,8 +14,8 @@ #include "base/test/simple_test_clock.h" #include "base/time/time.h" #include "components/safe_browsing_db/safebrowsing.pb.h" -#include "components/safe_browsing_db/testing_util.h" #include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_test_util.h" #include "content/public/test/test_browser_thread_bundle.h" #include "net/base/escape.h" #include "net/base/load_flags.h" @@ -26,14 +26,6 @@ using base::Time; using base::TimeDelta; -namespace { - -const char kClient[] = "unittest"; -const char kAppVer[] = "1.0"; -const char kKeyParam[] = "test_key_param"; - -} // namespace - namespace safe_browsing { namespace { @@ -76,12 +68,13 @@ class V4GetHashProtocolManagerTest : public PlatformTest { } std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager() { - V4ProtocolConfig config; - config.client_name = kClient; - config.version = kAppVer; - config.key_param = kKeyParam; - StoresToCheck stores_to_check({GetUrlMalwareId(), GetChromeUrlApiId()}); - return V4GetHashProtocolManager::Create(NULL, stores_to_check, config); + StoresToCheck stores_to_check( + {GetUrlMalwareId(), GetChromeUrlApiId(), + ListIdentifier(CHROME_PLATFORM, URL, SOCIAL_ENGINEERING_PUBLIC), + ListIdentifier(CHROME_PLATFORM, URL, + POTENTIALLY_HARMFUL_APPLICATION)}); + return V4GetHashProtocolManager::Create(NULL, stores_to_check, + GetTestV4ProtocolConfig()); } static void SetupFetcherToReturnOKResponse( @@ -285,13 +278,18 @@ TEST_F(V4GetHashProtocolManagerTest, TEST_F(V4GetHashProtocolManagerTest, TestGetHashRequest) { FindFullHashesRequest req; ThreatInfo* info = req.mutable_threat_info(); - info->add_platform_types(GetCurrentPlatformType()); - info->add_platform_types(CHROME_PLATFORM); + for (const PlatformType& p : + std::set<PlatformType>{GetCurrentPlatformType(), CHROME_PLATFORM}) { + info->add_platform_types(p); + } info->add_threat_entry_types(URL); - info->add_threat_types(MALWARE_THREAT); - info->add_threat_types(API_ABUSE); + for (const ThreatType& tt : + std::set<ThreatType>{MALWARE_THREAT, SOCIAL_ENGINEERING_PUBLIC, + POTENTIALLY_HARMFUL_APPLICATION, API_ABUSE}) { + info->add_threat_types(tt); + } HashPrefix one = "hashone"; HashPrefix two = "hashtwo"; @@ -331,6 +329,14 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashResponse) { m->mutable_threat_entry_metadata()->add_entries(); e->set_key("permission"); e->set_value("NOTIFICATIONS"); + // Add another ThreatMatch for a list we don't track. This response should + // get dropped. + m = res.add_matches(); + m->set_threat_type(THREAT_TYPE_UNSPECIFIED); + m->set_platform_type(CHROME_PLATFORM); + m->set_threat_entry_type(URL); + m->mutable_cache_duration()->set_seconds(300); + m->mutable_threat()->set_hash(full_hash); // Serialize. std::string res_data; @@ -341,6 +347,8 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashResponse) { EXPECT_TRUE(pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire)); EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire); + // Even though the server responded with two ThreatMatch responses, one + // should have been dropped. ASSERT_EQ(1ul, full_hash_infos.size()); const FullHashInfo& fhi = full_hash_infos[0]; EXPECT_EQ(full_hash, fhi.full_hash); @@ -408,6 +416,8 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashThreatPatternType) { pm->ParseHashResponse(se_data, &full_hash_infos, &cache_expire)); EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire); + // Ensure that the threat remains valid since we found a full hash match, + // even though the metadata information could not be parsed correctly. ASSERT_EQ(1ul, full_hash_infos.size()); const FullHashInfo& fhi = full_hash_infos[0]; EXPECT_EQ(full_hash, fhi.full_hash); @@ -470,9 +480,18 @@ TEST_F(V4GetHashProtocolManagerTest, TestParseHashThreatPatternType) { invalid_res.SerializeToString(&invalid_data); std::vector<FullHashInfo> full_hash_infos; base::Time cache_expire; - EXPECT_FALSE( + EXPECT_TRUE( pm->ParseHashResponse(invalid_data, &full_hash_infos, &cache_expire)); - EXPECT_EQ(0ul, full_hash_infos.size()); + + // Ensure that the threat remains valid since we found a full hash match, + // even though the metadata information could not be parsed correctly. + ASSERT_EQ(1ul, full_hash_infos.size()); + const auto& fhi = full_hash_infos[0]; + EXPECT_EQ(full_hash, fhi.full_hash); + EXPECT_EQ( + ListIdentifier(CHROME_PLATFORM, URL, POTENTIALLY_HARMFUL_APPLICATION), + fhi.list_id); + EXPECT_EQ(ThreatPatternType::NONE, fhi.metadata.threat_pattern_type); } } @@ -484,13 +503,14 @@ TEST_F(V4GetHashProtocolManagerTest, base::Time now = base::Time::UnixEpoch(); SetTestClock(now, pm.get()); + FullHash full_hash("Not to fret."); FindFullHashesResponse res; res.mutable_negative_cache_duration()->set_seconds(600); ThreatMatch* m = res.add_matches(); m->set_threat_type(API_ABUSE); m->set_platform_type(CHROME_PLATFORM); m->set_threat_entry_type(URL); - m->mutable_threat()->set_hash(FullHash("Not to fret.")); + m->mutable_threat()->set_hash(full_hash); ThreatEntryMetadata::MetadataEntry* e = m->mutable_threat_entry_metadata()->add_entries(); e->set_key("notpermission"); @@ -502,11 +522,14 @@ TEST_F(V4GetHashProtocolManagerTest, std::vector<FullHashInfo> full_hash_infos; base::Time cache_expire; - EXPECT_FALSE( - pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire)); + EXPECT_TRUE(pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire)); EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire); - EXPECT_EQ(0ul, full_hash_infos.size()); + ASSERT_EQ(1ul, full_hash_infos.size()); + const auto& fhi = full_hash_infos[0]; + EXPECT_EQ(full_hash, fhi.full_hash); + EXPECT_EQ(GetChromeUrlApiId(), fhi.list_id); + EXPECT_TRUE(fhi.metadata.api_permissions.empty()); } TEST_F(V4GetHashProtocolManagerTest, diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager.cc b/chromium/components/safe_browsing_db/v4_local_database_manager.cc index 1a81f7fcda2..ffef84ef025 100644 --- a/chromium/components/safe_browsing_db/v4_local_database_manager.cc +++ b/chromium/components/safe_browsing_db/v4_local_database_manager.cc @@ -13,10 +13,14 @@ #include "base/callback.h" #include "base/memory/ptr_util.h" #include "base/memory/ref_counted.h" +#include "base/metrics/histogram_macros.h" #include "components/safe_browsing_db/v4_feature_list.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "content/public/browser/browser_thread.h" +#include "crypto/sha2.h" using content::BrowserThread; +using base::TimeTicks; namespace safe_browsing { @@ -26,12 +30,50 @@ const ThreatSeverity kLeastSeverity = std::numeric_limits<ThreatSeverity>::max(); ListInfos GetListInfos() { - return ListInfos( - {ListInfo(true, "UrlMalware.store", GetUrlMalwareId(), - SB_THREAT_TYPE_URL_MALWARE), - ListInfo(true, "UrlSoceng.store", GetUrlSocEngId(), - SB_THREAT_TYPE_URL_PHISHING), - ListInfo(false, "", GetChromeUrlApiId(), SB_THREAT_TYPE_API_ABUSE)}); + // NOTE(vakh): When adding a store here, add the corresponding store-specific + // histograms also. + // NOTE(vakh): Delete file "AnyIpMalware.store". It has been renamed to + // "IpMalware.store". If it exists, it should be 75 bytes long. + // The first argument to ListInfo specifies whether to sync hash prefixes for + // that list. This can be false for two reasons: + // - The server doesn't support that list yet. Once the server adds support + // for it, it can be changed to true. + // - The list doesn't have hash prefixes to match. All requests lead to full + // hash checks. For instance: GetChromeUrlApiId() + +#if defined(GOOGLE_CHROME_BUILD) + const bool kSyncOnlyOnChromeBuilds = true; +#else + const bool kSyncOnlyOnChromeBuilds = false; +#endif + const bool kSyncAlways = true; + const bool kSyncNever = false; + return ListInfos({ + ListInfo(kSyncOnlyOnChromeBuilds, "CertCsdDownloadWhitelist.store", + GetCertCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED), + ListInfo(kSyncOnlyOnChromeBuilds, "ChromeFilenameClientIncident.store", + GetChromeFilenameClientIncidentId(), SB_THREAT_TYPE_UNUSED), + ListInfo(kSyncAlways, "IpMalware.store", GetIpMalwareId(), + SB_THREAT_TYPE_UNUSED), + ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdDownloadWhitelist.store", + GetUrlCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED), + ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdWhitelist.store", + GetUrlCsdWhitelistId(), SB_THREAT_TYPE_UNUSED), + ListInfo(kSyncAlways, "UrlSoceng.store", GetUrlSocEngId(), + SB_THREAT_TYPE_URL_PHISHING), + ListInfo(kSyncAlways, "UrlMalware.store", GetUrlMalwareId(), + SB_THREAT_TYPE_URL_MALWARE), + ListInfo(kSyncAlways, "UrlUws.store", GetUrlUwsId(), + SB_THREAT_TYPE_URL_UNWANTED), + ListInfo(kSyncAlways, "UrlMalBin.store", GetUrlMalBinId(), + SB_THREAT_TYPE_BINARY_MALWARE_URL), + ListInfo(kSyncAlways, "ChromeExtMalware.store", GetChromeExtMalwareId(), + SB_THREAT_TYPE_EXTENSION), + ListInfo(kSyncOnlyOnChromeBuilds, "ChromeUrlClientIncident.store", + GetChromeUrlClientIncidentId(), + SB_THREAT_TYPE_BLACKLISTED_RESOURCE), + ListInfo(kSyncNever, "", GetChromeUrlApiId(), SB_THREAT_TYPE_API_ABUSE), + }); } // Returns the severity information about a given SafeBrowsing list. The lowest @@ -40,11 +82,15 @@ ThreatSeverity GetThreatSeverity(const ListIdentifier& list_id) { switch (list_id.threat_type()) { case MALWARE_THREAT: case SOCIAL_ENGINEERING_PUBLIC: + case MALICIOUS_BINARY: return 0; - case API_ABUSE: + case UNWANTED_SOFTWARE: return 1; + case API_ABUSE: + return 2; default: - NOTREACHED() << "Unexpected ThreatType encountered in GetThreatSeverity"; + NOTREACHED() << "Unexpected ThreatType encountered: " + << list_id.threat_type(); return kLeastSeverity; } } @@ -55,13 +101,27 @@ V4LocalDatabaseManager::PendingCheck::PendingCheck( Client* client, ClientCallbackType client_callback_type, const StoresToCheck& stores_to_check, - const GURL& url) + const std::vector<GURL>& urls) : client(client), client_callback_type(client_callback_type), result_threat_type(SB_THREAT_TYPE_SAFE), stores_to_check(stores_to_check), - url(url) { - DCHECK_GT(ClientCallbackType::CHECK_MAX, client_callback_type); + urls(urls) { + for (const auto& url : urls) { + V4ProtocolManagerUtil::UrlToFullHashes(url, &full_hashes); + } +} + +V4LocalDatabaseManager::PendingCheck::PendingCheck( + Client* client, + ClientCallbackType client_callback_type, + const StoresToCheck& stores_to_check, + const std::set<FullHash>& full_hashes_set) + : client(client), + client_callback_type(client_callback_type), + result_threat_type(SB_THREAT_TYPE_SAFE), + stores_to_check(stores_to_check) { + full_hashes.assign(full_hashes_set.begin(), full_hashes_set.end()); } V4LocalDatabaseManager::PendingCheck::~PendingCheck() {} @@ -100,9 +160,11 @@ void V4LocalDatabaseManager::CancelCheck(Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); DCHECK(enabled_); - auto it = pending_clients_.find(client); - if (it != pending_clients_.end()) { - pending_clients_.erase(it); + auto pending_it = std::find_if( + std::begin(pending_checks_), std::end(pending_checks_), + [client](const PendingCheck* check) { return check->client == client; }); + if (pending_it != pending_checks_.end()) { + pending_checks_.erase(pending_it); } auto queued_it = @@ -139,73 +201,137 @@ bool V4LocalDatabaseManager::CheckBrowseUrl(const GURL& url, Client* client) { std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( client, ClientCallbackType::CHECK_BROWSE_URL, - StoresToCheck({GetUrlMalwareId(), GetUrlSocEngId()}), url); - if (!v4_database_) { - queued_checks_.push_back(std::move(check)); - return false; - } + StoresToCheck({GetUrlMalwareId(), GetUrlSocEngId(), GetUrlUwsId()}), + std::vector<GURL>(1, url)); - FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; - if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) { - return true; - } - - PerformFullHashCheck(std::move(check), full_hash_to_store_and_hash_prefixes); - return false; + return HandleCheck(std::move(check)); } bool V4LocalDatabaseManager::CheckDownloadUrl( const std::vector<GURL>& url_chain, Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); - // TODO(vakh): Implement this skeleton. - return true; + + if (!enabled_ || url_chain.empty()) { + return true; + } + + std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( + client, ClientCallbackType::CHECK_DOWNLOAD_URLS, + StoresToCheck({GetUrlMalBinId()}), url_chain); + + return HandleCheck(std::move(check)); } bool V4LocalDatabaseManager::CheckExtensionIDs( - const std::set<std::string>& extension_ids, + const std::set<FullHash>& extension_ids, Client* client) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + + if (!enabled_) { + return true; + } + + std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( + client, ClientCallbackType::CHECK_EXTENSION_IDS, + StoresToCheck({GetChromeExtMalwareId()}), extension_ids); + + return HandleCheck(std::move(check)); } bool V4LocalDatabaseManager::CheckResourceUrl(const GURL& url, Client* client) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + + StoresToCheck stores_to_check({GetChromeUrlClientIncidentId()}); + + if (!CanCheckUrl(url) || !AreStoresAvailableNow(stores_to_check)) { + // Fail open: Mark resource as safe immediately. + // TODO(nparker): This should queue the request if the DB isn't yet + // loaded, and later decide if this store is available. + // Currently this is the only store that requires full-hash-checks + // AND isn't supported on Chromium, so it's unique. + return true; + } + + std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( + client, ClientCallbackType::CHECK_RESOURCE_URL, stores_to_check, + std::vector<GURL>(1, url)); + + return HandleCheck(std::move(check)); } bool V4LocalDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + + StoresToCheck stores_to_check({GetUrlCsdWhitelistId()}); + if (!AreStoresAvailableNow(stores_to_check)) { + // Fail open: Whitelist everything. Otherwise we may run the + // CSD phishing/malware detector on popular domains and generate + // undue load on the client and server. This has the effect of disabling + // CSD phishing/malware detection until the store is first synced. + return true; + } + + return HandleUrlSynchronously(url, stores_to_check); } bool V4LocalDatabaseManager::MatchDownloadWhitelistString( const std::string& str) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + + StoresToCheck stores_to_check({GetCertCsdDownloadWhitelistId()}); + if (!AreStoresAvailableNow(stores_to_check)) { + // Fail close: Whitelist nothing. This may generate download-protection + // pings for whitelisted binaries, but that's fine. + return false; + } + + return HandleHashSynchronously(str, stores_to_check); } bool V4LocalDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + + StoresToCheck stores_to_check({GetUrlCsdDownloadWhitelistId()}); + if (!AreStoresAvailableNow(stores_to_check)) { + // Fail close: Whitelist nothing. This may generate download-protection + // pings for whitelisted domains, but that's fine. + return false; + } + + return HandleUrlSynchronously(url, stores_to_check); } bool V4LocalDatabaseManager::MatchMalwareIP(const std::string& ip_address) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return false; + if (!enabled_ || !v4_database_) { + return false; + } + + FullHash hashed_encoded_ip; + if (!V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash(ip_address, + &hashed_encoded_ip)) { + return false; + } + + return HandleHashSynchronously(hashed_encoded_ip, + StoresToCheck({GetIpMalwareId()})); } bool V4LocalDatabaseManager::MatchModuleWhitelistString( const std::string& str) { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + + StoresToCheck stores_to_check({GetChromeFilenameClientIncidentId()}); + if (!AreStoresAvailableNow(stores_to_check)) { + // Fail open: Whitelist everything. This has the effect of marking + // all DLLs as safe until the DB is synced and loaded. + return true; + } + + // str is the module's filename. Convert to hash. + FullHash hash = crypto::SHA256HashString(str); + return HandleHashSynchronously(hash, stores_to_check); } ThreatSource V4LocalDatabaseManager::GetThreatSource() const { @@ -213,9 +339,8 @@ ThreatSource V4LocalDatabaseManager::GetThreatSource() const { } bool V4LocalDatabaseManager::IsCsdWhitelistKillSwitchOn() { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + return false; } bool V4LocalDatabaseManager::IsDownloadProtectionEnabled() const { @@ -225,9 +350,8 @@ bool V4LocalDatabaseManager::IsDownloadProtectionEnabled() const { } bool V4LocalDatabaseManager::IsMalwareKillSwitchOn() { - // TODO(vakh): Implement this skeleton. DCHECK_CURRENTLY_ON(BrowserThread::IO); - return true; + return false; } bool V4LocalDatabaseManager::IsSupported() const { @@ -253,7 +377,7 @@ void V4LocalDatabaseManager::StopOnIOThread(bool shutdown) { enabled_ = false; - pending_clients_.clear(); + pending_checks_.clear(); RespondSafeToQueuedChecks(); @@ -275,7 +399,7 @@ void V4LocalDatabaseManager::StopOnIOThread(bool shutdown) { // End: SafeBrowsingDatabaseManager implementation // -void V4LocalDatabaseManager::DatabaseReady( +void V4LocalDatabaseManager::DatabaseReadyForChecks( std::unique_ptr<V4Database> v4_database) { DCHECK_CURRENTLY_ON(BrowserThread::IO); @@ -284,9 +408,15 @@ void V4LocalDatabaseManager::DatabaseReady( if (enabled_) { v4_database_ = std::move(v4_database); - // The database is in place. Start fetching updates now. - v4_update_protocol_manager_->ScheduleNextUpdate( - v4_database_->GetStoreStateMap()); + v4_database_->RecordFileSizeHistograms(); + + // The consistency of the stores read from the disk needs to verified. Post + // that task on the task runner. It calls |DatabaseReadyForUpdates| + // callback with the stores to reset, if any, and then we can schedule the + // database updates. + v4_database_->VerifyChecksum( + base::Bind(&V4LocalDatabaseManager::DatabaseReadyForUpdates, + weak_factory_.GetWeakPtr())); ProcessQueuedChecks(); } else { @@ -295,8 +425,20 @@ void V4LocalDatabaseManager::DatabaseReady( } } +void V4LocalDatabaseManager::DatabaseReadyForUpdates( + const std::vector<ListIdentifier>& stores_to_reset) { + if (enabled_) { + v4_database_->ResetStores(stores_to_reset); + + // The database is ready to process updates. Schedule them now. + v4_update_protocol_manager_->ScheduleNextUpdate( + v4_database_->GetStoreStateMap()); + } +} + void V4LocalDatabaseManager::DatabaseUpdated() { if (enabled_) { + v4_database_->RecordFileSizeHistograms(); v4_update_protocol_manager_->ScheduleNextUpdate( v4_database_->GetStoreStateMap()); } @@ -309,15 +451,18 @@ bool V4LocalDatabaseManager::GetPrefixMatches( DCHECK(enabled_); DCHECK(v4_database_); - DCHECK_GT(ClientCallbackType::CHECK_MAX, check->client_callback_type); - - if (check->client_callback_type == ClientCallbackType::CHECK_BROWSE_URL) { - std::unordered_set<FullHash> full_hashes; - V4ProtocolManagerUtil::UrlToFullHashes(check->url, &full_hashes); - StoreAndHashPrefixes matched_store_and_hash_prefixes; - for (const auto& full_hash : full_hashes) { - matched_store_and_hash_prefixes.clear(); + const base::TimeTicks before = TimeTicks::Now(); + if (check->client_callback_type == ClientCallbackType::CHECK_BROWSE_URL || + check->client_callback_type == ClientCallbackType::CHECK_DOWNLOAD_URLS || + check->client_callback_type == ClientCallbackType::CHECK_RESOURCE_URL || + check->client_callback_type == ClientCallbackType::CHECK_EXTENSION_IDS || + check->client_callback_type == ClientCallbackType::CHECK_OTHER) { + DCHECK(!check->full_hashes.empty()); + + full_hash_to_store_and_hash_prefixes->clear(); + for (const auto& full_hash : check->full_hashes) { + StoreAndHashPrefixes matched_store_and_hash_prefixes; v4_database_->GetStoresMatchingFullHash(full_hash, check->stores_to_check, &matched_store_and_hash_prefixes); if (!matched_store_and_hash_prefixes.empty()) { @@ -325,22 +470,31 @@ bool V4LocalDatabaseManager::GetPrefixMatches( matched_store_and_hash_prefixes; } } - - // No hash prefixes found in the local database so that resource must be - // safe. - return !full_hash_to_store_and_hash_prefixes->empty(); + } else { + NOTREACHED() << "Unexpected client_callback_type encountered."; } - NOTREACHED() << "Unexpected client_callback_type encountered"; - return false; + // TODO(vakh): Only log SafeBrowsing.V4GetPrefixMatches.Time once PVer3 code + // is removed. + // NOTE(vakh): This doesn't distinguish which stores it's searching through. + // However, the vast majority of the entries in this histogram will be from + // searching the three CHECK_BROWSE_URL stores. + base::TimeDelta diff = TimeTicks::Now() - before; + UMA_HISTOGRAM_TIMES("SB2.FilterCheck", diff); + UMA_HISTOGRAM_CUSTOM_TIMES("SafeBrowsing.V4GetPrefixMatches.Time", diff, + base::TimeDelta::FromMicroseconds(20), + base::TimeDelta::FromSeconds(1), 50); + return !full_hash_to_store_and_hash_prefixes->empty(); } void V4LocalDatabaseManager::GetSeverestThreatTypeAndMetadata( SBThreatType* result_threat_type, ThreatMetadata* metadata, + FullHash* matching_full_hash, const std::vector<FullHashInfo>& full_hash_infos) { DCHECK(result_threat_type); DCHECK(metadata); + DCHECK(matching_full_hash); ThreatSeverity most_severe_yet = kLeastSeverity; for (const FullHashInfo& fhi : full_hash_infos) { @@ -349,6 +503,7 @@ void V4LocalDatabaseManager::GetSeverestThreatTypeAndMetadata( most_severe_yet = severity; *result_threat_type = GetSBThreatTypeForList(fhi.list_id); *metadata = fhi.metadata; + *matching_full_hash = fhi.full_hash; } } } @@ -369,31 +524,84 @@ SBThreatType V4LocalDatabaseManager::GetSBThreatTypeForList( [&list_id](ListInfo const& li) { return li.list_id() == list_id; }); DCHECK(list_infos_.end() != it); DCHECK_NE(SB_THREAT_TYPE_SAFE, it->sb_threat_type()); + DCHECK_NE(SB_THREAT_TYPE_UNUSED, it->sb_threat_type()); return it->sb_threat_type(); } +bool V4LocalDatabaseManager::HandleCheck(std::unique_ptr<PendingCheck> check) { + if (!v4_database_) { + queued_checks_.push_back(std::move(check)); + return false; + } + + FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; + if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) { + return true; + } + + // Add check to pending_checks_ before scheduling PerformFullHashCheck so that + // even if the client calls CancelCheck before PerformFullHashCheck gets + // called, the check can be found in pending_checks_. + pending_checks_.insert(check.get()); + + // Post on the IO thread to enforce async behavior. + BrowserThread::PostTask( + BrowserThread::IO, FROM_HERE, + base::Bind(&V4LocalDatabaseManager::PerformFullHashCheck, this, + base::Passed(std::move(check)), + full_hash_to_store_and_hash_prefixes)); + + return false; +} + +bool V4LocalDatabaseManager::HandleHashSynchronously( + const FullHash& hash, + const StoresToCheck& stores_to_check) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + + std::set<FullHash> hashes{hash}; + std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( + nullptr, ClientCallbackType::CHECK_OTHER, stores_to_check, hashes); + + FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; + return GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes); +} + +bool V4LocalDatabaseManager::HandleUrlSynchronously( + const GURL& url, + const StoresToCheck& stores_to_check) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + + std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( + nullptr, ClientCallbackType::CHECK_OTHER, stores_to_check, + std::vector<GURL>(1, url)); + + FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; + return GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes); +} + void V4LocalDatabaseManager::OnFullHashResponse( - std::unique_ptr<PendingCheck> pending_check, + std::unique_ptr<PendingCheck> check, const std::vector<FullHashInfo>& full_hash_infos) { DCHECK_CURRENTLY_ON(BrowserThread::IO); if (!enabled_) { - DCHECK(pending_clients_.empty()); + DCHECK(pending_checks_.empty()); return; } - auto it = pending_clients_.find(pending_check->client); - if (it == pending_clients_.end()) { + const auto it = pending_checks_.find(check.get()); + if (it == pending_checks_.end()) { // The check has since been cancelled. return; } // Find out the most severe threat, if any, to report to the client. - GetSeverestThreatTypeAndMetadata(&pending_check->result_threat_type, - &pending_check->url_metadata, - full_hash_infos); - pending_clients_.erase(it); - RespondToClient(std::move(pending_check)); + GetSeverestThreatTypeAndMetadata(&check->result_threat_type, + &check->url_metadata, + &check->matching_full_hash, full_hash_infos); + pending_checks_.erase(it); + RespondToClient(std::move(check)); } void V4LocalDatabaseManager::PerformFullHashCheck( @@ -405,8 +613,6 @@ void V4LocalDatabaseManager::PerformFullHashCheck( DCHECK(enabled_); DCHECK(!full_hash_to_store_and_hash_prefixes.empty()); - pending_clients_.insert(check->client); - v4_get_hash_protocol_manager_->GetFullHashes( full_hash_to_store_and_hash_prefixes, base::Bind(&V4LocalDatabaseManager::OnFullHashResponse, @@ -415,31 +621,66 @@ void V4LocalDatabaseManager::PerformFullHashCheck( void V4LocalDatabaseManager::ProcessQueuedChecks() { DCHECK_CURRENTLY_ON(BrowserThread::IO); - for (auto& it : queued_checks_) { + + // Steal the queue to protect against reentrant CancelCheck() calls. + QueuedChecks checks; + checks.swap(queued_checks_); + + for (auto& it : checks) { FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; if (!GetPrefixMatches(it, &full_hash_to_store_and_hash_prefixes)) { RespondToClient(std::move(it)); } else { + pending_checks_.insert(it.get()); PerformFullHashCheck(std::move(it), full_hash_to_store_and_hash_prefixes); } } - queued_checks_.clear(); } void V4LocalDatabaseManager::RespondSafeToQueuedChecks() { DCHECK_CURRENTLY_ON(BrowserThread::IO); - for (std::unique_ptr<PendingCheck>& it : queued_checks_) { + + // Steal the queue to protect against reentrant CancelCheck() calls. + QueuedChecks checks; + checks.swap(queued_checks_); + + for (std::unique_ptr<PendingCheck>& it : checks) { RespondToClient(std::move(it)); } - queued_checks_.clear(); } void V4LocalDatabaseManager::RespondToClient( - std::unique_ptr<PendingCheck> pending_check) { - DCHECK(pending_check.get()); - DCHECK_EQ(ClientCallbackType::CHECK_BROWSE_URL, - pending_check->client_callback_type); - // TODO(vakh): Implement this skeleton. + std::unique_ptr<PendingCheck> check) { + DCHECK(check.get()); + + switch (check->client_callback_type) { + case ClientCallbackType::CHECK_BROWSE_URL: + DCHECK_EQ(1u, check->urls.size()); + check->client->OnCheckBrowseUrlResult( + check->urls[0], check->result_threat_type, check->url_metadata); + break; + + case ClientCallbackType::CHECK_DOWNLOAD_URLS: + check->client->OnCheckDownloadUrlResult(check->urls, + check->result_threat_type); + break; + + case ClientCallbackType::CHECK_RESOURCE_URL: + DCHECK_EQ(1u, check->urls.size()); + check->client->OnCheckResourceUrlResult( + check->urls[0], check->result_threat_type, check->matching_full_hash); + break; + + case ClientCallbackType::CHECK_EXTENSION_IDS: { + const std::set<FullHash> extension_ids(check->full_hashes.begin(), + check->full_hashes.end()); + check->client->OnCheckExtensionsResult(extension_ids); + break; + } + + case ClientCallbackType::CHECK_OTHER: + NOTREACHED() << "Unexpected client_callback_type encountered"; + } } void V4LocalDatabaseManager::SetupDatabase() { @@ -458,8 +699,9 @@ void V4LocalDatabaseManager::SetupDatabase() { // Do not create the database on the IO thread since this may be an expensive // operation. Instead, do that on the task_runner and when the new database // has been created, swap it out on the IO thread. - NewDatabaseReadyCallback db_ready_callback = base::Bind( - &V4LocalDatabaseManager::DatabaseReady, weak_factory_.GetWeakPtr()); + NewDatabaseReadyCallback db_ready_callback = + base::Bind(&V4LocalDatabaseManager::DatabaseReadyForChecks, + weak_factory_.GetWeakPtr()); V4Database::Create(task_runner_, base_path_, list_infos_, db_ready_callback); } @@ -481,4 +723,10 @@ void V4LocalDatabaseManager::UpdateRequestCompleted( db_updated_callback_); } +bool V4LocalDatabaseManager::AreStoresAvailableNow( + const StoresToCheck& stores_to_check) const { + return enabled_ && v4_database_ && + v4_database_->AreStoresAvailable(stores_to_check); +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager.h b/chromium/components/safe_browsing_db/v4_local_database_manager.h index 0449b8c5ac5..41598243228 100644 --- a/chromium/components/safe_browsing_db/v4_local_database_manager.h +++ b/chromium/components/safe_browsing_db/v4_local_database_manager.h @@ -9,6 +9,7 @@ // and database that holds the downloaded updates. #include <memory> +#include <unordered_set> #include "base/memory/weak_ptr.h" #include "components/safe_browsing_db/database_manager.h" @@ -45,7 +46,11 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { bool CheckBrowseUrl(const GURL& url, Client* client) override; bool CheckDownloadUrl(const std::vector<GURL>& url_chain, Client* client) override; - bool CheckExtensionIDs(const std::set<std::string>& extension_ids, + // TODO(vakh): |CheckExtensionIDs| in the base class accepts a set of + // std::strings but the overriding method in this class accepts a set of + // FullHash objects. Since FullHash is currently std::string, it compiles, + // but this difference should be eliminated. + bool CheckExtensionIDs(const std::set<FullHash>& extension_ids, Client* client) override; bool CheckResourceUrl(const GURL& url, Client* client) override; bool MatchCsdWhitelistUrl(const GURL& url) override; @@ -71,24 +76,48 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { // Must be initialized by calling StartOnIOThread() before using. V4LocalDatabaseManager(const base::FilePath& base_path); + ~V4LocalDatabaseManager() override; + + void SetTaskRunnerForTest( + const scoped_refptr<base::SequencedTaskRunner>& task_runner) { + task_runner_ = task_runner; + } + enum class ClientCallbackType { // This represents the case when we're trying to determine if a URL is // unsafe from the following perspectives: Malware, Phishing, UwS. CHECK_BROWSE_URL = 0, - // This should always be the last value. - CHECK_MAX + // This represents the case when we're trying to determine if any of the + // URLs in a vector of URLs is unsafe for downloading binaries. + CHECK_DOWNLOAD_URLS = 1, + + // This represents the case when we're trying to determine if a URL is an + // unsafe resource. + CHECK_RESOURCE_URL = 2, + + // This represents the case when we're trying to determine if a Chrome + // extension is a unsafe. + CHECK_EXTENSION_IDS = 3, + + // This represents the other cases when a check is being performed + // synchronously so a client callback isn't required. For instance, when + // trying to determing if an IP address is unsafe due to hosting Malware. + CHECK_OTHER = 4, }; // The information we need to process a URL safety reputation request and // respond to the SafeBrowsing client that asked for it. - // TODO(vakh): In its current form, it only includes information for - // |CheckBrowseUrl| method. Extend it to serve other methods on |client|. struct PendingCheck { PendingCheck(Client* client, ClientCallbackType client_callback_type, const StoresToCheck& stores_to_check, - const GURL& url); + const std::vector<GURL>& urls); + + PendingCheck(Client* client, + ClientCallbackType client_callback_type, + const StoresToCheck& stores_to_check, + const std::set<FullHash>& full_hashes); ~PendingCheck(); @@ -97,20 +126,34 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { // Determines which funtion from the |client| needs to be called once we // know whether the URL in |url| is safe or unsafe. - ClientCallbackType client_callback_type; + const ClientCallbackType client_callback_type; // The threat verdict for the URL being checked. SBThreatType result_threat_type; + // When the check was sent to the SafeBrowsing service. Used to record the + // time it takes to get the uncached full hashes from the service (or a + // cached full hash response). + base::TimeTicks full_hash_check_start; + // The SafeBrowsing lists to check hash prefixes in. - StoresToCheck stores_to_check; + const StoresToCheck stores_to_check; + + // The URLs that are being checked for being unsafe. The size of exactly + // one of |full_hashes| and |urls| should be greater than 0. + const std::vector<GURL> urls; - // The URL that is being checked for being unsafe. - GURL url; + // The full hashes that are being checked for being safe. The size of + // exactly one of |full_hashes| and |urls| should be greater than 0. + std::vector<FullHash> full_hashes; // The metadata associated with the full hash of the severest match found // for that URL. ThreatMetadata url_metadata; + + // The full hash that matched for a blacklisted resource URL. Used only for + // |CheckResourceUrl| case. + FullHash matching_full_hash; }; typedef std::vector<std::unique_ptr<PendingCheck>> QueuedChecks; @@ -121,42 +164,61 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { private: friend class V4LocalDatabaseManagerTest; - void SetTaskRunnerForTest( - const scoped_refptr<base::SequencedTaskRunner>& task_runner) { - task_runner_ = task_runner; - } FRIEND_TEST_ALL_PREFIXES(V4LocalDatabaseManagerTest, TestGetSeverestThreatTypeAndMetadata); - // The set of clients awaiting a full hash response. It is used for tracking - // which clients have cancelled their outstanding request. - typedef std::unordered_set<Client*> PendingClients; - - ~V4LocalDatabaseManager() override; + // The checks awaiting a full hash response from SafeBrowsing service. + typedef std::unordered_set<const PendingCheck*> PendingChecks; // Called when all the stores managed by the database have been read from - // disk after startup and the database is ready for use. - void DatabaseReady(std::unique_ptr<V4Database> v4_database); + // disk after startup and the database is ready for checking resource + // reputation. + void DatabaseReadyForChecks(std::unique_ptr<V4Database> v4_database); + + // Called when all the stores managed by the database have been verified for + // checksum correctness after startup and the database is ready for applying + // updates. + void DatabaseReadyForUpdates( + const std::vector<ListIdentifier>& stores_to_reset); // Called when the database has been updated and schedules the next update. void DatabaseUpdated(); - // Return the prefixes and the store they matched in, for a given URL. Returns - // true if a hash prefix match is found; false otherwise. + // Identifies the prefixes and the store they matched in, for a given |check|. + // Returns true if one or more hash prefix matches are found; false otherwise. bool GetPrefixMatches( const std::unique_ptr<PendingCheck>& check, FullHashToStoreAndHashPrefixesMap* full_hash_to_store_and_hash_prefixes); - // Finds the most severe |SBThreatType| and the corresponding |metadata| from - // |full_hash_infos|. + // Finds the most severe |SBThreatType| and the corresponding |metadata|, and + // |matching_full_hash| from |full_hash_infos|. void GetSeverestThreatTypeAndMetadata( SBThreatType* result_threat_type, ThreatMetadata* metadata, + FullHash* matching_full_hash, const std::vector<FullHashInfo>& full_hash_infos); // Returns the SBThreatType for a given ListIdentifier. SBThreatType GetSBThreatTypeForList(const ListIdentifier& list_id); + // Queues the check for async response if the database isn't ready yet. + // If the database is ready, checks the database for prefix matches and + // returns true immediately if there's no match. If a match is found, it + // schedules a task to perform full hash check and returns false. + bool HandleCheck(std::unique_ptr<PendingCheck> check); + + // Checks |stores_to_check| in database synchronously for hash prefixes + // matching |hash|. Returns true if there's a match; false otherwise. This is + // used for lists that have full hash information in the database. + bool HandleHashSynchronously(const FullHash& hash, + const StoresToCheck& stores_to_check); + + // Checks |stores_to_check| in database synchronously for hash prefixes + // matching the full hashes for |url|. See |HandleHashSynchronously| for + // details. + bool HandleUrlSynchronously(const GURL& url, + const StoresToCheck& stores_to_check); + // Called when the |v4_get_hash_protocol_manager_| has the full hash response // available for the URL that we requested. It determines the severest // threat type and responds to the |client| with that information. @@ -164,9 +226,9 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { const std::vector<FullHashInfo>& full_hash_infos); // Performs the full hash checking of the URL in |check|. - void PerformFullHashCheck(std::unique_ptr<PendingCheck> check, - const FullHashToStoreAndHashPrefixesMap& - full_hash_to_store_and_hash_prefixes); + virtual void PerformFullHashCheck(std::unique_ptr<PendingCheck> check, + const FullHashToStoreAndHashPrefixesMap& + full_hash_to_store_and_hash_prefixes); // When the database is ready to use, process the checks that were queued // while the database was loading from disk. @@ -194,6 +256,10 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { void UpdateRequestCompleted( std::unique_ptr<ParsedServerResponse> parsed_server_response); + // Return true if we're enabled and have loaded real data for all of + // these stores. + bool AreStoresAvailableNow(const StoresToCheck& stores_to_check) const; + // The base directory under which to create the files that contain hashes. const base::FilePath base_path_; @@ -210,9 +276,8 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { // name of the file on disk that would contain the prefixes, if applicable. ListInfos list_infos_; - // The set of clients that are waiting for a full hash response from the - // SafeBrowsing service. - PendingClients pending_clients_; + // The checks awaiting for a full hash response from the SafeBrowsing service. + PendingChecks pending_checks_; // The checks that need to be scheduled when the database becomes ready for // use. diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc index 6574a9515d0..85df2f85adb 100644 --- a/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc @@ -7,47 +7,202 @@ #include "base/memory/ref_counted.h" #include "base/run_loop.h" #include "base/test/test_simple_task_runner.h" +#include "base/threading/thread_task_runner_handle.h" #include "components/safe_browsing_db/v4_database.h" #include "components/safe_browsing_db/v4_local_database_manager.h" +#include "components/safe_browsing_db/v4_test_util.h" #include "content/public/test/test_browser_thread_bundle.h" +#include "crypto/sha2.h" #include "net/url_request/test_url_fetcher_factory.h" #include "testing/platform_test.h" namespace safe_browsing { +namespace { + +// Utility function for populating hashes. +FullHash HashForUrl(const GURL& url) { + std::vector<FullHash> full_hashes; + V4ProtocolManagerUtil::UrlToFullHashes(url, &full_hashes); + // ASSERT_GE(full_hashes.size(), 1u); + return full_hashes[0]; +} + +// Always returns misses from GetFullHashes(). +class FakeGetHashProtocolManager : public V4GetHashProtocolManager { + public: + FakeGetHashProtocolManager( + net::URLRequestContextGetter* request_context_getter, + const StoresToCheck& stores_to_check, + const V4ProtocolConfig& config) + : V4GetHashProtocolManager(request_context_getter, + stores_to_check, + config) {} + + void GetFullHashes(const FullHashToStoreAndHashPrefixesMap&, + FullHashCallback callback) override { + std::vector<FullHashInfo> full_hash_infos; + + // Async, since the real manager might use a fetcher. + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::Bind(callback, full_hash_infos)); + } +}; + +class FakeGetHashProtocolManagerFactory + : public V4GetHashProtocolManagerFactory { + public: + std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager( + net::URLRequestContextGetter* request_context_getter, + const StoresToCheck& stores_to_check, + const V4ProtocolConfig& config) override { + return base::MakeUnique<FakeGetHashProtocolManager>( + request_context_getter, stores_to_check, config); + } +}; + +// Use FakeGetHashProtocolManagerFactory in scope, then reset. +class ScopedFakeGetHashProtocolManagerFactory { + public: + ScopedFakeGetHashProtocolManagerFactory() { + V4GetHashProtocolManager::RegisterFactory( + base::MakeUnique<FakeGetHashProtocolManagerFactory>()); + } + ~ScopedFakeGetHashProtocolManagerFactory() { + V4GetHashProtocolManager::RegisterFactory(nullptr); + } +}; + +} // namespace + class FakeV4Database : public V4Database { public: - FakeV4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner, - std::unique_ptr<StoreMap> store_map, - const StoreAndHashPrefixes& store_and_hash_prefixes) - : V4Database(db_task_runner, std::move(store_map)), - store_and_hash_prefixes_(store_and_hash_prefixes) {} + static void Create( + const scoped_refptr<base::SequencedTaskRunner>& db_task_runner, + std::unique_ptr<StoreMap> store_map, + const StoreAndHashPrefixes& store_and_hash_prefixes, + NewDatabaseReadyCallback new_db_callback, + bool stores_available) { + // Mimics V4Database::Create + const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner = + base::MessageLoop::current()->task_runner(); + db_task_runner->PostTask( + FROM_HERE, + base::Bind(&FakeV4Database::CreateOnTaskRunner, db_task_runner, + base::Passed(&store_map), store_and_hash_prefixes, + callback_task_runner, new_db_callback, stores_available)); + } + // V4Database implementation void GetStoresMatchingFullHash( const FullHash& full_hash, const StoresToCheck& stores_to_check, StoreAndHashPrefixes* store_and_hash_prefixes) override { - *store_and_hash_prefixes = store_and_hash_prefixes_; + store_and_hash_prefixes->clear(); + for (const StoreAndHashPrefix& stored_sahp : store_and_hash_prefixes_) { + const PrefixSize& prefix_size = stored_sahp.hash_prefix.size(); + if (!full_hash.compare(0, prefix_size, stored_sahp.hash_prefix)) { + store_and_hash_prefixes->push_back(stored_sahp); + } + } + } + + bool AreStoresAvailable(const StoresToCheck& stores_to_check) const override { + return stores_available_; } private: - const StoreAndHashPrefixes& store_and_hash_prefixes_; + static void CreateOnTaskRunner( + const scoped_refptr<base::SequencedTaskRunner>& db_task_runner, + std::unique_ptr<StoreMap> store_map, + const StoreAndHashPrefixes& store_and_hash_prefixes, + const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, + NewDatabaseReadyCallback new_db_callback, + bool stores_available) { + // Mimics the semantics of V4Database::CreateOnTaskRunner + std::unique_ptr<FakeV4Database> fake_v4_database( + new FakeV4Database(db_task_runner, std::move(store_map), + store_and_hash_prefixes, stores_available)); + callback_task_runner->PostTask( + FROM_HERE, + base::Bind(new_db_callback, base::Passed(&fake_v4_database))); + } + + FakeV4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner, + std::unique_ptr<StoreMap> store_map, + const StoreAndHashPrefixes& store_and_hash_prefixes, + bool stores_available) + : V4Database(db_task_runner, std::move(store_map)), + store_and_hash_prefixes_(store_and_hash_prefixes), + stores_available_(stores_available) {} + + const StoreAndHashPrefixes store_and_hash_prefixes_; + const bool stores_available_; }; class TestClient : public SafeBrowsingDatabaseManager::Client { public: - TestClient(SBThreatType sb_threat_type, const GURL& url) - : expected_sb_threat_type(sb_threat_type), expected_url(url) {} + TestClient(SBThreatType sb_threat_type, + const GURL& url, + V4LocalDatabaseManager* manager_to_cancel = nullptr) + : expected_sb_threat_type(sb_threat_type), + expected_url(url), + on_check_browse_url_result_called_(false), + on_check_resource_url_result_called_(false), + manager_to_cancel_(manager_to_cancel) {} void OnCheckBrowseUrlResult(const GURL& url, SBThreatType threat_type, const ThreatMetadata& metadata) override { DCHECK_EQ(expected_url, url); DCHECK_EQ(expected_sb_threat_type, threat_type); + on_check_browse_url_result_called_ = true; + if (manager_to_cancel_) { + manager_to_cancel_->CancelCheck(this); + } + } + + void OnCheckResourceUrlResult(const GURL& url, + SBThreatType threat_type, + const std::string& threat_hash) override { + DCHECK_EQ(expected_url, url); + DCHECK_EQ(expected_sb_threat_type, threat_type); + // |threat_hash| is empty because GetFullHashes calls back with empty + // |full_hash_infos|. + DCHECK(threat_hash.empty()); + on_check_resource_url_result_called_ = true; } SBThreatType expected_sb_threat_type; GURL expected_url; + bool on_check_browse_url_result_called_; + bool on_check_resource_url_result_called_; + V4LocalDatabaseManager* manager_to_cancel_; +}; + +class FakeV4LocalDatabaseManager : public V4LocalDatabaseManager { + public: + void PerformFullHashCheck(std::unique_ptr<PendingCheck> check, + const FullHashToStoreAndHashPrefixesMap& + full_hash_to_store_and_hash_prefixes) override { + perform_full_hash_check_called_ = true; + } + + FakeV4LocalDatabaseManager(const base::FilePath& base_path) + : V4LocalDatabaseManager(base_path), + perform_full_hash_check_called_(false) {} + + static bool PerformFullHashCheckCalled( + scoped_refptr<safe_browsing::V4LocalDatabaseManager>& v4_ldbm) { + FakeV4LocalDatabaseManager* fake = + static_cast<FakeV4LocalDatabaseManager*>(v4_ldbm.get()); + return fake->perform_full_hash_check_called_; + } + + private: + ~FakeV4LocalDatabaseManager() override {} + + bool perform_full_hash_check_called_; }; class V4LocalDatabaseManagerTest : public PlatformTest { @@ -62,7 +217,7 @@ class V4LocalDatabaseManagerTest : public PlatformTest { v4_local_database_manager_ = make_scoped_refptr(new V4LocalDatabaseManager(base_dir_.GetPath())); - v4_local_database_manager_->SetTaskRunnerForTest(task_runner_); + SetTaskRunnerForTest(); StartLocalDatabaseManager(); } @@ -77,22 +232,55 @@ class V4LocalDatabaseManagerTest : public PlatformTest { v4_local_database_manager_->enabled_ = false; } + void ForceEnableLocalDatabaseManager() { + v4_local_database_manager_->enabled_ = true; + } + const V4LocalDatabaseManager::QueuedChecks& GetQueuedChecks() { return v4_local_database_manager_->queued_checks_; } - void ReplaceV4Database(const StoreAndHashPrefixes& store_and_hash_prefixes) { - v4_local_database_manager_->v4_database_.reset(new FakeV4Database( - task_runner_, base::MakeUnique<StoreMap>(), store_and_hash_prefixes)); + void ReplaceV4Database(const StoreAndHashPrefixes& store_and_hash_prefixes, + bool stores_available = false) { + // Disable the V4LocalDatabaseManager first so that if the callback to + // verify checksum has been scheduled, then it doesn't do anything when it + // is called back. + ForceDisableLocalDatabaseManager(); + // Wait to make sure that the callback gets executed if it has already been + // scheduled. + WaitForTasksOnTaskRunner(); + // Re-enable the V4LocalDatabaseManager otherwise the checks won't work and + // the fake database won't be set either. + ForceEnableLocalDatabaseManager(); + + NewDatabaseReadyCallback db_ready_callback = + base::Bind(&V4LocalDatabaseManager::DatabaseReadyForChecks, + base::Unretained(v4_local_database_manager_.get())); + FakeV4Database::Create(task_runner_, base::MakeUnique<StoreMap>(), + store_and_hash_prefixes, db_ready_callback, + stores_available); + WaitForTasksOnTaskRunner(); + } + + void ResetLocalDatabaseManager() { + StopLocalDatabaseManager(); + v4_local_database_manager_ = + make_scoped_refptr(new V4LocalDatabaseManager(base_dir_.GetPath())); + SetTaskRunnerForTest(); + StartLocalDatabaseManager(); + } + + void ResetV4Database() { + V4Database::Destroy(std::move(v4_local_database_manager_->v4_database_)); } - void ResetV4Database() { v4_local_database_manager_->v4_database_.reset(); } + void SetTaskRunnerForTest() { + v4_local_database_manager_->SetTaskRunnerForTest(task_runner_); + } void StartLocalDatabaseManager() { - v4_local_database_manager_->StartOnIOThread(NULL, V4ProtocolConfig()); - - task_runner_->RunPendingTasks(); - base::RunLoop().RunUntilIdle(); + v4_local_database_manager_->StartOnIOThread(NULL, + GetTestV4ProtocolConfig()); } void StopLocalDatabaseManager() { @@ -101,10 +289,28 @@ class V4LocalDatabaseManagerTest : public PlatformTest { } // Force destruction of the database. + WaitForTasksOnTaskRunner(); + } + + void WaitForTasksOnTaskRunner() { + // Wait for tasks on the task runner so we're sure that the + // V4LocalDatabaseManager has read the data from disk. task_runner_->RunPendingTasks(); base::RunLoop().RunUntilIdle(); } + // For those tests that need the fake manager + void SetupFakeManager() { + // StopLocalDatabaseManager before resetting it because that's what + // ~V4LocalDatabaseManager expects. + StopLocalDatabaseManager(); + v4_local_database_manager_ = + make_scoped_refptr(new FakeV4LocalDatabaseManager(base_dir_.GetPath())); + SetTaskRunnerForTest(); + StartLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + } + base::ScopedTempDir base_dir_; scoped_refptr<base::TestSimpleTaskRunner> task_runner_; content::TestBrowserThreadBundle thread_bundle_; @@ -112,15 +318,18 @@ class V4LocalDatabaseManagerTest : public PlatformTest { }; TEST_F(V4LocalDatabaseManagerTest, TestGetThreatSource) { + WaitForTasksOnTaskRunner(); EXPECT_EQ(ThreatSource::LOCAL_PVER4, v4_local_database_manager_->GetThreatSource()); } TEST_F(V4LocalDatabaseManagerTest, TestIsSupported) { + WaitForTasksOnTaskRunner(); EXPECT_TRUE(v4_local_database_manager_->IsSupported()); } TEST_F(V4LocalDatabaseManagerTest, TestCanCheckUrl) { + WaitForTasksOnTaskRunner(); EXPECT_TRUE( v4_local_database_manager_->CanCheckUrl(GURL("http://example.com/a/"))); EXPECT_TRUE( @@ -133,28 +342,32 @@ TEST_F(V4LocalDatabaseManagerTest, TestCanCheckUrl) { TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithEmptyStoresReturnsNoMatch) { + WaitForTasksOnTaskRunner(); // Both the stores are empty right now so CheckBrowseUrl should return true. EXPECT_TRUE(v4_local_database_manager_->CheckBrowseUrl( GURL("http://example.com/a/"), nullptr)); } TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithFakeDbReturnsMatch) { + WaitForTasksOnTaskRunner(); net::TestURLFetcherFactory factory; StoreAndHashPrefixes store_and_hash_prefixes; - store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), HashPrefix("aaaa")); + store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), + HashPrefix("eW\x1A\xF\xA9")); ReplaceV4Database(store_and_hash_prefixes); // The fake database returns a matched hash prefix. EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( GURL("http://example.com/a/"), nullptr)); + + // Wait for PerformFullHashCheck to complete. + WaitForTasksOnTaskRunner(); } TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlReturnsNoMatchWhenDisabled) { - StoreAndHashPrefixes store_and_hash_prefixes; - store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), HashPrefix("aaaa")); - ReplaceV4Database(store_and_hash_prefixes); + WaitForTasksOnTaskRunner(); // The same URL returns |false| in the previous test because // v4_local_database_manager_ is enabled. @@ -165,8 +378,10 @@ TEST_F(V4LocalDatabaseManagerTest, } TEST_F(V4LocalDatabaseManagerTest, TestGetSeverestThreatTypeAndMetadata) { - FullHashInfo fhi_malware(FullHash("Malware"), GetUrlMalwareId(), - base::Time::Now()); + WaitForTasksOnTaskRunner(); + + FullHash full_hash("Malware"); + FullHashInfo fhi_malware(full_hash, GetUrlMalwareId(), base::Time::Now()); fhi_malware.metadata.population_id = "malware_popid"; FullHashInfo fhi_api(FullHash("api"), GetChromeUrlApiId(), base::Time::Now()); @@ -176,18 +391,21 @@ TEST_F(V4LocalDatabaseManagerTest, TestGetSeverestThreatTypeAndMetadata) { SBThreatType result_threat_type; ThreatMetadata metadata; + FullHash matching_full_hash; v4_local_database_manager_->GetSeverestThreatTypeAndMetadata( - &result_threat_type, &metadata, fhis); + &result_threat_type, &metadata, &matching_full_hash, fhis); EXPECT_EQ(SB_THREAT_TYPE_URL_MALWARE, result_threat_type); EXPECT_EQ("malware_popid", metadata.population_id); + EXPECT_EQ(full_hash, matching_full_hash); // Reversing the list has no effect. std::reverse(std::begin(fhis), std::end(fhis)); v4_local_database_manager_->GetSeverestThreatTypeAndMetadata( - &result_threat_type, &metadata, fhis); + &result_threat_type, &metadata, &matching_full_hash, fhis); EXPECT_EQ(SB_THREAT_TYPE_URL_MALWARE, result_threat_type); EXPECT_EQ("malware_popid", metadata.population_id); + EXPECT_EQ(full_hash, matching_full_hash); } TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) { @@ -195,17 +413,11 @@ TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) { TestClient client(SB_THREAT_TYPE_SAFE, url); EXPECT_TRUE(GetQueuedChecks().empty()); v4_local_database_manager_->CheckBrowseUrl(url, &client); - // The database is available so the check shouldn't get queued. - EXPECT_TRUE(GetQueuedChecks().empty()); - - ResetV4Database(); - v4_local_database_manager_->CheckBrowseUrl(url, &client); // The database is unavailable so the check should get queued. EXPECT_EQ(1ul, GetQueuedChecks().size()); - // The following function calls StartOnIOThread which should load the - // database from disk and cause the queued check to be performed. - StartLocalDatabaseManager(); + // The following function waits for the DB to load. + WaitForTasksOnTaskRunner(); EXPECT_TRUE(GetQueuedChecks().empty()); ResetV4Database(); @@ -217,4 +429,296 @@ TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) { EXPECT_TRUE(GetQueuedChecks().empty()); } +// Verify that a window where checks cannot be cancelled is closed. +TEST_F(V4LocalDatabaseManagerTest, CancelPending) { + // Setup to receive full-hash misses. + ScopedFakeGetHashProtocolManagerFactory pin; + + // Reset the database manager so it picks up the replacement protocol manager. + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + // An URL and matching prefix. + const GURL url("http://example.com/a/"); + const HashPrefix hash_prefix("eW\x1A\xF\xA9"); + + // Put a match in the db that will cause a protocol-manager request. + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), hash_prefix); + ReplaceV4Database(store_and_hash_prefixes); + + // Test that a request flows through to the callback. + { + TestClient client(SB_THREAT_TYPE_SAFE, url); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client)); + EXPECT_FALSE(client.on_check_browse_url_result_called_); + WaitForTasksOnTaskRunner(); + EXPECT_TRUE(client.on_check_browse_url_result_called_); + } + + // Test that cancel prevents the callback from being called. + { + TestClient client(SB_THREAT_TYPE_SAFE, url); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client)); + v4_local_database_manager_->CancelCheck(&client); + EXPECT_FALSE(client.on_check_browse_url_result_called_); + WaitForTasksOnTaskRunner(); + EXPECT_FALSE(client.on_check_browse_url_result_called_); + } +} + +// When the database load flushes the queued requests, make sure that +// CancelCheck() is not fatal in the client callback. +TEST_F(V4LocalDatabaseManagerTest, CancelQueued) { + const GURL url("http://example.com/a/"); + + TestClient client1(SB_THREAT_TYPE_SAFE, url, + v4_local_database_manager_.get()); + TestClient client2(SB_THREAT_TYPE_SAFE, url); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client1)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client2)); + EXPECT_EQ(2ul, GetQueuedChecks().size()); + EXPECT_FALSE(client1.on_check_browse_url_result_called_); + EXPECT_FALSE(client2.on_check_browse_url_result_called_); + WaitForTasksOnTaskRunner(); + EXPECT_TRUE(client1.on_check_browse_url_result_called_); + EXPECT_TRUE(client2.on_check_browse_url_result_called_); +} + +// This test is somewhat similar to TestCheckBrowseUrlWithFakeDbReturnsMatch but +// it uses a fake V4LocalDatabaseManager to assert that PerformFullHashCheck is +// called async. +TEST_F(V4LocalDatabaseManagerTest, PerformFullHashCheckCalledAsync) { + SetupFakeManager(); + net::TestURLFetcherFactory factory; + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), + HashPrefix("eW\x1A\xF\xA9")); + ReplaceV4Database(store_and_hash_prefixes); + + // The fake database returns a matched hash prefix. + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + GURL("http://example.com/a/"), nullptr)); + + EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); + + // Wait for PerformFullHashCheck to complete. + WaitForTasksOnTaskRunner(); + + EXPECT_TRUE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); +} + +TEST_F(V4LocalDatabaseManagerTest, UsingWeakPtrDropsCallback) { + SetupFakeManager(); + net::TestURLFetcherFactory factory; + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), + HashPrefix("eW\x1A\xF\xA9")); + ReplaceV4Database(store_and_hash_prefixes); + + // The fake database returns a matched hash prefix. + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + GURL("http://example.com/a/"), nullptr)); + v4_local_database_manager_->StopOnIOThread(true); + + // Release the V4LocalDatabaseManager object right away before the callback + // gets called. When the callback gets called, without using a weak-ptr + // factory, this leads to a use after free. However, using the weak-ptr means + // that the callback is simply dropped. + v4_local_database_manager_ = nullptr; + + // Wait for the tasks scheduled by StopOnIOThread to complete. + WaitForTasksOnTaskRunner(); +} + +TEST_F(V4LocalDatabaseManagerTest, TestMatchCsdWhitelistUrl) { + SetupFakeManager(); + GURL good_url("http://safe.com"); + GURL other_url("http://iffy.com"); + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlCsdWhitelistId(), + HashForUrl(good_url)); + + ReplaceV4Database(store_and_hash_prefixes, false /* not available */); + // No match, but since we never loaded the whitelist (not available), + // it defaults to true. + EXPECT_TRUE(v4_local_database_manager_->MatchCsdWhitelistUrl(good_url)); + + ReplaceV4Database(store_and_hash_prefixes, true /* available */); + // Not whitelisted. + EXPECT_FALSE(v4_local_database_manager_->MatchCsdWhitelistUrl(other_url)); + // Whitelisted. + EXPECT_TRUE(v4_local_database_manager_->MatchCsdWhitelistUrl(good_url)); + + EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); +} + +TEST_F(V4LocalDatabaseManagerTest, TestMatchDownloadWhitelistString) { + SetupFakeManager(); + FullHash good_hash(crypto::SHA256HashString("Good .exe contents")); + FullHash other_hash(crypto::SHA256HashString("Other .exe contents")); + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetCertCsdDownloadWhitelistId(), + good_hash); + + ReplaceV4Database(store_and_hash_prefixes, false /* not available */); + // Verify it defaults to false when DB is not available. + EXPECT_FALSE( + v4_local_database_manager_->MatchDownloadWhitelistString(good_hash)); + + ReplaceV4Database(store_and_hash_prefixes, true /* available */); + // Not whitelisted. + EXPECT_FALSE( + v4_local_database_manager_->MatchDownloadWhitelistString(other_hash)); + // Whitelisted. + EXPECT_TRUE( + v4_local_database_manager_->MatchDownloadWhitelistString(good_hash)); + + EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); +} + +TEST_F(V4LocalDatabaseManagerTest, TestMatchDownloadWhitelistUrl) { + SetupFakeManager(); + GURL good_url("http://safe.com"); + GURL other_url("http://iffy.com"); + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetCertCsdDownloadWhitelistId(), + HashForUrl(good_url)); + + ReplaceV4Database(store_and_hash_prefixes, false /* not available */); + // Verify it defaults to false when DB is not available. + EXPECT_FALSE(v4_local_database_manager_->MatchDownloadWhitelistUrl(good_url)); + + ReplaceV4Database(store_and_hash_prefixes, true /* available */); + // Not whitelisted. + EXPECT_FALSE( + v4_local_database_manager_->MatchDownloadWhitelistUrl(other_url)); + // Whitelisted. + EXPECT_TRUE(v4_local_database_manager_->MatchDownloadWhitelistUrl(good_url)); + + EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); +} + +TEST_F(V4LocalDatabaseManagerTest, TestMatchMalwareIP) { + SetupFakeManager(); + + // >>> hashlib.sha1(socket.inet_pton(socket.AF_INET6, + // '::ffff:192.168.1.2')).digest() + chr(128) + // '\xb3\xe0z\xafAv#h\x9a\xcf<\xf3ee\x94\xda\xf6y\xb1\xad\x80' + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetIpMalwareId(), + FullHash("\xB3\xE0z\xAF" + "Av#h\x9A\xCF<\xF3" + "ee\x94\xDA\xF6y\xB1\xAD\x80")); + ReplaceV4Database(store_and_hash_prefixes); + + EXPECT_FALSE(v4_local_database_manager_->MatchMalwareIP("")); + // Not blacklisted. + EXPECT_FALSE(v4_local_database_manager_->MatchMalwareIP("192.168.1.1")); + // Blacklisted. + EXPECT_TRUE(v4_local_database_manager_->MatchMalwareIP("192.168.1.2")); + + EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); +} + +TEST_F(V4LocalDatabaseManagerTest, TestMatchModuleWhitelist) { + SetupFakeManager(); + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetChromeFilenameClientIncidentId(), + crypto::SHA256HashString("chrome.dll")); + + ReplaceV4Database(store_and_hash_prefixes, false /* not available */); + // No match, but since we never loaded the whitelist (not available), + // it defaults to true. + EXPECT_TRUE( + v4_local_database_manager_->MatchModuleWhitelistString("badstuff.dll")); + + ReplaceV4Database(store_and_hash_prefixes, true /* available */); + // Not whitelisted. + EXPECT_FALSE( + v4_local_database_manager_->MatchModuleWhitelistString("badstuff.dll")); + // Whitelisted. + EXPECT_TRUE( + v4_local_database_manager_->MatchModuleWhitelistString("chrome.dll")); + + EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( + v4_local_database_manager_)); +} + +// This verifies the fix for race in http://crbug.com/660293 +TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithSameClientAndCancel) { + ScopedFakeGetHashProtocolManagerFactory pin; + // Reset the database manager so it picks up the replacement protocol manager. + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), + HashPrefix("sÙ†\340\t\006_")); + ReplaceV4Database(store_and_hash_prefixes); + + GURL first_url("http://example.com/a"); + GURL second_url("http://example.com/"); + TestClient client(SB_THREAT_TYPE_SAFE, first_url); + // The fake database returns a matched hash prefix. + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(first_url, &client)); + + // That check gets queued. Now, let's cancel the check. After this, we should + // not receive a call for |OnCheckBrowseUrlResult| with |first_url|. + v4_local_database_manager_->CancelCheck(&client); + + // Now, re-use that client but for |second_url|. + client.expected_url = second_url; + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(second_url, &client)); + + // Wait for PerformFullHashCheck to complete. + WaitForTasksOnTaskRunner(); + // |on_check_browse_url_result_called_| is true only if OnCheckBrowseUrlResult + // gets called with the |url| equal to |expected_url|, which is |second_url| + // in + // this test. + EXPECT_TRUE(client.on_check_browse_url_result_called_); +} + +TEST_F(V4LocalDatabaseManagerTest, TestCheckResourceUrl) { + // Setup to receive full-hash misses. + ScopedFakeGetHashProtocolManagerFactory pin; + + // Reset the database manager so it picks up the replacement protocol manager. + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + // An URL and matching prefix. + const GURL url("http://example.com/a/"); + const HashPrefix hash_prefix("eW\x1A\xF\xA9"); + + // Put a match in the db that will cause a protocol-manager request. + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetChromeUrlClientIncidentId(), + hash_prefix); + ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */); + + TestClient client(SB_THREAT_TYPE_SAFE, url); + EXPECT_FALSE(v4_local_database_manager_->CheckResourceUrl(url, &client)); + EXPECT_FALSE(client.on_check_resource_url_result_called_); + WaitForTasksOnTaskRunner(); + EXPECT_TRUE(client.on_check_resource_url_result_called_); +} + +// TODO(nparker): Add tests for +// CheckDownloadUrl() +// CheckExtensionIDs() + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc b/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc index c349b6b9d8f..dad0d7917cd 100644 --- a/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc +++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc @@ -5,12 +5,14 @@ #include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "base/base64.h" -#include "base/metrics/sparse_histogram.h" +#include "base/metrics/histogram_macros.h" #include "base/rand_util.h" +#include "base/sha1.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" #include "crypto/sha2.h" #include "net/base/escape.h" +#include "net/base/ip_address.h" #include "net/http/http_request_headers.h" #include "url/url_util.h" @@ -85,19 +87,55 @@ return LINUX_PLATFORM; #endif } +const ListIdentifier GetCertCsdDownloadWhitelistId() { + return ListIdentifier(GetCurrentPlatformType(), CERT, CSD_DOWNLOAD_WHITELIST); +} + +const ListIdentifier GetChromeExtMalwareId() { + return ListIdentifier(CHROME_PLATFORM, CHROME_EXTENSION, MALWARE_THREAT); +} + const ListIdentifier GetChromeUrlApiId() { return ListIdentifier(CHROME_PLATFORM, URL, API_ABUSE); } +const ListIdentifier GetChromeFilenameClientIncidentId() { + return ListIdentifier(CHROME_PLATFORM, FILENAME, CLIENT_INCIDENT); +} + +const ListIdentifier GetChromeUrlClientIncidentId() { + return ListIdentifier(CHROME_PLATFORM, URL, CLIENT_INCIDENT); +} + +const ListIdentifier GetIpMalwareId() { + return ListIdentifier(GetCurrentPlatformType(), IP_RANGE, MALWARE_THREAT); +} + +const ListIdentifier GetUrlCsdDownloadWhitelistId() { + return ListIdentifier(GetCurrentPlatformType(), URL, CSD_DOWNLOAD_WHITELIST); +} + +const ListIdentifier GetUrlCsdWhitelistId() { + return ListIdentifier(GetCurrentPlatformType(), URL, CSD_WHITELIST); +} + const ListIdentifier GetUrlMalwareId() { return ListIdentifier(GetCurrentPlatformType(), URL, MALWARE_THREAT); } +const ListIdentifier GetUrlMalBinId() { + return ListIdentifier(GetCurrentPlatformType(), URL, MALICIOUS_BINARY); +} + const ListIdentifier GetUrlSocEngId() { return ListIdentifier(GetCurrentPlatformType(), URL, SOCIAL_ENGINEERING_PUBLIC); } +const ListIdentifier GetUrlUwsId() { + return ListIdentifier(GetCurrentPlatformType(), URL, UNWANTED_SOFTWARE); +} + // The Safe Browsing V4 server URL prefix. const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4"; @@ -159,7 +197,14 @@ ListIdentifier::ListIdentifier(const ListUpdateResponse& response) response.threat_entry_type(), response.threat_type()) {} -V4ProtocolConfig::V4ProtocolConfig() : disable_auto_update(false) {} +V4ProtocolConfig::V4ProtocolConfig(const std::string& client_name, + bool disable_auto_update, + const std::string& key_param, + const std::string& version) + : client_name(client_name), + disable_auto_update(disable_auto_update), + key_param(key_param), + version(version) {} V4ProtocolConfig::V4ProtocolConfig(const V4ProtocolConfig& other) = default; @@ -237,7 +282,7 @@ void V4ProtocolManagerUtil::UpdateHeaders(net::HttpRequestHeaders* headers) { // static void V4ProtocolManagerUtil::UrlToFullHashes( const GURL& url, - std::unordered_set<FullHash>* full_hashes) { + std::vector<FullHash>* full_hashes) { std::string canon_host, canon_path, canon_query; CanonicalizeUrl(url, &canon_host, &canon_path, &canon_query); @@ -252,7 +297,7 @@ void V4ProtocolManagerUtil::UrlToFullHashes( GeneratePathVariantsToCheck(canon_path, canon_query, &paths); for (const std::string& host : hosts) { for (const std::string& path : paths) { - full_hashes->insert(crypto::SHA256HashString(host + path)); + full_hashes->push_back(crypto::SHA256HashString(host + path)); } } } @@ -505,4 +550,37 @@ void V4ProtocolManagerUtil::SetClientInfoFromConfig( client_info->set_client_version(config.version); } +// static +bool V4ProtocolManagerUtil::GetIPV6AddressFromString( + const std::string& ip_address, + net::IPAddress* address) { + DCHECK(address); + if (!address->AssignFromIPLiteral(ip_address)) + return false; + if (address->IsIPv4()) + *address = net::ConvertIPv4ToIPv4MappedIPv6(*address); + return address->IsIPv6(); +} + +// static +bool V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash( + const std::string& ip_address, + FullHash* hashed_encoded_ip) { + net::IPAddress address; + if (!GetIPV6AddressFromString(ip_address, &address)) { + return false; + } + std::string packed_ip = net::IPAddressToPackedString(address); + if (packed_ip.empty()) { + return false; + } + + const std::string hash = base::SHA1HashString(packed_ip); + DCHECK_EQ(20u, hash.size()); + hashed_encoded_ip->resize(hash.size() + 1); + hashed_encoded_ip->replace(0, hash.size(), hash); + (*hashed_encoded_ip)[hash.size()] = static_cast<unsigned char>(128); + return true; +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util.h b/chromium/components/safe_browsing_db/v4_protocol_manager_util.h index 34e24125d7e..707b82da144 100644 --- a/chromium/components/safe_browsing_db/v4_protocol_manager_util.h +++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util.h @@ -20,6 +20,7 @@ namespace net { class HttpRequestHeaders; +class IPAddress; } // namespace net namespace safe_browsing { @@ -49,23 +50,33 @@ struct V4ProtocolConfig { // The safe browsing client name sent in each request. std::string client_name; - // Current product version sent in each request. - std::string version; + // Disable auto-updates using a command line switch. + bool disable_auto_update; // The Google API key. std::string key_param; - // Disable auto-updates using a command line switch? - bool disable_auto_update; + // Current product version sent in each request. + std::string version; - V4ProtocolConfig(); + V4ProtocolConfig(const std::string& client_name, + bool disable_auto_update, + const std::string& key_param, + const std::string& version); V4ProtocolConfig(const V4ProtocolConfig& other); ~V4ProtocolConfig(); + + private: + V4ProtocolConfig(); }; // Different types of threats that SafeBrowsing protects against. This is the // type that's returned to the clients of SafeBrowsing in Chromium. enum SBThreatType { + // This type can be used for lists that can be checked synchronously so a + // client callback isn't required, or for whitelists. + SB_THREAT_TYPE_UNUSED, + // No threat at all. SB_THREAT_TYPE_SAFE, @@ -130,9 +141,18 @@ struct ListIdentifier { std::ostream& operator<<(std::ostream& os, const ListIdentifier& id); PlatformType GetCurrentPlatformType(); +const ListIdentifier GetCertCsdDownloadWhitelistId(); +const ListIdentifier GetChromeExtMalwareId(); const ListIdentifier GetChromeUrlApiId(); +const ListIdentifier GetChromeFilenameClientIncidentId(); +const ListIdentifier GetChromeUrlClientIncidentId(); +const ListIdentifier GetIpMalwareId(); +const ListIdentifier GetUrlCsdDownloadWhitelistId(); +const ListIdentifier GetUrlCsdWhitelistId(); const ListIdentifier GetUrlMalwareId(); +const ListIdentifier GetUrlMalBinId(); const ListIdentifier GetUrlSocEngId(); +const ListIdentifier GetUrlUwsId(); // Represents the state of each store. typedef base::hash_map<ListIdentifier, std::string> StoreStateMap; @@ -140,7 +160,6 @@ typedef base::hash_map<ListIdentifier, std::string> StoreStateMap; // Sever response, parsed in vector form. typedef std::vector<std::unique_ptr<ListUpdateResponse>> ParsedServerResponse; -// TODO(vakh): Consider using a std::pair for this. // Holds the hash prefix and the store that it matched in. struct StoreAndHashPrefix { public: @@ -248,7 +267,7 @@ class V4ProtocolManagerUtil { // Generate the set of FullHashes to check for |url|. static void UrlToFullHashes(const GURL& url, - std::unordered_set<FullHash>* full_hashes); + std::vector<FullHash>* full_hashes); static bool FullHashToHashPrefix(const FullHash& full_hash, PrefixSize prefix_size, @@ -263,6 +282,16 @@ class V4ProtocolManagerUtil { static void SetClientInfoFromConfig(ClientInfo* client_info, const V4ProtocolConfig& config); + static bool GetIPV6AddressFromString(const std::string& ip_address, + net::IPAddress* address); + + // Converts a IPV4 or IPV6 address in |ip_address| to the SHA1 hash of the + // corresponding packed IPV6 address in |hashed_encoded_ip|, and adds an + // extra byte containing the value 128 at the end. This is done to match the + // server implementation for calculating the hash prefix of an IP address. + static bool IPAddressToEncodedIPV6Hash(const std::string& ip_address, + FullHash* hashed_encoded_ip); + private: V4ProtocolManagerUtil(){}; FRIEND_TEST_ALL_PREFIXES(V4ProtocolManagerUtilTest, TestBackOffLogic); diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc b/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc index 20e808576bd..0680d3cc060 100644 --- a/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util_unittest.cc @@ -9,6 +9,7 @@ #include "base/base64.h" #include "base/strings/stringprintf.h" #include "base/time/time.h" +#include "components/safe_browsing_db/v4_test_util.h" #include "net/base/escape.h" #include "net/http/http_request_headers.h" #include "testing/gtest/include/gtest/gtest.h" @@ -18,10 +19,6 @@ using base::TimeDelta; namespace { -const char kClient[] = "unittest"; -const char kAppVer[] = "1.0"; -const char kKeyParam[] = "test_key_param"; - bool VectorContains(const std::vector<std::string>& data, const std::string& str) { return std::find(data.begin(), data.end(), str) != data.end(); @@ -32,12 +29,6 @@ bool VectorContains(const std::vector<std::string>& data, namespace safe_browsing { class V4ProtocolManagerUtilTest : public testing::Test { - protected: - void PopulateV4ProtocolConfig(V4ProtocolConfig* config) { - config->client_name = kClient; - config->version = kAppVer; - config->key_param = kKeyParam; - } }; TEST_F(V4ProtocolManagerUtilTest, TestBackOffLogic) { @@ -115,13 +106,11 @@ TEST_F(V4ProtocolManagerUtilTest, TestBackOffLogic) { } TEST_F(V4ProtocolManagerUtilTest, TestGetRequestUrlAndUpdateHeaders) { - V4ProtocolConfig config; - PopulateV4ProtocolConfig(&config); - net::HttpRequestHeaders headers; GURL gurl; V4ProtocolManagerUtil::GetRequestUrlAndHeaders("request_base64", "someMethod", - config, &gurl, &headers); + GetTestV4ProtocolConfig(), + &gurl, &headers); std::string expectedUrl = "https://safebrowsing.googleapis.com/v4/someMethod?" "$req=request_base64&$ct=application/x-protobuf&key=test_key_param"; @@ -249,4 +238,35 @@ TEST_F(V4ProtocolManagerUtilTest, CanonicalizeUrl) { } } +TEST_F(V4ProtocolManagerUtilTest, TestIPAddressToEncodedIPV6) { + // To verify the test values, here's the python code: + // >> import socket, hashlib, binascii + // >> hashlib.sha1(socket.inet_pton(socket.AF_INET6, input)).digest() + + // chr(128) + // For example: + // >>> hashlib.sha1(socket.inet_pton(socket.AF_INET6, + // '::ffff:192.168.1.1')).digest() + chr(128) + // 'X\xf8\xa1\x17I\xe6Pl\xfd\xdb\xbb\xa0\x0c\x02\x9d#\n|\xe7\xcd\x80' + std::vector<std::tuple<bool, std::string, std::string>> test_cases = { + std::make_tuple(false, "", ""), + std::make_tuple( + true, "192.168.1.1", + "X\xF8\xA1\x17I\xE6Pl\xFD\xDB\xBB\xA0\f\x2\x9D#\n|\xE7\xCD\x80"), + std::make_tuple( + true, "::", + "\xE1)\xF2|Q\x3\xBC\\\xC4K\xCD\xF0\xA1^\x16\rDPf\xFF\x80")}; + for (size_t i = 0; i < test_cases.size(); i++) { + DVLOG(1) << "Running case: " << i; + bool success = std::get<0>(test_cases[i]); + const auto& input = std::get<1>(test_cases[i]); + const auto& expected_output = std::get<2>(test_cases[i]); + std::string encoded_ip; + ASSERT_EQ(success, V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash( + input, &encoded_ip)); + if (success) { + ASSERT_EQ(expected_output, encoded_ip); + } + } +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_store.cc b/chromium/components/safe_browsing_db/v4_store.cc index d00573156ef..889c55c4b66 100644 --- a/chromium/components/safe_browsing_db/v4_store.cc +++ b/chromium/components/safe_browsing_db/v4_store.cc @@ -22,8 +22,28 @@ namespace safe_browsing { namespace { -const uint32_t kFileMagic = 0x600D71FE; +// UMA related strings. +// Part 1: Represent the overall operation being performed. +const char kProcessFullUpdate[] = "SafeBrowsing.V4ProcessFullUpdate"; +const char kProcessPartialUpdate[] = "SafeBrowsing.V4ProcessPartialUpdate"; +const char kReadFromDisk[] = "SafeBrowsing.V4ReadFromDisk"; +// Part 2: Represent the sub-operation being performed as part of the larger +// operation from part 1. +const char kApplyUpdate[] = ".ApplyUpdate"; +const char kDecodeAdditions[] = ".DecodeAdditions"; +const char kDecodeRemovals[] = ".DecodeRemovals"; +const char kMergeUpdate[] = ".MergeUpdate"; +// Part 3: Represent the unit of value being measured and logged. +const char kResult[] = ".Result"; +const char kTime[] = ".Time"; +// Part 4 (optional): Represent the name of the list for which the metric is +// being logged. For instance, ".UrlSoceng". +// UMA metric names for this file are generated by appending one value each, +// in order, from parts [1, 2, and 3], or [1, 2, 3, and 4]. For example: +// SafeBrowsing.V4ProcessPartialUpdate.ApplyUpdate.Result, or +// SafeBrowsing.V4ProcessPartialUpdate.ApplyUpdate.Result.UrlSoceng +const uint32_t kFileMagic = 0x600D71FE; const uint32_t kFileVersion = 9; std::string GetUmaSuffixForStore(const base::FilePath& file_path) { @@ -31,26 +51,25 @@ std::string GetUmaSuffixForStore(const base::FilePath& file_path) { ".%" PRIsFP, file_path.BaseName().RemoveExtension().value().c_str()); } -void RecordTimeWithAndWithoutStore(const std::string& metric, - base::TimeDelta time, - const base::FilePath& file_path) { - std::string suffix = GetUmaSuffixForStore(file_path); - +void RecordTimeWithAndWithoutSuffix(const std::string& metric, + base::TimeDelta time, + const base::FilePath& file_path) { // The histograms below are a modified expansion of the // UMA_HISTOGRAM_LONG_TIMES macro adapted to allow for a dynamically suffixed // histogram name. // Note: The factory creates and owns the histogram. const int kBucketCount = 100; base::HistogramBase* histogram = base::Histogram::FactoryTimeGet( - metric, base::TimeDelta::FromMilliseconds(1), + metric + kTime, base::TimeDelta::FromMilliseconds(1), base::TimeDelta::FromMinutes(1), kBucketCount, base::HistogramBase::kUmaTargetedHistogramFlag); if (histogram) { histogram->AddTime(time); } + std::string suffix = GetUmaSuffixForStore(file_path); base::HistogramBase* histogram_suffix = base::Histogram::FactoryTimeGet( - metric + suffix, base::TimeDelta::FromMilliseconds(1), + metric + kTime + suffix, base::TimeDelta::FromMilliseconds(1), base::TimeDelta::FromMinutes(1), kBucketCount, base::HistogramBase::kUmaTargetedHistogramFlag); if (histogram_suffix) { @@ -58,74 +77,87 @@ void RecordTimeWithAndWithoutStore(const std::string& metric, } } +void RecordEnumWithAndWithoutSuffix(const std::string& metric, + int32_t value, + int32_t maximum, + const base::FilePath& file_path) { + // The histograms below are an expansion of the UMA_HISTOGRAM_ENUMERATION + // macro adapted to allow for a dynamically suffixed histogram name. + // Note: The factory creates and owns the histogram. + base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( + metric + kResult, 1, maximum, maximum + 1, + base::HistogramBase::kUmaTargetedHistogramFlag); + if (histogram) { + histogram->Add(value); + } + + std::string suffix = GetUmaSuffixForStore(file_path); + base::HistogramBase* histogram_suffix = base::LinearHistogram::FactoryGet( + metric + kResult + suffix, 1, maximum, maximum + 1, + base::HistogramBase::kUmaTargetedHistogramFlag); + if (histogram_suffix) { + histogram_suffix->Add(value); + } +} + void RecordAddUnlumpedHashesTime(base::TimeDelta time) { - UMA_HISTOGRAM_LONG_TIMES("SafeBrowsing.V4AddUnlumpedHashesTime", time); + UMA_HISTOGRAM_LONG_TIMES("SafeBrowsing.V4AddUnlumpedHashes.Time", time); } -void RecordApplyUpdateResult(ApplyUpdateResult result) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4ApplyUpdateResult", result, - APPLY_UPDATE_RESULT_MAX); +void RecordApplyUpdateResult(const std::string& base_metric, + ApplyUpdateResult result, + const base::FilePath& file_path) { + RecordEnumWithAndWithoutSuffix(base_metric + kApplyUpdate, result, + APPLY_UPDATE_RESULT_MAX, file_path); } -void RecordApplyUpdateResultWhenReadingFromDisk(ApplyUpdateResult result) { - UMA_HISTOGRAM_ENUMERATION( - "SafeBrowsing.V4ApplyUpdateResultWhenReadingFromDisk", result, - APPLY_UPDATE_RESULT_MAX); +void RecordApplyUpdateTime(const std::string& base_metric, + base::TimeDelta time, + const base::FilePath& file_path) { + RecordTimeWithAndWithoutSuffix(base_metric + kApplyUpdate, time, file_path); } -void RecordDecodeAdditionsResult(V4DecodeResult result) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4DecodeAdditionsResult", result, - DECODE_RESULT_MAX); +void RecordDecodeAdditionsResult(const std::string& base_metric, + V4DecodeResult result, + const base::FilePath& file_path) { + RecordEnumWithAndWithoutSuffix(base_metric + kDecodeAdditions, result, + DECODE_RESULT_MAX, file_path); } -void RecordDecodeAdditionsTime(base::TimeDelta time, +void RecordDecodeAdditionsTime(const std::string& base_metric, + base::TimeDelta time, const base::FilePath& file_path) { - RecordTimeWithAndWithoutStore("SafeBrowsing.V4DecodeAdditionsTime", time, - file_path); + RecordTimeWithAndWithoutSuffix(base_metric + kDecodeAdditions, time, + file_path); } -void RecordDecodeRemovalsResult(V4DecodeResult result) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4DecodeRemovalsResult", result, - DECODE_RESULT_MAX); +void RecordDecodeRemovalsResult(const std::string& base_metric, + V4DecodeResult result, + const base::FilePath& file_path) { + RecordEnumWithAndWithoutSuffix(base_metric + kDecodeRemovals, result, + DECODE_RESULT_MAX, file_path); } -void RecordDecodeRemovalsTime(base::TimeDelta time, +void RecordDecodeRemovalsTime(const std::string& base_metric, + base::TimeDelta time, const base::FilePath& file_path) { - RecordTimeWithAndWithoutStore("SafeBrowsing.V4DecodeRemovalsTime", time, - file_path); + RecordTimeWithAndWithoutSuffix(base_metric + kDecodeRemovals, time, + file_path); } -void RecordMergeUpdateTime(base::TimeDelta time, +void RecordMergeUpdateTime(const std::string& base_metric, + base::TimeDelta time, const base::FilePath& file_path) { - RecordTimeWithAndWithoutStore("SafeBrowsing.V4MergeUpdateTime", time, - file_path); -} - -void RecordProcessFullUpdateTime(base::TimeDelta time, - const base::FilePath& file_path) { - RecordTimeWithAndWithoutStore("SafeBrowsing.V4ProcessFullUpdateTime", time, - file_path); -} - -void RecordProcessPartialUpdateTime(base::TimeDelta time, - const base::FilePath& file_path) { - RecordTimeWithAndWithoutStore("SafeBrowsing.V4ProcessPartialUpdateTime", time, - file_path); -} - -void RecordReadFromDiskTime(base::TimeDelta time, - const base::FilePath& file_path) { - RecordTimeWithAndWithoutStore("SafeBrowsing.V4ReadFromDiskTime", time, - file_path); + RecordTimeWithAndWithoutSuffix(base_metric + kMergeUpdate, time, file_path); } void RecordStoreReadResult(StoreReadResult result) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreReadResult", result, + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreRead.Result", result, STORE_READ_RESULT_MAX); } void RecordStoreWriteResult(StoreWriteResult result) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreWriteResult", result, + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4StoreWrite.Result", result, STORE_WRITE_RESULT_MAX); } @@ -149,7 +181,7 @@ std::ostream& operator<<(std::ostream& os, const V4Store& store) { V4Store* V4StoreFactory::CreateV4Store( const scoped_refptr<base::SequencedTaskRunner>& task_runner, const base::FilePath& store_path) { - V4Store* new_store = new V4Store(task_runner, store_path); + V4Store* new_store = new V4Store(task_runner, store_path, 0); new_store->Initialize(); return new_store; } @@ -159,14 +191,25 @@ void V4Store::Initialize() { DCHECK(state_.empty()); StoreReadResult store_read_result = ReadFromDisk(); + has_valid_data_ = (store_read_result == READ_SUCCESS); RecordStoreReadResult(store_read_result); } -V4Store::V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner, - const base::FilePath& store_path) - : store_path_(store_path), task_runner_(task_runner) {} +bool V4Store::HasValidData() const { + return has_valid_data_; +} -V4Store::~V4Store() {} +V4Store::V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner, + const base::FilePath& store_path, + const int64_t old_file_size) + : file_size_(old_file_size), + has_valid_data_(false), + store_path_(store_path), + task_runner_(task_runner) {} + +V4Store::~V4Store() { + DCHECK(task_runner_->RunsTasksOnCurrentThread()); +} std::string V4Store::DebugString() const { std::string state_base64; @@ -176,41 +219,54 @@ std::string V4Store::DebugString() const { store_path_.value().c_str(), state_base64.c_str()); } -bool V4Store::Reset() { - // TODO(vakh): Implement skeleton. +// static +void V4Store::Destroy(std::unique_ptr<V4Store> v4_store) { + V4Store* v4_store_raw = v4_store.release(); + if (v4_store_raw) { + v4_store_raw->task_runner_->DeleteSoon(FROM_HERE, v4_store_raw); + } +} + +void V4Store::Reset() { + expected_checksum_.clear(); + hash_prefix_map_.clear(); state_ = ""; - return true; } ApplyUpdateResult V4Store::ProcessPartialUpdateAndWriteToDisk( + const std::string& metric, const HashPrefixMap& hash_prefix_map_old, std::unique_ptr<ListUpdateResponse> response) { DCHECK(response->has_response_type()); DCHECK_EQ(ListUpdateResponse::PARTIAL_UPDATE, response->response_type()); - TimeTicks before = TimeTicks::Now(); - ApplyUpdateResult result = ProcessUpdate(hash_prefix_map_old, response); + ApplyUpdateResult result = ProcessUpdate( + metric, hash_prefix_map_old, response, false /* delay_checksum check */); if (result == APPLY_UPDATE_SUCCESS) { - RecordProcessPartialUpdateTime(TimeTicks::Now() - before, store_path_); - // TODO(vakh): Create a ListUpdateResponse containing RICE encoded - // hash prefixes and response_type as FULL_UPDATE, and write that to disk. + Checksum checksum = response->checksum(); + response.reset(); + RecordStoreWriteResult(WriteToDisk(checksum)); } return result; } ApplyUpdateResult V4Store::ProcessFullUpdateAndWriteToDisk( + const std::string& metric, std::unique_ptr<ListUpdateResponse> response) { - TimeTicks before = TimeTicks::Now(); - ApplyUpdateResult result = ProcessFullUpdate(response); + ApplyUpdateResult result = + ProcessFullUpdate(metric, response, false /* delay_checksum check */); if (result == APPLY_UPDATE_SUCCESS) { - RecordStoreWriteResult(WriteToDisk(std::move(response))); - RecordProcessFullUpdateTime(TimeTicks::Now() - before, store_path_); + Checksum checksum = response->checksum(); + response.reset(); + RecordStoreWriteResult(WriteToDisk(checksum)); } return result; } ApplyUpdateResult V4Store::ProcessFullUpdate( - const std::unique_ptr<ListUpdateResponse>& response) { + const std::string& metric, + const std::unique_ptr<ListUpdateResponse>& response, + bool delay_checksum_check) { DCHECK(response->has_response_type()); DCHECK_EQ(ListUpdateResponse::FULL_UPDATE, response->response_type()); // TODO(vakh): For a full update, we don't need to process the update in @@ -218,12 +274,14 @@ ApplyUpdateResult V4Store::ProcessFullUpdate( // checksum. It might save some CPU cycles to store the full update as-is and // walk the list of hash prefixes in lexographical order only for checksum // calculation. - return ProcessUpdate(HashPrefixMap(), response); + return ProcessUpdate(metric, HashPrefixMap(), response, delay_checksum_check); } ApplyUpdateResult V4Store::ProcessUpdate( + const std::string& metric, const HashPrefixMap& hash_prefix_map_old, - const std::unique_ptr<ListUpdateResponse>& response) { + const std::unique_ptr<ListUpdateResponse>& response, + bool delay_checksum_check) { const RepeatedField<int32>* raw_removals = nullptr; RepeatedField<int32> rice_removals; size_t removals_size = response->removals_size(); @@ -243,11 +301,11 @@ ApplyUpdateResult V4Store::ProcessUpdate( rice_indices.num_entries(), rice_indices.encoded_data(), &rice_removals); - RecordDecodeRemovalsResult(decode_result); + RecordDecodeRemovalsResult(metric, decode_result, store_path_); if (decode_result != DECODE_SUCCESS) { return RICE_DECODING_FAILURE; } - RecordDecodeRemovalsTime(TimeTicks::Now() - before, store_path_); + RecordDecodeRemovalsTime(metric, TimeTicks::Now() - before, store_path_); raw_removals = &rice_removals; } else { NOTREACHED() << "Unexpected compression_type type: " << compression_type; @@ -256,8 +314,8 @@ ApplyUpdateResult V4Store::ProcessUpdate( } HashPrefixMap hash_prefix_map; - ApplyUpdateResult apply_update_result = - UpdateHashPrefixMapFromAdditions(response->additions(), &hash_prefix_map); + ApplyUpdateResult apply_update_result = UpdateHashPrefixMapFromAdditions( + metric, response->additions(), &hash_prefix_map); if (apply_update_result != APPLY_UPDATE_SUCCESS) { return apply_update_result; } @@ -268,12 +326,27 @@ ApplyUpdateResult V4Store::ProcessUpdate( } TimeTicks before = TimeTicks::Now(); - apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map, - raw_removals, expected_checksum); - if (apply_update_result != APPLY_UPDATE_SUCCESS) { - return apply_update_result; + if (delay_checksum_check) { + DCHECK(hash_prefix_map_old.empty()); + DCHECK(!raw_removals); + // We delay the checksum check at startup to be able to load the DB + // quickly. In this case, the |hash_prefix_map_old| should be empty, so just + // copy over the |hash_prefix_map|. + hash_prefix_map_ = hash_prefix_map; + + // Calculate the checksum asynchronously later and if it doesn't match, + // reset the store. + expected_checksum_ = expected_checksum; + + apply_update_result = APPLY_UPDATE_SUCCESS; + } else { + apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map, + raw_removals, expected_checksum); + if (apply_update_result != APPLY_UPDATE_SUCCESS) { + return apply_update_result; + } } - RecordMergeUpdateTime(TimeTicks::Now() - before, store_path_); + RecordMergeUpdateTime(metric, TimeTicks::Now() - before, store_path_); state_ = response->new_client_state(); return APPLY_UPDATE_SUCCESS; @@ -284,14 +357,18 @@ void V4Store::ApplyUpdate( const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, UpdatedStoreReadyCallback callback) { std::unique_ptr<V4Store> new_store( - new V4Store(this->task_runner_, this->store_path_)); + new V4Store(task_runner_, store_path_, file_size_)); ApplyUpdateResult apply_update_result; + std::string metric; + TimeTicks before = TimeTicks::Now(); if (response->response_type() == ListUpdateResponse::PARTIAL_UPDATE) { + metric = kProcessPartialUpdate; apply_update_result = new_store->ProcessPartialUpdateAndWriteToDisk( - hash_prefix_map_, std::move(response)); + metric, hash_prefix_map_, std::move(response)); } else if (response->response_type() == ListUpdateResponse::FULL_UPDATE) { + metric = kProcessFullUpdate; apply_update_result = - new_store->ProcessFullUpdateAndWriteToDisk(std::move(response)); + new_store->ProcessFullUpdateAndWriteToDisk(metric, std::move(response)); } else { apply_update_result = UNEXPECTED_RESPONSE_TYPE_FAILURE; NOTREACHED() << "Failure: Unexpected response type: " @@ -299,21 +376,26 @@ void V4Store::ApplyUpdate( } if (apply_update_result == APPLY_UPDATE_SUCCESS) { - // new_store is done updating, pass it to the callback. - callback_task_runner->PostTask( - FROM_HERE, base::Bind(callback, base::Passed(&new_store))); + new_store->has_valid_data_ = true; + RecordApplyUpdateTime(metric, TimeTicks::Now() - before, store_path_); } else { - DVLOG(1) << "Failure: ApplyUpdate: reason: " << apply_update_result - << "; store: " << *this; - // new_store failed updating. Pass a nullptr to the callback. - callback_task_runner->PostTask(FROM_HERE, base::Bind(callback, nullptr)); + new_store.reset(); + DLOG(WARNING) << "Failure: ApplyUpdate: reason: " << apply_update_result + << "; store: " << *this; } - RecordApplyUpdateResult(apply_update_result); + RecordApplyUpdateResult(metric, apply_update_result, store_path_); + + // Posting the task should be the last thing to do in this function. + // Otherwise, the posted task can end up running in parallel. If that + // happens, the old store will get destoyed and can lead to use-after-free in + // this function. + callback_task_runner->PostTask( + FROM_HERE, base::Bind(callback, base::Passed(&new_store))); } -// static ApplyUpdateResult V4Store::UpdateHashPrefixMapFromAdditions( + const std::string& metric, const RepeatedPtrField<ThreatEntrySet>& additions, HashPrefixMap* additions_map) { for (const auto& addition : additions) { @@ -335,11 +417,12 @@ ApplyUpdateResult V4Store::UpdateHashPrefixMapFromAdditions( V4DecodeResult decode_result = V4RiceDecoder::DecodePrefixes( rice_hashes.first_value(), rice_hashes.rice_parameter(), rice_hashes.num_entries(), rice_hashes.encoded_data(), &raw_hashes); - RecordDecodeAdditionsResult(decode_result); + RecordDecodeAdditionsResult(metric, decode_result, store_path_); if (decode_result != DECODE_SUCCESS) { return RICE_DECODING_FAILURE; } else { - RecordDecodeAdditionsTime(TimeTicks::Now() - before, store_path_); + RecordDecodeAdditionsTime(metric, TimeTicks::Now() - before, + store_path_); char* raw_hashes_start = reinterpret_cast<char*>(raw_hashes.data()); size_t raw_hashes_size = sizeof(uint32_t) * raw_hashes.size(); @@ -450,7 +533,15 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, const HashPrefixMap& additions_map, const RepeatedField<int32>* raw_removals, const std::string& expected_checksum) { + DCHECK(task_runner_->RunsTasksOnCurrentThread()); DCHECK(hash_prefix_map_.empty()); + + bool calculate_checksum = !expected_checksum.empty(); + if (calculate_checksum && + (expected_checksum.size() != crypto::kSHA256Length)) { + return CHECKSUM_MISMATCH_FAILURE; + } + hash_prefix_map_.clear(); ReserveSpaceInPrefixMap(old_prefixes_map, &hash_prefix_map_); ReserveSpaceInPrefixMap(additions_map, &hash_prefix_map_); @@ -472,7 +563,6 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, // At least one of the maps still has elements that need to be merged into the // new store. - bool calculate_checksum = !expected_checksum.empty(); std::unique_ptr<crypto::SecureHash> checksum_ctx( crypto::SecureHash::Create(crypto::SecureHash::SHA256)); @@ -502,7 +592,7 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, next_smallest_prefix_size = next_smallest_prefix_old.size(); // Update the iterator map, which means that we have merged one hash - // prefix of size |next_size_for_old| from the old store. + // prefix of size |next_smallest_prefix_size| from the old store. old_iterator_map[next_smallest_prefix_size] += next_smallest_prefix_size; if (!raw_removals || removals_iter == raw_removals->end() || @@ -511,7 +601,7 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, hash_prefix_map_[next_smallest_prefix_size] += next_smallest_prefix_old; if (calculate_checksum) { - checksum_ctx->Update(base::string_as_array(&next_smallest_prefix_old), + checksum_ctx->Update(next_smallest_prefix_old.data(), next_smallest_prefix_size); } } else { @@ -532,9 +622,8 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, next_smallest_prefix_additions; if (calculate_checksum) { - checksum_ctx->Update( - base::string_as_array(&next_smallest_prefix_additions), - next_smallest_prefix_size); + checksum_ctx->Update(next_smallest_prefix_additions.data(), + next_smallest_prefix_size); } // Update the iterator map, which means that we have merged one hash @@ -554,15 +643,21 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, } if (calculate_checksum) { - std::string checksum(crypto::kSHA256Length, 0); - checksum_ctx->Finish(base::string_as_array(&checksum), checksum.size()); - if (checksum != expected_checksum) { - std::string checksum_base64, expected_checksum_base64; - base::Base64Encode(checksum, &checksum_base64); - base::Base64Encode(expected_checksum, &expected_checksum_base64); - DVLOG(1) << "Failure: Checksum mismatch: calculated: " << checksum_base64 - << " expected: " << expected_checksum_base64; - return CHECKSUM_MISMATCH_FAILURE; + char checksum[crypto::kSHA256Length]; + checksum_ctx->Finish(checksum, sizeof(checksum)); + for (size_t i = 0; i < crypto::kSHA256Length; i++) { + if (checksum[i] != expected_checksum[i]) { +#if DCHECK_IS_ON() + std::string checksum_b64, expected_checksum_b64; + base::Base64Encode(base::StringPiece(checksum, arraysize(checksum)), + &checksum_b64); + base::Base64Encode(expected_checksum, &expected_checksum_b64); + DVLOG(1) << "Failure: Checksum mismatch: calculated: " << checksum_b64 + << "; expected: " << expected_checksum_b64 + << "; store: " << *this; +#endif + return CHECKSUM_MISMATCH_FAILURE; + } } } @@ -572,20 +667,26 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, StoreReadResult V4Store::ReadFromDisk() { DCHECK(task_runner_->RunsTasksOnCurrentThread()); + V4StoreFileFormat file_format; + int64_t file_size; TimeTicks before = TimeTicks::Now(); - std::string contents; - bool read_success = base::ReadFileToString(store_path_, &contents); - if (!read_success) { - return FILE_UNREADABLE_FAILURE; - } + { + // A temporary scope to make sure that |contents| get destroyed as soon as + // we are doing using it. + std::string contents; + bool read_success = base::ReadFileToString(store_path_, &contents); + if (!read_success) { + return FILE_UNREADABLE_FAILURE; + } - if (contents.empty()) { - return FILE_EMPTY_FAILURE; - } + if (contents.empty()) { + return FILE_EMPTY_FAILURE; + } - V4StoreFileFormat file_format; - if (!file_format.ParseFromString(contents)) { - return PROTO_PARSING_FAILURE; + if (!file_format.ParseFromString(contents)) { + return PROTO_PARSING_FAILURE; + } + file_size = static_cast<int64_t>(contents.size()); } if (file_format.magic_number() != kFileMagic) { @@ -604,39 +705,42 @@ StoreReadResult V4Store::ReadFromDisk() { std::unique_ptr<ListUpdateResponse> response(new ListUpdateResponse); response->Swap(file_format.mutable_list_update_response()); - ApplyUpdateResult apply_update_result = ProcessFullUpdate(response); - RecordApplyUpdateResultWhenReadingFromDisk(apply_update_result); + ApplyUpdateResult apply_update_result = ProcessFullUpdate( + kReadFromDisk, response, true /* delay_checksum check */); + RecordApplyUpdateResult(kReadFromDisk, apply_update_result, store_path_); if (apply_update_result != APPLY_UPDATE_SUCCESS) { hash_prefix_map_.clear(); return HASH_PREFIX_MAP_GENERATION_FAILURE; } - RecordReadFromDiskTime(TimeTicks::Now() - before, store_path_); + RecordApplyUpdateTime(kReadFromDisk, TimeTicks::Now() - before, store_path_); + + // Update |file_size_| now because we parsed the file correctly. + file_size_ = file_size; return READ_SUCCESS; } -StoreWriteResult V4Store::WriteToDisk( - std::unique_ptr<ListUpdateResponse> response) const { - // Do not write partial updates to the disk. - // After merging the updates, the ListUpdateResponse passed to this method - // should be a FULL_UPDATE. - if (!response->has_response_type() || - response->response_type() != ListUpdateResponse::FULL_UPDATE) { - DVLOG(1) << "Failure: response->has_response_type(): " - << response->has_response_type() - << " : response->response_type(): " << response->response_type(); - return INVALID_RESPONSE_TYPE_FAILURE; +StoreWriteResult V4Store::WriteToDisk(const Checksum& checksum) { + V4StoreFileFormat file_format; + ListUpdateResponse* lur = file_format.mutable_list_update_response(); + *(lur->mutable_checksum()) = checksum; + lur->set_new_client_state(state_); + lur->set_response_type(ListUpdateResponse::FULL_UPDATE); + for (auto map_iter : hash_prefix_map_) { + ThreatEntrySet* additions = lur->add_additions(); + // TODO(vakh): Write RICE encoded hash prefixes on disk. Not doing so + // currently since it takes a long time to decode them on startup, which + // blocks resource load. See: http://crbug.com/654819 + additions->set_compression_type(RAW); + additions->mutable_raw_hashes()->set_prefix_size(map_iter.first); + additions->mutable_raw_hashes()->set_raw_hashes(map_iter.second); } // Attempt writing to a temporary file first and at the end, swap the files. const base::FilePath new_filename = TemporaryFileForFilename(store_path_); - V4StoreFileFormat file_format; file_format.set_magic_number(kFileMagic); file_format.set_version_number(kFileVersion); - ListUpdateResponse* response_to_write = - file_format.mutable_list_update_response(); - response_to_write->Swap(response.get()); std::string file_format_string; file_format.SerializeToString(&file_format_string); size_t written = base::WriteFile(new_filename, file_format_string.data(), @@ -647,6 +751,9 @@ StoreWriteResult V4Store::WriteToDisk( return UNABLE_TO_RENAME_FAILURE; } + // Update |file_size_| now because we wrote the file correctly. + file_size_ = static_cast<int64_t>(written); + return WRITE_SUCCESS; } @@ -654,7 +761,7 @@ HashPrefix V4Store::GetMatchingHashPrefix(const FullHash& full_hash) { // It should never be the case that more than one hash prefixes match a given // full hash. However, if that happens, this method returns any one of them. // It does not guarantee which one of those will be returned. - DCHECK_EQ(32u, full_hash.size()); + DCHECK(full_hash.size() == 32u || full_hash.size() == 21u); for (const auto& pair : hash_prefix_map_) { const PrefixSize& prefix_size = pair.first; const HashPrefixes& hash_prefixes = pair.second; @@ -690,4 +797,71 @@ bool V4Store::HashPrefixMatches(const HashPrefix& hash_prefix, } } +bool V4Store::VerifyChecksum() { + DCHECK(task_runner_->RunsTasksOnCurrentThread()); + + if (expected_checksum_.empty()) { + // Nothing to check here folks! + // TODO(vakh): Do not allow empty checksums. + return true; + } + + IteratorMap iterator_map; + HashPrefix next_smallest_prefix; + InitializeIteratorMap(hash_prefix_map_, &iterator_map); + bool has_unmerged = GetNextSmallestUnmergedPrefix( + hash_prefix_map_, iterator_map, &next_smallest_prefix); + + std::unique_ptr<crypto::SecureHash> checksum_ctx( + crypto::SecureHash::Create(crypto::SecureHash::SHA256)); + while (has_unmerged) { + PrefixSize next_smallest_prefix_size; + next_smallest_prefix_size = next_smallest_prefix.size(); + + // Update the iterator map, which means that we have read one hash + // prefix of size |next_smallest_prefix_size| from hash_prefix_map_. + iterator_map[next_smallest_prefix_size] += next_smallest_prefix_size; + + checksum_ctx->Update(next_smallest_prefix.data(), + next_smallest_prefix_size); + + // Find the next smallest unmerged element in the map. + has_unmerged = GetNextSmallestUnmergedPrefix(hash_prefix_map_, iterator_map, + &next_smallest_prefix); + } + + char checksum[crypto::kSHA256Length]; + checksum_ctx->Finish(checksum, sizeof(checksum)); + for (size_t i = 0; i < crypto::kSHA256Length; i++) { + if (checksum[i] != expected_checksum_[i]) { + RecordApplyUpdateResult(kReadFromDisk, CHECKSUM_MISMATCH_FAILURE, + store_path_); +#if DCHECK_IS_ON() + std::string checksum_b64, expected_checksum_b64; + base::Base64Encode(base::StringPiece(checksum, arraysize(checksum)), + &checksum_b64); + base::Base64Encode(expected_checksum_, &expected_checksum_b64); + DVLOG(1) << "Failure: Checksum mismatch: calculated: " << checksum_b64 + << "; expected: " << expected_checksum_b64 + << "; store: " << *this; +#endif + return false; + } + } + return true; +} + +int64_t V4Store::RecordAndReturnFileSize(const std::string& base_metric) { + std::string suffix = GetUmaSuffixForStore(store_path_); + // Histogram properties as in UMA_HISTOGRAM_COUNTS macro. + base::HistogramBase* histogram = base::Histogram::FactoryGet( + base_metric + suffix, 1, 1000000, 50, + base::HistogramBase::kUmaTargetedHistogramFlag); + if (histogram) { + const int64_t file_size_kilobytes = file_size_ / 1024; + histogram->Add(file_size_kilobytes); + } + return file_size_; +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_store.h b/chromium/components/safe_browsing_db/v4_store.h index f0172808c8f..0deb76a268c 100644 --- a/chromium/components/safe_browsing_db/v4_store.h +++ b/chromium/components/safe_browsing_db/v4_store.h @@ -15,7 +15,7 @@ namespace safe_browsing { class V4Store; -typedef base::Callback<void(std::unique_ptr<V4Store>)> +typedef base::Callback<void(std::unique_ptr<V4Store> new_store)> UpdatedStoreReadyCallback; // The sorted list of hash prefixes. @@ -158,8 +158,14 @@ class V4Store { // The |task_runner| is used to ensure that the operations in this file are // performed on the correct thread. |store_path| specifies the location on // disk for this file. The constructor doesn't read the store file from disk. + // If the store is being created to apply an update to the old store, then + // |old_file_size| is the size of the existing file on disk for this store; + // 0 otherwise. This is needed so that we can correctly report the size of + // store file on disk, even if writing the new file fails after successfully + // applying an update. V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner, - const base::FilePath& store_path); + const base::FilePath& store_path, + const int64_t old_file_size = 0); virtual ~V4Store(); const std::string& state() const { return state_; } @@ -167,8 +173,12 @@ class V4Store { const base::FilePath& store_path() const { return store_path_; } void ApplyUpdate(std::unique_ptr<ListUpdateResponse> response, - const scoped_refptr<base::SingleThreadTaskRunner>&, - UpdatedStoreReadyCallback); + const scoped_refptr<base::SingleThreadTaskRunner>& runner, + UpdatedStoreReadyCallback callback); + + // Records (in kilobytes) and returns the size of the file on disk for this + // store using |base_metric| as prefix and the filename as suffix. + int64_t RecordAndReturnFileSize(const std::string& base_metric); // If a hash prefix in this store matches |full_hash|, returns that hash // prefix; otherwise returns an empty hash prefix. @@ -176,12 +186,28 @@ class V4Store { std::string DebugString() const; + // Schedules the destruction of the V4Store object pointed to by |v4_store|, + // on the task runner. + static void Destroy(std::unique_ptr<V4Store> v4_store); + // Reads the store file from disk and populates the in-memory representation // of the hash prefixes. void Initialize(); - // Reset internal state and delete the backing file. - virtual bool Reset(); + // True if this store has valid contents, either from a successful read + // from disk or a full update. This does not mean the checksum was verified. + virtual bool HasValidData() const; + + // Reset internal state. + void Reset(); + + // Scheduled after reading the store file from disk on startup. When run, it + // ensures that the checksum of the hash prefixes in lexicographical sorted + // order matches the expected value in |expected_checksum_|. Returns true if + // it matches; false otherwise. Checksum verification can take a long time, + // so it is performed outside of the hotpath of loading SafeBrowsing database, + // which blocks resource loads. + bool VerifyChecksum(); private: FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestReadFromEmptyFile); @@ -241,11 +267,15 @@ class V4Store { TestHashPrefixExistsInMapWithDifferentSizes); FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestHashPrefixDoesNotExistInMapWithDifferentSizes); + FRIEND_TEST_ALL_PREFIXES(V4StoreTest, GetMatchingHashPrefixSize32Or21); FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestAdditionsWithRiceEncodingFailsWithInvalidInput); FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestAdditionsWithRiceEncodingSucceeds); FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds); FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestMergeUpdatesFailsChecksum); + FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestChecksumErrorOnStartup); + FRIEND_TEST_ALL_PREFIXES(V4StoreTest, WriteToDiskFails); + FRIEND_TEST_ALL_PREFIXES(V4StoreTest, FullUpdateFailsChecksumSynchronously); friend class V4StoreTest; // If |prefix_size| is within expected range, and |raw_hashes_length| is a @@ -293,57 +323,84 @@ class V4Store { // Merges the prefix map from the old store (|old_hash_prefix_map|) and the // update (additions_map) to populate the prefix map for the current store. // The indices in the |raw_removals| list, which may be NULL, are not merged. - // The SHA256 checksum of the final list of hash prefixes, in lexographically - // sorted order, must match |expected_checksum| (if it's not empty). - ApplyUpdateResult MergeUpdate(const HashPrefixMap& old_hash_prefix_map, - const HashPrefixMap& additions_map, - const ::google::protobuf::RepeatedField< - ::google::protobuf::int32>* raw_removals, - const std::string& expected_checksum); + // The SHA256 checksum of the final list of hash prefixes, in + // lexicographically sorted order, must match |expected_checksum| (if it's not + // empty). + ApplyUpdateResult MergeUpdate( + const HashPrefixMap& old_hash_prefix_map, + const HashPrefixMap& additions_map, + const ::google::protobuf::RepeatedField<::google::protobuf::int32>* + raw_removals, + const std::string& expected_checksum); // Processes the FULL_UPDATE |response| from the server, and writes the // merged V4Store to disk. If processing the |response| succeeds, it returns - // APPLY_UPDATE_SUCCESS. + // APPLY_UPDATE_SUCCESS. The UMA metrics for all interesting sub-operations + // use the prefix |metric|. // This method is only called when we receive a FULL_UPDATE from the server. ApplyUpdateResult ProcessFullUpdateAndWriteToDisk( + const std::string& metric, std::unique_ptr<ListUpdateResponse> response); // Processes a FULL_UPDATE |response| and updates the V4Store. If processing // the |response| succeeds, it returns APPLY_UPDATE_SUCCESS. // This method is called when we receive a FULL_UPDATE from the server, and - // when we read a store file from disk on startup. + // when we read a store file from disk on startup. The UMA metrics for all + // interesting sub-operations use the prefix |metric|. Delays the checksum + // check if |delay_checksum_check| is true. ApplyUpdateResult ProcessFullUpdate( - const std::unique_ptr<ListUpdateResponse>& response); + const std::string& metric, + const std::unique_ptr<ListUpdateResponse>& response, + bool delay_checksum_check); // Merges the hash prefixes in |hash_prefix_map_old| and |response|, updates // the |hash_prefix_map_| and |state_| in the V4Store, and writes the merged // store to disk. If processing succeeds, it returns APPLY_UPDATE_SUCCESS. // This method is only called when we receive a PARTIAL_UPDATE from the - // server. + // server. The UMA metrics for all interesting sub-operations use the prefix + // |metric|. ApplyUpdateResult ProcessPartialUpdateAndWriteToDisk( + const std::string& metric, const HashPrefixMap& hash_prefix_map_old, std::unique_ptr<ListUpdateResponse> response); // Merges the hash prefixes in |hash_prefix_map_old| and |response|, and // updates the |hash_prefix_map_| and |state_| in the V4Store. If processing - // succeeds, it returns APPLY_UPDATE_SUCCESS. + // succeeds, it returns APPLY_UPDATE_SUCCESS. The UMA metrics for all + // interesting sub-operations use the prefix |metric|. Delays the checksum + // check if |delay_checksum_check| is true. ApplyUpdateResult ProcessUpdate( + const std::string& metric, const HashPrefixMap& hash_prefix_map_old, - const std::unique_ptr<ListUpdateResponse>& response); + const std::unique_ptr<ListUpdateResponse>& response, + bool delay_checksum_check); // Reads the state of the store from the file on disk and returns the reason // for the failure or reports success. StoreReadResult ReadFromDisk(); // Updates the |additions_map| with the additions received in the partial - // update from the server. + // update from the server. The UMA metrics for all interesting sub-operations + // use the prefix |metric|. ApplyUpdateResult UpdateHashPrefixMapFromAdditions( + const std::string& metric, const ::google::protobuf::RepeatedPtrField<ThreatEntrySet>& additions, HashPrefixMap* additions_map); - // Writes the FULL_UPDATE |response| to disk as a V4StoreFileFormat proto. - StoreWriteResult WriteToDisk( - std::unique_ptr<ListUpdateResponse> response) const; + // Writes the hash_prefix_map_ to disk as a V4StoreFileFormat proto. + // |checksum| is used to set the |checksum| field in the final proto. + StoreWriteResult WriteToDisk(const Checksum& checksum); + + // The checksum value as read from the disk, until it is verified. Once + // verified, it is cleared. + std::string expected_checksum_; + + // The size of the file on disk for this store. + int64_t file_size_; + + // True if the file was successfully read+parsed or was populated from + // a full update. + bool has_valid_data_; // The state of the store as returned by the PVer4 server in the last applied // update response. diff --git a/chromium/components/safe_browsing_db/v4_store_unittest.cc b/chromium/components/safe_browsing_db/v4_store_unittest.cc index 90c6525fa88..2d2e3d76720 100644 --- a/chromium/components/safe_browsing_db/v4_store_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_store_unittest.cc @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "base/base64.h" #include "base/bind.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" @@ -56,25 +57,35 @@ class V4StoreTest : public PlatformTest { file_format_string.size()); } - void UpdatedStoreReadyAfterRiceRemovals(bool* called_back, - std::unique_ptr<V4Store> new_store) { + void UpdatedStoreReady(bool* called_back, + bool expect_store, + std::unique_ptr<V4Store> store) { *called_back = true; - EXPECT_EQ(2u, new_store->hash_prefix_map_.size()); - EXPECT_EQ("22222", new_store->hash_prefix_map_[5]); - EXPECT_EQ("abcd", new_store->hash_prefix_map_[4]); + if (expect_store) { + ASSERT_TRUE(store); + EXPECT_EQ(2u, store->hash_prefix_map_.size()); + EXPECT_EQ("22222", store->hash_prefix_map_[5]); + EXPECT_EQ("abcd", store->hash_prefix_map_[4]); + } else { + ASSERT_FALSE(store); + } + + updated_store_ = std::move(store); } base::ScopedTempDir temp_dir_; base::FilePath store_path_; scoped_refptr<base::TestSimpleTaskRunner> task_runner_; content::TestBrowserThreadBundle thread_bundle_; + std::unique_ptr<V4Store> updated_store_; }; TEST_F(V4StoreTest, TestReadFromEmptyFile) { base::CloseFile(base::OpenFile(store_path_, "wb+")); - EXPECT_EQ(FILE_EMPTY_FAILURE, - V4Store(task_runner_, store_path_).ReadFromDisk()); + V4Store store(task_runner_, store_path_); + EXPECT_EQ(FILE_EMPTY_FAILURE, store.ReadFromDisk()); + EXPECT_FALSE(store.HasValidData()); } TEST_F(V4StoreTest, TestReadFromAbsentFile) { @@ -125,36 +136,11 @@ TEST_F(V4StoreTest, TestReadFromNoHashPrefixesFile) { list_update_response.set_platform_type(LINUX_PLATFORM); list_update_response.set_response_type(ListUpdateResponse::FULL_UPDATE); WriteFileFormatProtoToFile(0x600D71FE, 9, &list_update_response); - EXPECT_EQ(READ_SUCCESS, V4Store(task_runner_, store_path_).ReadFromDisk()); -} - -TEST_F(V4StoreTest, TestWriteNoResponseType) { - EXPECT_EQ(INVALID_RESPONSE_TYPE_FAILURE, - V4Store(task_runner_, store_path_) - .WriteToDisk(base::WrapUnique(new ListUpdateResponse))); -} - -TEST_F(V4StoreTest, TestWritePartialResponseType) { - std::unique_ptr<ListUpdateResponse> list_update_response( - new ListUpdateResponse); - list_update_response->set_response_type(ListUpdateResponse::PARTIAL_UPDATE); - EXPECT_EQ(INVALID_RESPONSE_TYPE_FAILURE, - V4Store(task_runner_, store_path_) - .WriteToDisk(std::move(list_update_response))); -} - -TEST_F(V4StoreTest, TestWriteFullResponseType) { - std::unique_ptr<ListUpdateResponse> list_update_response( - new ListUpdateResponse); - list_update_response->set_response_type(ListUpdateResponse::FULL_UPDATE); - list_update_response->set_new_client_state("test_client_state"); - EXPECT_EQ(WRITE_SUCCESS, V4Store(task_runner_, store_path_) - .WriteToDisk(std::move(list_update_response))); - - V4Store read_store(task_runner_, store_path_); - EXPECT_EQ(READ_SUCCESS, read_store.ReadFromDisk()); - EXPECT_EQ("test_client_state", read_store.state_); - EXPECT_TRUE(read_store.hash_prefix_map_.empty()); + V4Store store(task_runner_, store_path_); + EXPECT_EQ(READ_SUCCESS, store.ReadFromDisk()); + EXPECT_TRUE(store.hash_prefix_map_.empty()); + EXPECT_EQ(14, store.file_size_); + EXPECT_FALSE(store.HasValidData()); } TEST_F(V4StoreTest, TestAddUnlumpedHashesWithInvalidAddition) { @@ -568,22 +554,13 @@ TEST_F(V4StoreTest, TestMergeUpdatesRemovesMultipleAcrossDifferentSizes) { } TEST_F(V4StoreTest, TestReadFullResponseWithValidHashPrefixMap) { - std::unique_ptr<ListUpdateResponse> lur(new ListUpdateResponse); - lur->set_response_type(ListUpdateResponse::FULL_UPDATE); - lur->set_new_client_state("test_client_state"); - lur->set_platform_type(WINDOWS_PLATFORM); - lur->set_threat_entry_type(URL); - lur->set_threat_type(MALWARE_THREAT); - ThreatEntrySet* additions = lur->add_additions(); - additions->set_compression_type(RAW); - additions->mutable_raw_hashes()->set_prefix_size(5); - additions->mutable_raw_hashes()->set_raw_hashes("00000abcde"); - additions = lur->add_additions(); - additions->set_compression_type(RAW); - additions->mutable_raw_hashes()->set_prefix_size(4); - additions->mutable_raw_hashes()->set_raw_hashes("00000abc"); - EXPECT_EQ(WRITE_SUCCESS, - V4Store(task_runner_, store_path_).WriteToDisk(std::move(lur))); + V4Store write_store(task_runner_, store_path_); + write_store.hash_prefix_map_[4] = "00000abc"; + write_store.hash_prefix_map_[5] = "00000abcde"; + write_store.state_ = "test_client_state"; + EXPECT_FALSE(base::PathExists(write_store.store_path_)); + EXPECT_EQ(WRITE_SUCCESS, write_store.WriteToDisk(Checksum())); + EXPECT_TRUE(base::PathExists(write_store.store_path_)); V4Store read_store (task_runner_, store_path_); EXPECT_EQ(READ_SUCCESS, read_store.ReadFromDisk()); @@ -591,6 +568,7 @@ TEST_F(V4StoreTest, TestReadFullResponseWithValidHashPrefixMap) { ASSERT_EQ(2u, read_store.hash_prefix_map_.size()); EXPECT_EQ("00000abc", read_store.hash_prefix_map_[4]); EXPECT_EQ("00000abcde", read_store.hash_prefix_map_[5]); + EXPECT_EQ(71, read_store.file_size_); } // This tests fails to read the prefix map from the disk because the file on @@ -598,23 +576,18 @@ TEST_F(V4StoreTest, TestReadFullResponseWithValidHashPrefixMap) { // size is 5 so the parser isn't able to split the hash prefixes list // completely. TEST_F(V4StoreTest, TestReadFullResponseWithInvalidHashPrefixMap) { - std::unique_ptr<ListUpdateResponse> lur(new ListUpdateResponse); - lur->set_response_type(ListUpdateResponse::FULL_UPDATE); - lur->set_new_client_state("test_client_state"); - lur->set_platform_type(WINDOWS_PLATFORM); - lur->set_threat_entry_type(URL); - lur->set_threat_type(MALWARE_THREAT); - ThreatEntrySet* additions = lur->add_additions(); - additions->set_compression_type(RAW); - additions->mutable_raw_hashes()->set_prefix_size(5); - additions->mutable_raw_hashes()->set_raw_hashes("abcdef"); - EXPECT_EQ(WRITE_SUCCESS, - V4Store(task_runner_, store_path_).WriteToDisk(std::move(lur))); + V4Store write_store(task_runner_, store_path_); + write_store.hash_prefix_map_[5] = "abcdef"; + write_store.state_ = "test_client_state"; + EXPECT_FALSE(base::PathExists(write_store.store_path_)); + EXPECT_EQ(WRITE_SUCCESS, write_store.WriteToDisk(Checksum())); + EXPECT_TRUE(base::PathExists(write_store.store_path_)); V4Store read_store(task_runner_, store_path_); EXPECT_EQ(HASH_PREFIX_MAP_GENERATION_FAILURE, read_store.ReadFromDisk()); EXPECT_TRUE(read_store.state_.empty()); EXPECT_TRUE(read_store.hash_prefix_map_.empty()); + EXPECT_EQ(0, read_store.file_size_); } TEST_F(V4StoreTest, TestHashPrefixExistsAtTheBeginning) { @@ -700,6 +673,22 @@ TEST_F(V4StoreTest, TestHashPrefixDoesNotExistInMapWithDifferentSizes) { EXPECT_TRUE(store.GetMatchingHashPrefix(full_hash).empty()); } +TEST_F(V4StoreTest, GetMatchingHashPrefixSize32Or21) { + HashPrefix prefix = "0123"; + V4Store store(task_runner_, store_path_); + store.hash_prefix_map_[4] = prefix; + + FullHash full_hash_21 = "0123456789ABCDEF01234"; + EXPECT_EQ(prefix, store.GetMatchingHashPrefix(full_hash_21)); + FullHash full_hash_32 = "0123456789ABCDEF0123456789ABCDEF"; + EXPECT_EQ(prefix, store.GetMatchingHashPrefix(full_hash_32)); +#if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON) + // This hits a DCHECK so it is release mode only. + FullHash full_hash_22 = "0123456789ABCDEF012345"; + EXPECT_EQ(prefix, store.GetMatchingHashPrefix(full_hash_22)); +#endif +} + #if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON) // This test hits a NOTREACHED so it is a release mode only test. TEST_F(V4StoreTest, TestAdditionsWithRiceEncodingFailsWithInvalidInput) { @@ -710,7 +699,8 @@ TEST_F(V4StoreTest, TestAdditionsWithRiceEncodingFailsWithInvalidInput) { HashPrefixMap additions_map; EXPECT_EQ(RICE_DECODING_FAILURE, V4Store(task_runner_, store_path_) - .UpdateHashPrefixMapFromAdditions(additions, &additions_map)); + .UpdateHashPrefixMapFromAdditions("V4Metric", additions, + &additions_map)); } #endif @@ -731,7 +721,8 @@ TEST_F(V4StoreTest, TestAdditionsWithRiceEncodingSucceeds) { HashPrefixMap additions_map; EXPECT_EQ(APPLY_UPDATE_SUCCESS, V4Store(task_runner_, store_path_) - .UpdateHashPrefixMapFromAdditions(additions, &additions_map)); + .UpdateHashPrefixMapFromAdditions("V4Metric", additions, + &additions_map)); EXPECT_EQ(1u, additions_map.size()); EXPECT_EQ(std::string("\x5\0\0\0\fL\x93\xADV\x7F\xF6o\xCEo1\x81", 16), additions_map[4]); @@ -754,6 +745,7 @@ TEST_F(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds) { EXPECT_EQ(APPLY_UPDATE_SUCCESS, store.MergeUpdate(prefix_map_old, prefix_map_additions, nullptr, expected_checksum)); + EXPECT_FALSE(store.HasValidData()); // Never actually read from disk. // At this point, the store map looks like this: // 4: 1111abcdefgh @@ -773,14 +765,20 @@ TEST_F(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds) { bool called_back = false; UpdatedStoreReadyCallback store_ready_callback = - base::Bind(&V4StoreTest::UpdatedStoreReadyAfterRiceRemovals, - base::Unretained(this), &called_back); + base::Bind(&V4StoreTest::UpdatedStoreReady, base::Unretained(this), + &called_back, true /* expect_store */); + EXPECT_FALSE(base::PathExists(store.store_path_)); store.ApplyUpdate(std::move(lur), task_runner_, store_ready_callback); + EXPECT_TRUE(base::PathExists(store.store_path_)); + task_runner_->RunPendingTasks(); base::RunLoop().RunUntilIdle(); // This ensures that the callback was called. EXPECT_TRUE(called_back); + // ApplyUpdate was successful, so we have valid data. + ASSERT_TRUE(updated_store_); + EXPECT_TRUE(updated_store_->HasValidData()); } TEST_F(V4StoreTest, TestMergeUpdatesFailsChecksum) { @@ -800,4 +798,86 @@ TEST_F(V4StoreTest, TestMergeUpdatesFailsChecksum) { .MergeUpdate(prefix_map_old, HashPrefixMap(), nullptr, "aawc")); } +TEST_F(V4StoreTest, TestChecksumErrorOnStartup) { + // First the case of checksum not matching after reading from disk. + ListUpdateResponse list_update_response; + list_update_response.set_new_client_state("test_client_state"); + list_update_response.set_platform_type(LINUX_PLATFORM); + list_update_response.set_response_type(ListUpdateResponse::FULL_UPDATE); + list_update_response.mutable_checksum()->set_sha256( + std::string(crypto::kSHA256Length, 0)); + WriteFileFormatProtoToFile(0x600D71FE, 9, &list_update_response); + V4Store store(task_runner_, store_path_); + EXPECT_TRUE(store.expected_checksum_.empty()); + EXPECT_EQ(READ_SUCCESS, store.ReadFromDisk()); + EXPECT_TRUE(!store.expected_checksum_.empty()); + EXPECT_EQ(69, store.file_size_); + EXPECT_EQ("test_client_state", store.state()); + + EXPECT_FALSE(store.VerifyChecksum()); + + // Now the case of checksum matching after reading from disk. + // Proof of checksum mismatch using python: + // >>> import hashlib + // >>> m = hashlib.sha256() + // >>> m.update("abcde") + // >>> import base64 + // >>> encoded = base64.b64encode(m.digest()) + // >>> encoded + // 'NrvlDtloQdEEQ7y2cNZVTwo0t2G+Z+ycSorSwMRMpCw=' + std::string expected_checksum; + base::Base64Decode("NrvlDtloQdEEQ7y2cNZVTwo0t2G+Z+ycSorSwMRMpCw=", + &expected_checksum); + ThreatEntrySet* additions = list_update_response.add_additions(); + additions->set_compression_type(RAW); + additions->mutable_raw_hashes()->set_prefix_size(5); + additions->mutable_raw_hashes()->set_raw_hashes("abcde"); + list_update_response.mutable_checksum()->set_sha256(expected_checksum); + WriteFileFormatProtoToFile(0x600D71FE, 9, &list_update_response); + V4Store another_store(task_runner_, store_path_); + EXPECT_TRUE(another_store.expected_checksum_.empty()); + + EXPECT_EQ(READ_SUCCESS, another_store.ReadFromDisk()); + EXPECT_TRUE(!another_store.expected_checksum_.empty()); + EXPECT_EQ("test_client_state", another_store.state()); + EXPECT_EQ(69, store.file_size_); + + EXPECT_TRUE(another_store.VerifyChecksum()); +} + +TEST_F(V4StoreTest, WriteToDiskFails) { + // Pass the directory name as file name so that when the code tries to rename + // the temp store file to |store_path_| it fails. + EXPECT_EQ(UNABLE_TO_RENAME_FAILURE, + V4Store(task_runner_, temp_dir_.GetPath()).WriteToDisk(Checksum())); +} + +TEST_F(V4StoreTest, FullUpdateFailsChecksumSynchronously) { + V4Store store(task_runner_, store_path_); + bool called_back = false; + UpdatedStoreReadyCallback store_ready_callback = + base::Bind(&V4StoreTest::UpdatedStoreReady, base::Unretained(this), + &called_back, false /* expect_store */); + EXPECT_FALSE(base::PathExists(store.store_path_)); + EXPECT_FALSE(store.HasValidData()); // Never actually read from disk. + + // Now create a response with invalid checksum. + std::unique_ptr<ListUpdateResponse> lur(new ListUpdateResponse); + lur->set_response_type(ListUpdateResponse::FULL_UPDATE); + lur->mutable_checksum()->set_sha256(std::string(crypto::kSHA256Length, 0)); + store.ApplyUpdate(std::move(lur), task_runner_, store_ready_callback); + // The update should fail synchronously and not create a store file. + EXPECT_FALSE(base::PathExists(store.store_path_)); + + // Run everything on the task runner to ensure there are no pending tasks. + task_runner_->RunPendingTasks(); + base::RunLoop().RunUntilIdle(); + + // This ensures that the callback was called. + EXPECT_TRUE(called_back); + // Ensure that the file is still not created. + EXPECT_FALSE(base::PathExists(store.store_path_)); + EXPECT_FALSE(updated_store_); +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_test_util.cc b/chromium/components/safe_browsing_db/v4_test_util.cc new file mode 100644 index 00000000000..a37279569e2 --- /dev/null +++ b/chromium/components/safe_browsing_db/v4_test_util.cc @@ -0,0 +1,30 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_test_util.h" + +namespace safe_browsing { + +namespace { + +const char kClient[] = "unittest"; +const char kAppVer[] = "1.0"; +const char kKeyParam[] = "test_key_param"; + +} // namespace + +V4ProtocolConfig GetTestV4ProtocolConfig(bool disable_auto_update) { + return V4ProtocolConfig(kClient, disable_auto_update, kKeyParam, kAppVer); +} + +std::ostream& operator<<(std::ostream& os, const ThreatMetadata& meta) { + os << "{threat_pattern_type=" << static_cast<int>(meta.threat_pattern_type) + << ", api_permissions=["; + for (auto p : meta.api_permissions) + os << p << ","; + return os << "], population_id=" << meta.population_id << "}"; +} + +} // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_test_util.h b/chromium/components/safe_browsing_db/v4_test_util.h new file mode 100644 index 00000000000..d21170cecf7 --- /dev/null +++ b/chromium/components/safe_browsing_db/v4_test_util.h @@ -0,0 +1,23 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_SAFE_BROWSING_DB_V4_TEST_UTIL_H_ +#define COMPONENTS_SAFE_BROWSING_DB_V4_TEST_UTIL_H_ + +// Contains methods useful for tests. + +#include <ostream> + +namespace safe_browsing { + +struct ThreatMetadata; +struct V4ProtocolConfig; + +V4ProtocolConfig GetTestV4ProtocolConfig(bool disable_auto_update = false); + +std::ostream& operator<<(std::ostream& os, const ThreatMetadata& meta); + +} // namespace safe_browsing + +#endif // COMPONENTS_SAFE_BROWSING_DB_V4_TEST_UTIL_H_ diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc b/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc index ae43b2e3b40..47736aa73bf 100644 --- a/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc +++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc @@ -50,13 +50,13 @@ enum ParseResultType { // Record parsing errors of an update result. void RecordParseUpdateResult(ParseResultType result_type) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.ParseV4UpdateResult", result_type, + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.V4Update.Parse.Result", result_type, PARSE_RESULT_TYPE_MAX); } void RecordUpdateResult(safe_browsing::V4OperationResult result) { UMA_HISTOGRAM_ENUMERATION( - "SafeBrowsing.V4UpdateResult", result, + "SafeBrowsing.V4Update.Result", result, safe_browsing::V4OperationResult::OPERATION_RESULT_MAX); } @@ -70,6 +70,9 @@ static const int kV4TimerStartIntervalSecMin = 60; // Maximum time, in seconds, from start up before we must issue an update query. static const int kV4TimerStartIntervalSecMax = 300; +// Maximum time, in seconds, to wait for a response to an update request. +static const int kV4TimerUpdateWaitSecMax = 30; + // The default V4UpdateProtocolManagerFactory. class V4UpdateProtocolManagerFactoryImpl : public V4UpdateProtocolManagerFactory { @@ -142,8 +145,6 @@ void V4UpdateProtocolManager::ScheduleNextUpdate( void V4UpdateProtocolManager::ScheduleNextUpdateWithBackoff(bool back_off) { DCHECK(CalledOnValidThread()); - // TODO(vakh): Set disable_auto_update correctly using the command line - // switch. if (config_.disable_auto_update) { DCHECK(!IsUpdateScheduled()); return; @@ -250,7 +251,6 @@ bool V4UpdateProtocolManager::ParseUpdateResponse( base::TimeDelta::FromSeconds(minimum_wait_duration_seconds); } - // TODO(vakh): Do something useful with this response. for (ListUpdateResponse& list_update_response : *response.mutable_list_update_responses()) { if (!list_update_response.has_platform_type()) { @@ -295,7 +295,17 @@ void V4UpdateProtocolManager::IssueUpdateRequest() { request_->SetLoadFlags(net::LOAD_DISABLE_CACHE); request_->SetRequestContext(request_context_getter_.get()); request_->Start(); - // TODO(vakh): Handle request timeout. + + // Begin the update request timeout. + timeout_timer_.Start(FROM_HERE, + TimeDelta::FromSeconds(kV4TimerUpdateWaitSecMax), this, + &V4UpdateProtocolManager::HandleTimeout); +} + +void V4UpdateProtocolManager::HandleTimeout() { + UMA_HISTOGRAM_BOOLEAN("SafeBrowsing.V4Update.TimedOut", true); + request_.reset(); + ScheduleNextUpdateWithBackoff(false); } // net::URLFetcherDelegate implementation ---------------------------------- @@ -305,10 +315,13 @@ void V4UpdateProtocolManager::OnURLFetchComplete( const net::URLFetcher* source) { DCHECK(CalledOnValidThread()); + timeout_timer_.Stop(); + int response_code = source->GetResponseCode(); net::URLRequestStatus status = source->GetStatus(); V4ProtocolManagerUtil::RecordHttpResponseOrErrorCode( - "SafeBrowsing.V4UpdateHttpResponseOrErrorCode", status, response_code); + "SafeBrowsing.V4Update.Network.Result", status, response_code); + UMA_HISTOGRAM_BOOLEAN("SafeBrowsing.V4Update.TimedOut", false); last_response_time_ = Time::Now(); @@ -325,7 +338,7 @@ void V4UpdateProtocolManager::OnURLFetchComplete( } request_.reset(); - UMA_HISTOGRAM_COUNTS("SafeBrowsing.V4UpdateResponseSizeKB", + UMA_HISTOGRAM_COUNTS("SafeBrowsing.V4Update.ResponseSizeKB", data.size() / 1024); // The caller should update its state now, based on parsed_server_response. diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager.h b/chromium/components/safe_browsing_db/v4_update_protocol_manager.h index 7da1667c66f..a3b4cd538e0 100644 --- a/chromium/components/safe_browsing_db/v4_update_protocol_manager.h +++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager.h @@ -87,6 +87,9 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate, TestGetUpdatesWithOneBackoff); FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, TestBase64EncodingUsesUrlEncoding); + FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, TestDisableAutoUpdates); + FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, + TestGetUpdatesHasTimeout); friend class V4UpdateProtocolManagerFactoryImpl; // Fills a FetchThreatListUpdatesRequest protocol buffer for a request. @@ -111,6 +114,10 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate, // Resets the update error counter and multiplier. void ResetUpdateErrors(); + // Called when update request times out. Cancels the existing request and + // re-sends the update request. + void HandleTimeout(); + // Updates internal update and backoff state for each update response error, // assuming that the current time is |now|. void HandleUpdateError(const base::Time& now); @@ -172,6 +179,10 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate, base::Time last_response_time_; + // Used to interrupt and re-schedule update requests that take too long to + // complete. + base::OneShotTimer timeout_timer_; + DISALLOW_COPY_AND_ASSIGN(V4UpdateProtocolManager); }; diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc index e8596e68e89..cdef7382501 100644 --- a/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc @@ -15,6 +15,7 @@ #include "base/time/time.h" #include "components/safe_browsing_db/safebrowsing.pb.h" #include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_test_util.h" #include "net/base/escape.h" #include "net/base/load_flags.h" #include "net/base/net_errors.h" @@ -25,14 +26,6 @@ using base::Time; using base::TimeDelta; -namespace { - -const char kClient[] = "unittest"; -const char kAppVer[] = "1.0"; -const char kKeyParam[] = "test_key_param"; - -} // namespace - namespace safe_browsing { class V4UpdateProtocolManagerTest : public PlatformTest { @@ -65,19 +58,11 @@ class V4UpdateProtocolManagerTest : public PlatformTest { } } - V4ProtocolConfig GetProtocolConfig() { - V4ProtocolConfig config; - config.client_name = kClient; - config.version = kAppVer; - config.key_param = kKeyParam; - config.disable_auto_update = false; - return config; - } - std::unique_ptr<V4UpdateProtocolManager> CreateProtocolManager( - const std::vector<ListUpdateResponse>& expected_lurs) { + const std::vector<ListUpdateResponse>& expected_lurs, + bool disable_auto_update = false) { return V4UpdateProtocolManager::Create( - NULL, GetProtocolConfig(), + NULL, GetTestV4ProtocolConfig(disable_auto_update), base::Bind(&V4UpdateProtocolManagerTest::ValidateGetUpdatesResults, base::Unretained(this), expected_lurs)); } @@ -163,8 +148,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesErrorHandlingNetwork) { EXPECT_FALSE(pm->IsUpdateScheduled()); - runner->RunPendingTasks(); - net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); DCHECK(fetcher); // Failed request status should result in error. @@ -197,8 +180,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesErrorHandlingResponseCode) { EXPECT_FALSE(pm->IsUpdateScheduled()); - runner->RunPendingTasks(); - net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); DCHECK(fetcher); fetcher->set_status(net::URLRequestStatus()); @@ -233,8 +214,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesNoError) { EXPECT_FALSE(pm->IsUpdateScheduled()); - runner->RunPendingTasks(); - net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); DCHECK(fetcher); fetcher->set_status(net::URLRequestStatus()); @@ -268,8 +247,6 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesWithOneBackoff) { EXPECT_FALSE(pm->IsUpdateScheduled()); - runner->RunPendingTasks(); - net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); DCHECK(fetcher); fetcher->set_status(net::URLRequestStatus()); @@ -285,6 +262,7 @@ TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesWithOneBackoff) { // Retry, now no backoff. expect_callback_to_be_called_ = true; + // Call RunPendingTasks to ensure that the request is sent after backoff. runner->RunPendingTasks(); fetcher = factory.GetFetcherByID(1); @@ -320,4 +298,64 @@ TEST_F(V4UpdateProtocolManagerTest, TestBase64EncodingUsesUrlEncoding) { // the '-' case is sufficient to prove that we are using URL encoding. } +TEST_F(V4UpdateProtocolManagerTest, TestDisableAutoUpdates) { + scoped_refptr<base::TestSimpleTaskRunner> runner( + new base::TestSimpleTaskRunner()); + base::ThreadTaskRunnerHandle runner_handler(runner); + net::TestURLFetcherFactory factory; + std::unique_ptr<V4UpdateProtocolManager> pm(CreateProtocolManager( + std::vector<ListUpdateResponse>(), true /* disable_auto_update */)); + + // Initial state. No errors. + pm->ScheduleNextUpdate(std::move(store_state_map_)); + EXPECT_FALSE(pm->IsUpdateScheduled()); + + runner->RunPendingTasks(); + EXPECT_FALSE(pm->IsUpdateScheduled()); + + net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); + DCHECK(!fetcher); +} + +TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesHasTimeout) { + scoped_refptr<base::TestSimpleTaskRunner> runner( + new base::TestSimpleTaskRunner()); + base::ThreadTaskRunnerHandle runner_handler(runner); + net::TestURLFetcherFactory factory; + std::vector<ListUpdateResponse> expected_lurs; + SetupExpectedListUpdateResponse(&expected_lurs); + std::unique_ptr<V4UpdateProtocolManager> pm( + CreateProtocolManager(expected_lurs)); + runner->ClearPendingTasks(); + + // Initial state. No errors. + EXPECT_EQ(0ul, pm->update_error_count_); + EXPECT_EQ(1ul, pm->update_back_off_mult_); + expect_callback_to_be_called_ = true; + pm->store_state_map_ = std::move(store_state_map_); + pm->IssueUpdateRequest(); + + net::TestURLFetcher* timeout_fetcher = factory.GetFetcherByID(0); + DCHECK(timeout_fetcher); + // Don't set anything on the fetcher. Let it time out. + runner->RunPendingTasks(); + + net::TestURLFetcher* fetcher = factory.GetFetcherByID(1); + DCHECK(!fetcher); + // Now wait for the next request to be scheduled. + runner->RunPendingTasks(); + + // There should be another fetcher now. + fetcher = factory.GetFetcherByID(1); + DCHECK(fetcher); + fetcher->set_status(net::URLRequestStatus()); + fetcher->set_response_code(net::HTTP_OK); + fetcher->SetResponseString(GetExpectedV4UpdateResponse(expected_lurs)); + fetcher->delegate()->OnURLFetchComplete(fetcher); + + // No error, back off multiplier is unchanged. + EXPECT_EQ(0ul, pm->update_error_count_); + EXPECT_EQ(1ul, pm->update_back_off_mult_); +} + } // namespace safe_browsing |