diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-09-18 14:34:04 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2017-10-04 11:15:27 +0000 |
commit | e6430e577f105ad8813c92e75c54660c4985026e (patch) | |
tree | 88115e5d1fb471fea807111924dcccbeadbf9e4f /chromium/components/safe_browsing_db | |
parent | 53d399fe6415a96ea6986ec0d402a9c07da72453 (diff) | |
download | qtwebengine-chromium-e6430e577f105ad8813c92e75c54660c4985026e.tar.gz |
BASELINE: Update Chromium to 61.0.3163.99
Change-Id: I8452f34574d88ca2b27af9bd56fc9ff3f16b1367
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
Diffstat (limited to 'chromium/components/safe_browsing_db')
33 files changed, 990 insertions, 187 deletions
diff --git a/chromium/components/safe_browsing_db/BUILD.gn b/chromium/components/safe_browsing_db/BUILD.gn index fb9286f8e4c..c3f376dcbeb 100644 --- a/chromium/components/safe_browsing_db/BUILD.gn +++ b/chromium/components/safe_browsing_db/BUILD.gn @@ -33,6 +33,7 @@ group("safe_browsing_db_shared") { ":prefix_set", ":safebrowsing_proto", ":util", + ":v4_feature_list", # Used by SafeBrowsingService "//components/safe_browsing/common:safe_browsing_prefs", ] } @@ -71,6 +72,10 @@ static_library("database_manager") { "//net", "//url", ] + + public_deps = [ + ":safebrowsing_proto", + ] } static_library("hit_report") { @@ -108,6 +113,7 @@ static_library("remote_database_manager") { ":database_manager", ":safe_browsing_api_handler", ":v4_get_hash_protocol_manager", + ":v4_protocol_manager_util", "//base:base", "//components/variations", "//content/public/browser", @@ -159,6 +165,7 @@ static_library("test_database_manager") { ] deps = [ ":database_manager", + ":v4_protocol_manager_util", "//base:base", "//net", ] @@ -210,6 +217,7 @@ static_library("v4_feature_list") { ] deps = [ "//base", + "//components/safe_browsing:features", ] } @@ -268,6 +276,15 @@ source_set("v4_protocol_manager_util") { ] } +if (is_android) { + import("//build/config/android/rules.gni") + java_cpp_enum("sb_threat_values") { + sources = [ + "v4_protocol_manager_util.h", + ] + } +} + source_set("v4_rice") { sources = [ "v4_rice.cc", @@ -369,6 +386,7 @@ source_set("v4_local_database_manager_unittest") { deps = [ ":v4_database", ":v4_local_database_manager", + ":v4_protocol_manager_util", ":v4_test_util", "//base", "//base/test:test_support", @@ -397,19 +415,15 @@ source_set("v4_update_protocol_manager_unittest") { ] } -source_set("unit_tests") { +source_set("unit_tests_shared") { testonly = true sources = [ "database_manager_unittest.cc", "prefix_set_unittest.cc", "util_unittest.cc", - "v4_database_unittest.cc", "v4_get_hash_protocol_manager_unittest.cc", - "v4_local_database_manager_unittest.cc", "v4_protocol_manager_util_unittest.cc", - "v4_rice_unittest.cc", - "v4_store_unittest.cc", - "v4_update_protocol_manager_unittest.cc", + "whitelist_checker_client_unittest.cc", ] deps = [ ":database_manager", @@ -417,8 +431,36 @@ source_set("unit_tests") { ":safebrowsing_proto", ":test_database_manager", ":util", - ":v4_database", ":v4_get_hash_protocol_manager", + ":v4_protocol_manager_util", + ":v4_test_util", + ":whitelist_checker_client", + "//base", + "//content/public/browser", + "//content/test:test_support", + "//net", + "//net:test_support", + "//testing/gtest", + ] + if (is_win) { + # TODO(jschuh): crbug.com/167187 fix size_t to int truncations. + cflags = [ "/wd4267" ] # Conversion from size_t to 'type'. + } +} + +source_set("unit_tests_desktop") { + testonly = true + sources = [ + "v4_database_unittest.cc", + "v4_local_database_manager_unittest.cc", + "v4_rice_unittest.cc", + "v4_store_unittest.cc", + "v4_update_protocol_manager_unittest.cc", + ] + deps = [ + ":unit_tests_shared", + ":util", + ":v4_database", ":v4_local_database_manager", ":v4_protocol_manager_util", ":v4_rice", @@ -453,10 +495,12 @@ source_set("unit_tests_mobile") { ":remote_database_manager", ":safe_browsing_api_handler", ":safe_browsing_api_handler_util", + ":unit_tests_shared", ":util", ":v4_test_util", "//base", "//components/variations", + "//content/test:test_support", "//testing/gtest", "//url", ] @@ -465,3 +509,32 @@ source_set("unit_tests_mobile") { cflags = [ "/wd4267" ] # Conversion from size_t to 'type'. } } + +static_library("whitelist_checker_client") { + sources = [ + "whitelist_checker_client.cc", + "whitelist_checker_client.h", + ] + deps = [ + ":database_manager", + "//base:base", + ] +} + +source_set("whitelist_checker_client_unittest") { + testonly = true + sources = [ + "whitelist_checker_client_unittest.cc", + ] + deps = [ + ":database_manager", + ":test_database_manager", + ":whitelist_checker_client", + "//base:base", + "//base/test:test_support", + "//content/public/browser", + "//content/test:test_support", + "//testing/gmock:gmock", + "//testing/gtest:gtest", + ] +} diff --git a/chromium/components/safe_browsing_db/DEPS b/chromium/components/safe_browsing_db/DEPS index 55dc17a90ee..341bbf0b8ca 100644 --- a/chromium/components/safe_browsing_db/DEPS +++ b/chromium/components/safe_browsing_db/DEPS @@ -1,6 +1,7 @@ include_rules = [ "+components/data_use_measurement/core", "+components/safe_browsing/common/safe_browsing_prefs.h", + "+components/safe_browsing/features.h", "+components/variations", "+components/version_info", "+content/public/browser", diff --git a/chromium/components/safe_browsing_db/OWNERS b/chromium/components/safe_browsing_db/OWNERS index 2f08295890d..50851251692 100644 --- a/chromium/components/safe_browsing_db/OWNERS +++ b/chromium/components/safe_browsing_db/OWNERS @@ -1,6 +1,3 @@ -mattm@chromium.org +jialiul@chromium.org nparker@chromium.org vakh@chromium.org - -# For reporting-related changes: -jialiul@chromium.org diff --git a/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.cc b/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.cc index 8040c2caa50..3acd9a4153a 100644 --- a/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.cc +++ b/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.cc @@ -10,8 +10,10 @@ #include "base/android/jni_android.h" #include "base/android/jni_array.h" #include "base/android/jni_string.h" +#include "base/containers/flat_set.h" #include "base/metrics/histogram_macros.h" #include "components/safe_browsing_db/safe_browsing_api_handler_util.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "content/public/browser/browser_thread.h" #include "jni/SafeBrowsingApiBridge_jni.h" @@ -48,6 +50,8 @@ int SBThreatTypeToJavaThreatType(const SBThreatType& sb_threat_type) { return safe_browsing::JAVA_THREAT_TYPE_POTENTIALLY_HARMFUL_APPLICATION; case SB_THREAT_TYPE_URL_UNWANTED: return safe_browsing::JAVA_THREAT_TYPE_UNWANTED_SOFTWARE; + case SB_THREAT_TYPE_SUBRESOURCE_FILTER: + return safe_browsing::JAVA_THREAT_TYPE_SUBRESOURCE_FILTER; default: NOTREACHED(); return 0; @@ -55,9 +59,9 @@ int SBThreatTypeToJavaThreatType(const SBThreatType& sb_threat_type) { } // Convert a vector of SBThreatTypes to JavaIntArray of Java threat types. -ScopedJavaLocalRef<jintArray> SBThreatTypesToJavaArray( +ScopedJavaLocalRef<jintArray> SBThreatTypeSetToJavaArray( JNIEnv* env, - const std::vector<SBThreatType>& threat_types) { + const SBThreatTypeSet& threat_types) { DCHECK(threat_types.size() > 0); int int_threat_types[threat_types.size()]; int* itr = &int_threat_types[0]; @@ -149,7 +153,7 @@ bool SafeBrowsingApiHandlerBridge::CheckApiIsSupported() { void SafeBrowsingApiHandlerBridge::StartURLCheck( const SafeBrowsingApiHandler::URLCheckCallbackMeta& callback, const GURL& url, - const std::vector<SBThreatType>& threat_types) { + const SBThreatTypeSet& threat_types) { DCHECK_CURRENTLY_ON(BrowserThread::IO); if (!CheckApiIsSupported()) { @@ -166,14 +170,12 @@ void SafeBrowsingApiHandlerBridge::StartURLCheck( DVLOG(1) << "Starting check " << callback_id << " for URL " << url; - // Default threat types, to support upstream code that doesn't yet set them. - std::vector<SBThreatType> local_threat_types(threat_types); - DCHECK(!local_threat_types.empty()); + DCHECK(!threat_types.empty()); JNIEnv* env = AttachCurrentThread(); ScopedJavaLocalRef<jstring> j_url = ConvertUTF8ToJavaString(env, url.spec()); ScopedJavaLocalRef<jintArray> j_threat_types = - SBThreatTypesToJavaArray(env, local_threat_types); + SBThreatTypeSetToJavaArray(env, threat_types); Java_SafeBrowsingApiBridge_startUriLookup(env, j_api_handler_, callback_id, j_url, j_threat_types); diff --git a/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.h b/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.h index 678437affbe..40eda4d2149 100644 --- a/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.h +++ b/chromium/components/safe_browsing_db/android/safe_browsing_api_handler_bridge.h @@ -15,6 +15,7 @@ #include "base/android/jni_android.h" #include "base/macros.h" #include "components/safe_browsing_db/safe_browsing_api_handler.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "url/gurl.h" namespace safe_browsing { @@ -28,7 +29,7 @@ class SafeBrowsingApiHandlerBridge : public SafeBrowsingApiHandler { // Makes Native->Java call to check the URL against Safe Browsing lists. void StartURLCheck(const URLCheckCallbackMeta& callback, const GURL& url, - const std::vector<SBThreatType>& threat_types) override; + const SBThreatTypeSet& threat_types) override; private: // Creates the j_api_handler_ if it hasn't been already. If the API is not diff --git a/chromium/components/safe_browsing_db/database_manager.cc b/chromium/components/safe_browsing_db/database_manager.cc index 0155efc9f8f..6633c417ced 100644 --- a/chromium/components/safe_browsing_db/database_manager.cc +++ b/chromium/components/safe_browsing_db/database_manager.cc @@ -15,7 +15,11 @@ using content::BrowserThread; namespace safe_browsing { -SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager() : enabled_(false) {} +SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager() + : base::RefCountedDeleteOnSequence<SafeBrowsingDatabaseManager>( + content::BrowserThread::GetTaskRunnerForThread( + content::BrowserThread::IO)), + enabled_(false) {} SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() { DCHECK(!v4_get_hash_protocol_manager_); diff --git a/chromium/components/safe_browsing_db/database_manager.h b/chromium/components/safe_browsing_db/database_manager.h index 01ae7caa105..c50ef687f99 100644 --- a/chromium/components/safe_browsing_db/database_manager.h +++ b/chromium/components/safe_browsing_db/database_manager.h @@ -15,9 +15,10 @@ #include "base/gtest_prod_util.h" #include "base/macros.h" -#include "base/memory/ref_counted.h" +#include "base/memory/ref_counted_delete_on_sequence.h" #include "components/safe_browsing_db/hit_report.h" #include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "content/public/common/resource_type.h" #include "url/gurl.h" @@ -27,12 +28,20 @@ class URLRequestContextGetter; namespace safe_browsing { +// Value returned by some Check*Whitelist() calls that may or may not have an +// immediate answer. +enum class AsyncMatch { + ASYNC, // No answer yet -- Client will get a callback + MATCH, // URL matches the list. No callback. + NO_MATCH, // URL doesn't match. No callback. +}; + struct V4ProtocolConfig; class V4GetHashProtocolManager; // Base class to either the locally-managed or a remotely-managed database. class SafeBrowsingDatabaseManager - : public base::RefCountedThreadSafe<SafeBrowsingDatabaseManager> { + : public base::RefCountedDeleteOnSequence<SafeBrowsingDatabaseManager> { public: // Callers requesting a result should derive from this class. // The destructor should call db_manager->CancelCheck(client) if a @@ -64,6 +73,10 @@ class SafeBrowsingDatabaseManager virtual void OnCheckResourceUrlResult(const GURL& url, SBThreatType threat_type, const std::string& threat_hash) {} + + // Called when the result of checking a whitelist is known. + // Currently only used for CSD whitelist. + virtual void OnCheckWhitelistUrlResult(bool is_whitelisted) {} }; // @@ -88,6 +101,8 @@ class SafeBrowsingDatabaseManager virtual bool CanCheckResourceType( content::ResourceType resource_type) const = 0; + virtual bool CanCheckSubresourceFilter() const = 0; + // Returns true if the url's scheme can be checked. virtual bool CanCheckUrl(const GURL& url) const = 0; @@ -112,20 +127,19 @@ class SafeBrowsingDatabaseManager // and "client" is called asynchronously with the result when it is ready. virtual bool CheckApiBlacklistUrl(const GURL& url, Client* client); + // Check if the |url| matches any of the full-length hashes from the client- + // side phishing detection whitelist. The 3-state return value indicates + // the result or that the Client will get a callback later with the result. + virtual AsyncMatch CheckCsdWhitelistUrl(const GURL& url, Client* client) = 0; + // Called on the IO thread to check if the given url is safe or not. If we // can synchronously determine that the url is safe, CheckUrl returns true. // Otherwise it returns false, and "client" is called asynchronously with the - // result when it is ready. - virtual bool CheckBrowseUrl(const GURL& url, Client* client) = 0; - - // Called on the IO thread to check if the given url belongs to the - // subresource filter list. If the url doesn't belong to the list, the check - // happens synchronously, otherwise it returns false, and "client" is called - // asynchronously with the result when it is ready. - // Currently supported only on desktop. Returns TRUE if the list is not yet - // available. - virtual bool CheckUrlForSubresourceFilter(const GURL& url, - Client* client) = 0; + // result when it is ready. The URL will only be checked for the threat types + // in |threat_types|. + virtual bool CheckBrowseUrl(const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) = 0; // Check if the prefix for |url| is in safebrowsing download add lists. // Result will be passed to callback in |client|. @@ -143,15 +157,24 @@ class SafeBrowsingDatabaseManager // to callback in |client|. virtual bool CheckResourceUrl(const GURL& url, Client* client) = 0; + // Called on the IO thread to check if the given url belongs to a list the + // subresource cares about. If the url doesn't belong to any such list and the + // check can happen synchronously, returns true. Otherwise it returns false, + // and "client" is called asynchronously with the result when it is ready. + // Returns true if the list is not yet available. + virtual bool CheckUrlForSubresourceFilter(const GURL& url, + Client* client) = 0; + // - // Methods to synchronously check whether a URL, or full hash, or IP address - // or a DLL file is safe. + // Match*(): Methods to synchronously check if various types are safe. // // Check if the |url| matches any of the full-length hashes from the client- // side phishing detection whitelist. Returns true if there was a match and // false otherwise. To make sure we are conservative we will return true if // an error occurs. This method must be called on the IO thread. + // + // DEPRECATED. ref: http://crbug.com/714300 virtual bool MatchCsdWhitelistUrl(const GURL& url) = 0; // Check if |str| matches any of the full-length hashes from the download @@ -241,7 +264,8 @@ class SafeBrowsingDatabaseManager virtual ~SafeBrowsingDatabaseManager(); - friend class base::RefCountedThreadSafe<SafeBrowsingDatabaseManager>; + friend class base::RefCountedDeleteOnSequence<SafeBrowsingDatabaseManager>; + friend class base::DeleteHelper<SafeBrowsingDatabaseManager>; FRIEND_TEST_ALL_PREFIXES(SafeBrowsingDatabaseManagerTest, CheckApiBlacklistUrlPrefixes); diff --git a/chromium/components/safe_browsing_db/database_manager_unittest.cc b/chromium/components/safe_browsing_db/database_manager_unittest.cc index 5180687b6f3..5669978dcef 100644 --- a/chromium/components/safe_browsing_db/database_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/database_manager_unittest.cc @@ -17,6 +17,7 @@ #include "base/memory/ref_counted.h" #include "base/run_loop.h" #include "base/single_thread_task_runner.h" +#include "base/synchronization/waitable_event.h" #include "base/threading/thread_task_runner_handle.h" #include "components/safe_browsing_db/test_database_manager.h" #include "components/safe_browsing_db/v4_protocol_manager_util.h" @@ -68,8 +69,9 @@ class SafeBrowsingDatabaseManagerTest : public testing::Test { } void TearDown() override { - base::RunLoop().RunUntilIdle(); db_manager_->StopOnIOThread(false); + db_manager_ = nullptr; + base::RunLoop().RunUntilIdle(); } std::string GetStockV4GetHashResponse() { diff --git a/chromium/components/safe_browsing_db/hit_report.h b/chromium/components/safe_browsing_db/hit_report.h index 4fecc44c9fa..9df6a900bf8 100644 --- a/chromium/components/safe_browsing_db/hit_report.h +++ b/chromium/components/safe_browsing_db/hit_report.h @@ -21,6 +21,7 @@ enum class ThreatSource { LOCAL_PVER4, // From V4LocalDatabaseManager, protocol v4 REMOTE, // From RemoteSafeBrowsingDatabaseManager CLIENT_SIDE_DETECTION, // From ClientSideDetectionHost + PASSWORD_PROTECTION_SERVICE, // From PasswordProtectionService }; // Data to report about the contents of a particular threat (malware, phishing, diff --git a/chromium/components/safe_browsing_db/remote_database_manager.cc b/chromium/components/safe_browsing_db/remote_database_manager.cc index b620426b711..5762d41831c 100644 --- a/chromium/components/safe_browsing_db/remote_database_manager.cc +++ b/chromium/components/safe_browsing_db/remote_database_manager.cc @@ -13,6 +13,7 @@ #include "base/timer/elapsed_timer.h" #include "components/safe_browsing_db/safe_browsing_api_handler.h" #include "components/safe_browsing_db/v4_get_hash_protocol_manager.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "components/variations/variations_associated_data.h" #include "content/public/browser/browser_thread.h" @@ -172,18 +173,26 @@ bool RemoteSafeBrowsingDatabaseManager::CanCheckResourceType( return resource_types_to_check_.count(resource_type) > 0; } +bool RemoteSafeBrowsingDatabaseManager::CanCheckSubresourceFilter() const { + return true; +} + bool RemoteSafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const { - return url.SchemeIs(url::kHttpsScheme) || url.SchemeIs(url::kHttpScheme) || - url.SchemeIs(url::kFtpScheme); + return url.SchemeIsHTTPOrHTTPS() || url.SchemeIs(url::kFtpScheme) || + url.SchemeIsWSOrWSS(); } bool RemoteSafeBrowsingDatabaseManager::ChecksAreAlwaysAsync() const { return true; } -bool RemoteSafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url, - Client* client) { +bool RemoteSafeBrowsingDatabaseManager::CheckBrowseUrl( + const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); + DCHECK(!threat_types.empty()); + DCHECK(SBThreatTypeSetIsValidForCheckBrowseUrl(threat_types)); if (!enabled_) return true; @@ -202,8 +211,7 @@ bool RemoteSafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url, DCHECK(api_handler) << "SafeBrowsingApiHandler was never constructed"; api_handler->StartURLCheck( base::Bind(&ClientRequest::OnRequestDoneWeak, req->GetWeakPtr()), url, - {SB_THREAT_TYPE_URL_MALWARE, SB_THREAT_TYPE_URL_PHISHING, - SB_THREAT_TYPE_URL_UNWANTED}); + threat_types); LogPendingChecks(current_requests_.size()); current_requests_.push_back(req.release()); @@ -236,6 +244,8 @@ bool RemoteSafeBrowsingDatabaseManager::CheckUrlForSubresourceFilter( const GURL& url, Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); + DCHECK(CanCheckSubresourceFilter()); + if (!enabled_ || !CanCheckUrl(url)) return true; @@ -249,7 +259,8 @@ bool RemoteSafeBrowsingDatabaseManager::CheckUrlForSubresourceFilter( DCHECK(api_handler) << "SafeBrowsingApiHandler was never constructed"; api_handler->StartURLCheck( base::Bind(&ClientRequest::OnRequestDoneWeak, req->GetWeakPtr()), url, - {SB_THREAT_TYPE_SUBRESOURCE_FILTER}); + CreateSBThreatTypeSet( + {SB_THREAT_TYPE_SUBRESOURCE_FILTER, SB_THREAT_TYPE_URL_PHISHING})); LogPendingChecks(current_requests_.size()); current_requests_.push_back(req.release()); @@ -258,6 +269,13 @@ bool RemoteSafeBrowsingDatabaseManager::CheckUrlForSubresourceFilter( return false; } +AsyncMatch RemoteSafeBrowsingDatabaseManager::CheckCsdWhitelistUrl( + const GURL& url, + Client* client) { + NOTREACHED(); + return AsyncMatch::MATCH; +} + bool RemoteSafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) { NOTREACHED(); return true; diff --git a/chromium/components/safe_browsing_db/remote_database_manager.h b/chromium/components/safe_browsing_db/remote_database_manager.h index 1f13ca391fb..c2a30ff2016 100644 --- a/chromium/components/safe_browsing_db/remote_database_manager.h +++ b/chromium/components/safe_browsing_db/remote_database_manager.h @@ -41,13 +41,17 @@ class RemoteSafeBrowsingDatabaseManager : public SafeBrowsingDatabaseManager { void CancelCheck(Client* client) override; bool CanCheckResourceType(content::ResourceType resource_type) const override; + bool CanCheckSubresourceFilter() const override; bool CanCheckUrl(const GURL& url) const override; bool ChecksAreAlwaysAsync() const override; - bool CheckBrowseUrl(const GURL& url, Client* client) override; + bool CheckBrowseUrl(const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) override; bool CheckDownloadUrl(const std::vector<GURL>& url_chain, Client* client) override; bool CheckExtensionIDs(const std::set<std::string>& extension_ids, Client* client) override; + AsyncMatch CheckCsdWhitelistUrl(const GURL& url, Client* client) override; bool CheckResourceUrl(const GURL& url, Client* client) override; bool CheckUrlForSubresourceFilter(const GURL& url, Client* client) override; bool MatchCsdWhitelistUrl(const GURL& url) override; diff --git a/chromium/components/safe_browsing_db/remote_database_manager_unittest.cc b/chromium/components/safe_browsing_db/remote_database_manager_unittest.cc index 284b3358c88..4a2b49a99d7 100644 --- a/chromium/components/safe_browsing_db/remote_database_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/remote_database_manager_unittest.cc @@ -8,10 +8,12 @@ #include "base/logging.h" #include "base/metrics/field_trial.h" +#include "base/run_loop.h" #include "base/strings/stringprintf.h" #include "base/time/time.h" #include "components/safe_browsing_db/safe_browsing_api_handler.h" #include "components/variations/variations_associated_data.h" +#include "content/public/test/test_browser_thread_bundle.h" #include "testing/gtest/include/gtest/gtest.h" namespace safe_browsing { @@ -22,7 +24,7 @@ class TestSafeBrowsingApiHandler : public SafeBrowsingApiHandler { public: void StartURLCheck(const URLCheckCallbackMeta& callback, const GURL& url, - const std::vector<SBThreatType>& threat_types) override {} + const SBThreatTypeSet& threat_types) override {} }; } // namespace @@ -37,6 +39,11 @@ class RemoteDatabaseManagerTest : public testing::Test { db_ = new RemoteSafeBrowsingDatabaseManager(); } + void TearDown() override { + db_ = nullptr; + base::RunLoop().RunUntilIdle(); + } + // Setup the two field trial params. These are read in db_'s ctor. void SetFieldTrialParams(const std::string types_to_check_val) { // Destroy the existing FieldTrialList before creating a new one to avoid @@ -59,6 +66,7 @@ class RemoteDatabaseManagerTest : public testing::Test { group_name, params)); } + content::TestBrowserThreadBundle thread_bundle_; std::unique_ptr<base::FieldTrialList> field_trials_; TestSafeBrowsingApiHandler api_handler_; scoped_refptr<RemoteSafeBrowsingDatabaseManager> db_; diff --git a/chromium/components/safe_browsing_db/safe_browsing_api_handler.h b/chromium/components/safe_browsing_db/safe_browsing_api_handler.h index fa01a6077af..f2e7abaf63c 100644 --- a/chromium/components/safe_browsing_db/safe_browsing_api_handler.h +++ b/chromium/components/safe_browsing_db/safe_browsing_api_handler.h @@ -13,6 +13,7 @@ #include "base/callback.h" #include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "url/gurl.h" namespace safe_browsing { @@ -30,7 +31,7 @@ class SafeBrowsingApiHandler { // Makes Native->Java call and invokes callback when check is done. virtual void StartURLCheck(const URLCheckCallbackMeta& callback, const GURL& url, - const std::vector<SBThreatType>& threat_types) = 0; + const SBThreatTypeSet& threat_types) = 0; virtual ~SafeBrowsingApiHandler() {} diff --git a/chromium/components/safe_browsing_db/test_database_manager.cc b/chromium/components/safe_browsing_db/test_database_manager.cc index 423097e1318..343ce155d65 100644 --- a/chromium/components/safe_browsing_db/test_database_manager.cc +++ b/chromium/components/safe_browsing_db/test_database_manager.cc @@ -23,6 +23,10 @@ bool TestSafeBrowsingDatabaseManager::CanCheckResourceType( return false; } +bool TestSafeBrowsingDatabaseManager::CanCheckSubresourceFilter() const { + return false; +} + bool TestSafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const { NOTIMPLEMENTED(); return false; @@ -33,8 +37,10 @@ bool TestSafeBrowsingDatabaseManager::ChecksAreAlwaysAsync() const { return false; } -bool TestSafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url, - Client* client) { +bool TestSafeBrowsingDatabaseManager::CheckBrowseUrl( + const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) { NOTIMPLEMENTED(); return true; } @@ -66,6 +72,13 @@ bool TestSafeBrowsingDatabaseManager::CheckUrlForSubresourceFilter( return true; } +AsyncMatch TestSafeBrowsingDatabaseManager::CheckCsdWhitelistUrl( + const GURL& url, + Client* client) { + NOTIMPLEMENTED(); + return AsyncMatch::MATCH; +} + bool TestSafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) { NOTIMPLEMENTED(); return true; diff --git a/chromium/components/safe_browsing_db/test_database_manager.h b/chromium/components/safe_browsing_db/test_database_manager.h index c2cd803fa9a..51183720336 100644 --- a/chromium/components/safe_browsing_db/test_database_manager.h +++ b/chromium/components/safe_browsing_db/test_database_manager.h @@ -10,6 +10,7 @@ #include <vector> #include "components/safe_browsing_db/database_manager.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" namespace safe_browsing { @@ -24,8 +25,12 @@ class TestSafeBrowsingDatabaseManager void CancelCheck(Client* client) override; bool CanCheckResourceType(content::ResourceType resource_type) const override; bool CanCheckUrl(const GURL& url) const override; + bool CanCheckSubresourceFilter() const override; bool ChecksAreAlwaysAsync() const override; - bool CheckBrowseUrl(const GURL& url, Client* client) override; + bool CheckBrowseUrl(const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) override; + AsyncMatch CheckCsdWhitelistUrl(const GURL& url, Client* client) override; bool CheckDownloadUrl(const std::vector<GURL>& url_chain, Client* client) override; bool CheckExtensionIDs(const std::set<std::string>& extension_ids, @@ -47,7 +52,7 @@ class TestSafeBrowsingDatabaseManager void StopOnIOThread(bool shutdown) override; protected: - ~TestSafeBrowsingDatabaseManager() override {}; + ~TestSafeBrowsingDatabaseManager() override {} }; } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/util.h b/chromium/components/safe_browsing_db/util.h index dd03f5fe03f..ff47d837205 100644 --- a/chromium/components/safe_browsing_db/util.h +++ b/chromium/components/safe_browsing_db/util.h @@ -24,7 +24,7 @@ class GURL; namespace safe_browsing { // Metadata that indicates what kind of URL match this is. -enum class ThreatPatternType { +enum class ThreatPatternType : int { NONE = 0, // Pattern type didn't appear in the metadata MALWARE_LANDING = 1, // The match is a malware landing page MALWARE_DISTRIBUTION = 2, // The match is a malware distribution page diff --git a/chromium/components/safe_browsing_db/v4_database.cc b/chromium/components/safe_browsing_db/v4_database.cc index 9164951b2fb..fc23c9b68d2 100644 --- a/chromium/components/safe_browsing_db/v4_database.cc +++ b/chromium/components/safe_browsing_db/v4_database.cc @@ -9,6 +9,7 @@ #include "base/lazy_instance.h" #include "base/memory/ptr_util.h" #include "base/metrics/histogram_macros.h" +#include "base/task_runner_util.h" #include "base/threading/thread_task_runner_handle.h" #include "components/safe_browsing_db/v4_database.h" #include "content/public/browser/browser_thread.h" @@ -30,6 +31,19 @@ base::LazyInstance<std::unique_ptr<V4DatabaseFactory>>::Leaky g_db_factory = base::LazyInstance<std::unique_ptr<V4StoreFactory>>::Leaky g_store_factory = LAZY_INSTANCE_INITIALIZER; +// Verifies the checksums on a collection of stores. +// Returns the IDs of stores whose checksums failed to verify. +std::vector<ListIdentifier> VerifyChecksums( + std::vector<std::pair<ListIdentifier, V4Store*>> stores) { + std::vector<ListIdentifier> stores_to_reset; + for (const auto& store_map_iter : stores) { + if (!store_map_iter.second->VerifyChecksum()) { + stores_to_reset.push_back(store_map_iter.first); + } + } + return stores_to_reset; +} + } // namespace std::unique_ptr<V4Database> V4DatabaseFactory::Create( @@ -65,7 +79,7 @@ void V4Database::CreateOnTaskRunner( const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, NewDatabaseReadyCallback new_db_callback, const TimeTicks create_start_time) { - DCHECK(db_task_runner->RunsTasksOnCurrentThread()); + DCHECK(db_task_runner->RunsTasksInCurrentSequence()); if (!g_store_factory.Get()) g_store_factory.Get() = base::MakeUnique<V4StoreFactory>(); @@ -119,7 +133,7 @@ V4Database::V4Database( db_task_runner_(db_task_runner), pending_store_updates_(0), weak_factory_on_io_(this) { - DCHECK(db_task_runner->RunsTasksOnCurrentThread()); + DCHECK(db_task_runner->RunsTasksInCurrentSequence()); } // static @@ -133,7 +147,7 @@ void V4Database::Destroy(std::unique_ptr<V4Database> v4_database) { } V4Database::~V4Database() { - DCHECK(db_task_runner_->RunsTasksOnCurrentThread()); + DCHECK(db_task_runner_->RunsTasksInCurrentSequence()); } void V4Database::ApplyUpdate( @@ -254,28 +268,19 @@ void V4Database::ResetStores( void V4Database::VerifyChecksum( DatabaseReadyForUpdatesCallback db_ready_for_updates_callback) { DCHECK_CURRENTLY_ON(BrowserThread::IO); - // TODO(vakh): Consider using PostTaskAndReply instead. - const scoped_refptr<base::SingleThreadTaskRunner> callback_task_runner = - base::ThreadTaskRunnerHandle::Get(); - db_task_runner_->PostTask( - FROM_HERE, - base::Bind(&V4Database::VerifyChecksumOnTaskRunner, - weak_factory_on_io_.GetWeakPtr(), callback_task_runner, - db_ready_for_updates_callback)); -} -void V4Database::VerifyChecksumOnTaskRunner( - const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, - DatabaseReadyForUpdatesCallback db_ready_for_updates_callback) { - std::vector<ListIdentifier> stores_to_reset; - for (const auto& store_map_iter : *store_map_) { - if (!store_map_iter.second->VerifyChecksum()) { - stores_to_reset.push_back(store_map_iter.first); - } + // Make a threadsafe copy of store_map_ w/raw pointers that we can hand to + // the DB thread. The V4Stores ptrs are guaranteed to be valid because their + // deletion would be sequenced on the DB thread, after this posted task is + // serviced. + std::vector<std::pair<ListIdentifier, V4Store*>> stores; + for (const auto& next_store : *store_map_) { + stores.push_back(std::make_pair(next_store.first, next_store.second.get())); } - callback_task_runner->PostTask( - FROM_HERE, base::Bind(db_ready_for_updates_callback, stores_to_reset)); + base::PostTaskAndReplyWithResult(db_task_runner_.get(), FROM_HERE, + base::Bind(&VerifyChecksums, stores), + db_ready_for_updates_callback); } bool V4Database::IsStoreAvailable(const ListIdentifier& identifier) const { diff --git a/chromium/components/safe_browsing_db/v4_database.h b/chromium/components/safe_browsing_db/v4_database.h index fd8b3af8148..0146830f08c 100644 --- a/chromium/components/safe_browsing_db/v4_database.h +++ b/chromium/components/safe_browsing_db/v4_database.h @@ -167,7 +167,11 @@ class V4Database { V4Database(const scoped_refptr<base::SequencedTaskRunner>& db_task_runner, std::unique_ptr<StoreMap> store_map); - // Map of ListIdentifier to the V4Store. + // The collection of V4Stores, keyed by ListIdentifier. + // The map itself lives on the V4Database's parent thread, but its V4Store + // objects live on the db_task_runner_thread. + // TODO(vakh): Consider writing a container object which encapsulates or + // harmonizes thread affinity for the associative container and the data. const std::unique_ptr<StoreMap> store_map_; private: diff --git a/chromium/components/safe_browsing_db/v4_database_unittest.cc b/chromium/components/safe_browsing_db/v4_database_unittest.cc index 66db6fa5b95..de42e1a3bf9 100644 --- a/chromium/components/safe_browsing_db/v4_database_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_database_unittest.cc @@ -24,7 +24,7 @@ class FakeV4Store : public V4Store { const bool hash_prefix_matches) : V4Store( task_runner, - base::FilePath(store_path.value() + FILE_PATH_LITERAL(".fake"))), + base::FilePath(store_path.value() + FILE_PATH_LITERAL(".store"))), hash_prefix_should_match_(hash_prefix_matches) {} HashPrefix GetMatchingHashPrefix(const FullHash& full_hash) override { @@ -103,13 +103,13 @@ class V4DatabaseTest : public PlatformTest { SB_THREAT_TYPE_URL_MALWARE); expected_identifiers_.push_back(win_malware_id_); expected_store_paths_.push_back( - database_dirname_.AppendASCII("win_url_malware.fake")); + database_dirname_.AppendASCII("win_url_malware.store")); list_infos_.emplace_back(true, "linux_url_malware", linux_malware_id_, SB_THREAT_TYPE_URL_MALWARE); expected_identifiers_.push_back(linux_malware_id_); expected_store_paths_.push_back( - database_dirname_.AppendASCII("linux_url_malware.fake")); + database_dirname_.AppendASCII("linux_url_malware.store")); } void DatabaseUpdated() {} diff --git a/chromium/components/safe_browsing_db/v4_feature_list.cc b/chromium/components/safe_browsing_db/v4_feature_list.cc index 9d166e9f15b..8185e6fc16d 100644 --- a/chromium/components/safe_browsing_db/v4_feature_list.cc +++ b/chromium/components/safe_browsing_db/v4_feature_list.cc @@ -2,22 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "base/feature_list.h" #include "components/safe_browsing_db/v4_feature_list.h" +#include "base/feature_list.h" +#include "components/safe_browsing/features.h" + namespace safe_browsing { namespace V4FeatureList { -namespace { - -const base::Feature kLocalDatabaseManagerEnabled{ - "SafeBrowsingV4LocalDatabaseManagerEnabled", - base::FEATURE_DISABLED_BY_DEFAULT}; - -const base::Feature kV4OnlyEnabled{"SafeBrowsingV4OnlyEnabled", - base::FEATURE_DISABLED_BY_DEFAULT}; - bool IsV4OnlyEnabled() { return base::FeatureList::IsEnabled(kV4OnlyEnabled); } @@ -27,8 +20,6 @@ bool IsLocalDatabaseManagerEnabled() { IsV4OnlyEnabled(); } -} // namespace - V4UsageStatus GetV4UsageStatus() { V4UsageStatus v4_usage_status; if (safe_browsing::V4FeatureList::IsV4OnlyEnabled()) { diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc index 77488c2c422..9cfd8ffa13e 100644 --- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc +++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.cc @@ -255,10 +255,12 @@ V4GetHashProtocolManager::V4GetHashProtocolManager( threat_types_.assign(threat_types.begin(), threat_types.end()); } -V4GetHashProtocolManager::~V4GetHashProtocolManager() {} +V4GetHashProtocolManager::~V4GetHashProtocolManager() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); +} void V4GetHashProtocolManager::ClearCache() { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); full_hash_cache_.clear(); } @@ -267,7 +269,7 @@ void V4GetHashProtocolManager::GetFullHashes( full_hash_to_store_and_hash_prefixes, FullHashCallback callback) { DCHECK_CURRENTLY_ON(BrowserThread::IO); - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(!full_hash_to_store_and_hash_prefixes.empty()); std::vector<HashPrefix> prefixes_to_request; @@ -506,7 +508,7 @@ void V4GetHashProtocolManager::GetHashUrlAndHeaders( } void V4GetHashProtocolManager::HandleGetHashError(const Time& now) { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); TimeDelta next = V4ProtocolManagerUtil::GetNextBackOffInterval( &gethash_error_count_, &gethash_back_off_mult_); next_gethash_time_ = now + next; @@ -738,7 +740,7 @@ void V4GetHashProtocolManager::MergeResults( // SafeBrowsing request responses are handled here. void V4GetHashProtocolManager::OnURLFetchComplete( const net::URLFetcher* source) { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK_CURRENTLY_ON(BrowserThread::IO); PendingHashRequests::iterator it = pending_hash_requests_.find(source); diff --git a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h index 7d9c2c2aff1..6e43e8897c9 100644 --- a/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h +++ b/chromium/components/safe_browsing_db/v4_get_hash_protocol_manager.h @@ -20,7 +20,7 @@ #include "base/gtest_prod_util.h" #include "base/macros.h" -#include "base/threading/non_thread_safe.h" +#include "base/sequence_checker.h" #include "base/time/default_clock.h" #include "base/time/time.h" #include "base/timer/timer.h" @@ -139,8 +139,7 @@ struct FullHashCallbackInfo { class V4GetHashProtocolManagerFactory; -class V4GetHashProtocolManager : public net::URLFetcherDelegate, - public base::NonThreadSafe { +class V4GetHashProtocolManager : public net::URLFetcherDelegate { public: // Invoked when GetFullHashesWithApis completes. // Parameters: @@ -341,6 +340,8 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate, std::vector<ThreatEntryType> threat_entry_types_; std::vector<ThreatType> threat_types_; + SEQUENCE_CHECKER(sequence_checker_); + DISALLOW_COPY_AND_ASSIGN(V4GetHashProtocolManager); }; diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager.cc b/chromium/components/safe_browsing_db/v4_local_database_manager.cc index 02617ce96cd..326ed5b0fdb 100644 --- a/chromium/components/safe_browsing_db/v4_local_database_manager.cc +++ b/chromium/components/safe_browsing_db/v4_local_database_manager.cc @@ -11,10 +11,12 @@ #include "base/bind_helpers.h" #include "base/callback.h" +#include "base/files/file_util.h" #include "base/memory/ptr_util.h" #include "base/memory/ref_counted.h" +#include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" -#include "base/threading/sequenced_worker_pool.h" +#include "base/task_scheduler/post_task.h" #include "components/safe_browsing_db/v4_feature_list.h" #include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "content/public/browser/browser_thread.h" @@ -30,11 +32,14 @@ namespace { const ThreatSeverity kLeastSeverity = std::numeric_limits<ThreatSeverity>::max(); +// The list of the name of any store files that are no longer used and can be +// safely deleted from the disk. There's no overlap allowed between the files +// on this list and the list returned by GetListInfos(). +const char* const kStoreFileNamesToDelete[] = {"AnyIpMalware.store"}; + ListInfos GetListInfos() { // NOTE(vakh): When adding a store here, add the corresponding store-specific // histograms also. - // NOTE(vakh): Delete file "AnyIpMalware.store". It has been renamed to - // "IpMalware.store". If it exists, it should be 75 bytes long. // The first argument to ListInfo specifies whether to sync hash prefixes for // that list. This can be false for two reasons: // - The server doesn't support that list yet. Once the server adds support @@ -59,7 +64,7 @@ ListInfos GetListInfos() { ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdDownloadWhitelist.store", GetUrlCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED), ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdWhitelist.store", - GetUrlCsdWhitelistId(), SB_THREAT_TYPE_UNUSED), + GetUrlCsdWhitelistId(), SB_THREAT_TYPE_CSD_WHITELIST), ListInfo(kSyncAlways, "UrlSoceng.store", GetUrlSocEngId(), SB_THREAT_TYPE_URL_PHISHING), ListInfo(kSyncAlways, "UrlMalware.store", GetUrlMalwareId(), @@ -67,7 +72,7 @@ ListInfos GetListInfos() { ListInfo(kSyncAlways, "UrlUws.store", GetUrlUwsId(), SB_THREAT_TYPE_URL_UNWANTED), ListInfo(kSyncAlways, "UrlMalBin.store", GetUrlMalBinId(), - SB_THREAT_TYPE_BINARY_MALWARE_URL), + SB_THREAT_TYPE_URL_BINARY_MALWARE), ListInfo(kSyncAlways, "ChromeExtMalware.store", GetChromeExtMalwareId(), SB_THREAT_TYPE_EXTENSION), ListInfo(kSyncOnlyOnChromeBuilds, "ChromeUrlClientIncident.store", @@ -96,6 +101,8 @@ ThreatSeverity GetThreatSeverity(const ListIdentifier& list_id) { case CLIENT_INCIDENT: case SUBRESOURCE_FILTER: return 2; + case CSD_WHITELIST: + return 3; default: NOTREACHED() << "Unexpected ThreatType encountered: " << list_id.threat_type(); @@ -103,6 +110,34 @@ ThreatSeverity GetThreatSeverity(const ListIdentifier& list_id) { } } +// This is only valid for types that are passed to GetBrowseUrl(). +ListIdentifier GetUrlIdFromSBThreatType(SBThreatType sb_threat_type) { + switch (sb_threat_type) { + case SB_THREAT_TYPE_URL_MALWARE: + return GetUrlMalwareId(); + + case SB_THREAT_TYPE_URL_PHISHING: + return GetUrlSocEngId(); + + case SB_THREAT_TYPE_URL_UNWANTED: + return GetUrlUwsId(); + + default: + NOTREACHED(); + // Compiler requires a return statement here. + return GetUrlMalwareId(); + } +} + +StoresToCheck CreateStoresToCheckFromSBThreatTypeSet( + const SBThreatTypeSet& threat_types) { + StoresToCheck stores_to_check; + for (SBThreatType sb_threat_type : threat_types) { + stores_to_check.insert(GetUrlIdFromSBThreatType(sb_threat_type)); + } + return stores_to_check; +} + } // namespace V4LocalDatabaseManager::PendingCheck::PendingCheck( @@ -118,7 +153,6 @@ V4LocalDatabaseManager::PendingCheck::PendingCheck( for (const auto& url : urls) { V4ProtocolManagerUtil::UrlToFullHashes(url, &full_hashes); } - DCHECK(full_hashes.size()); full_hash_threat_types.assign(full_hashes.size(), SB_THREAT_TYPE_SAFE); } @@ -152,10 +186,14 @@ V4LocalDatabaseManager::V4LocalDatabaseManager( : base_path_(base_path), extended_reporting_level_callback_(extended_reporting_level_callback), list_infos_(GetListInfos()), + task_runner_(base::CreateSequencedTaskRunnerWithTraits( + {base::MayBlock(), base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})), weak_factory_(this) { DCHECK(!base_path_.empty()); DCHECK(!list_infos_.empty()); + DeleteUnusedStoreFiles(); + DVLOG(1) << "V4LocalDatabaseManager::V4LocalDatabaseManager: " << "base_path_: " << base_path_.AsUTF8Unsafe(); } @@ -195,17 +233,25 @@ bool V4LocalDatabaseManager::CanCheckResourceType( return true; } +bool V4LocalDatabaseManager::CanCheckSubresourceFilter() const { + return true; +} + bool V4LocalDatabaseManager::CanCheckUrl(const GURL& url) const { - return url.SchemeIs(url::kHttpsScheme) || url.SchemeIs(url::kHttpScheme) || - url.SchemeIs(url::kFtpScheme); + return url.SchemeIsHTTPOrHTTPS() || url.SchemeIs(url::kFtpScheme) || + url.SchemeIsWSOrWSS(); } bool V4LocalDatabaseManager::ChecksAreAlwaysAsync() const { return false; } -bool V4LocalDatabaseManager::CheckBrowseUrl(const GURL& url, Client* client) { +bool V4LocalDatabaseManager::CheckBrowseUrl(const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); + DCHECK(!threat_types.empty()); + DCHECK(SBThreatTypeSetIsValidForCheckBrowseUrl(threat_types)); if (!enabled_ || !CanCheckUrl(url)) { return true; @@ -213,7 +259,7 @@ bool V4LocalDatabaseManager::CheckBrowseUrl(const GURL& url, Client* client) { std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( client, ClientCallbackType::CHECK_BROWSE_URL, - StoresToCheck({GetUrlMalwareId(), GetUrlSocEngId(), GetUrlUwsId()}), + CreateStoresToCheckFromSBThreatTypeSet(threat_types), std::vector<GURL>(1, url)); return HandleCheck(std::move(check)); @@ -275,6 +321,7 @@ bool V4LocalDatabaseManager::CheckResourceUrl(const GURL& url, Client* client) { bool V4LocalDatabaseManager::CheckUrlForSubresourceFilter(const GURL& url, Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); + DCHECK(CanCheckSubresourceFilter()); StoresToCheck stores_to_check( {GetUrlSocEngId(), GetUrlSubresourceFilterId()}); @@ -289,15 +336,34 @@ bool V4LocalDatabaseManager::CheckUrlForSubresourceFilter(const GURL& url, return HandleCheck(std::move(check)); } -bool V4LocalDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) { +AsyncMatch V4LocalDatabaseManager::CheckCsdWhitelistUrl(const GURL& url, + Client* client) { DCHECK_CURRENTLY_ON(BrowserThread::IO); StoresToCheck stores_to_check({GetUrlCsdWhitelistId()}); - if (!AreAllStoresAvailableNow(stores_to_check)) { + if (!AreAllStoresAvailableNow(stores_to_check) || !CanCheckUrl(url)) { // Fail open: Whitelist everything. Otherwise we may run the // CSD phishing/malware detector on popular domains and generate - // undue load on the client and server. This has the effect of disabling - // CSD phishing/malware detection until the store is first synced. + // undue load on the client and server, or send Password Reputation + // requests on popular sites. This has the effect of disabling + // CSD phishing/malware detection and password reputation service + // until the store is first synced and/or loaded from disk. + return AsyncMatch::MATCH; + } + + std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>( + client, ClientCallbackType::CHECK_CSD_WHITELIST, stores_to_check, + std::vector<GURL>(1, url)); + + return HandleWhitelistCheck(std::move(check)); +} + +bool V4LocalDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + + StoresToCheck stores_to_check({GetUrlCsdWhitelistId()}); + if (!AreAllStoresAvailableNow(stores_to_check)) { + // Fail open: Whitelist everything. See CheckCsdWhitelistUrl. return true; } @@ -473,6 +539,33 @@ void V4LocalDatabaseManager::DatabaseUpdated() { } } +void V4LocalDatabaseManager::DeleteUnusedStoreFiles() { + for (auto* const store_filename_to_delete : kStoreFileNamesToDelete) { + // Is the file marked for deletion also being used for a valid V4Store? + auto it = std::find_if(std::begin(list_infos_), std::end(list_infos_), + [&store_filename_to_delete](ListInfo const& li) { + return li.filename() == store_filename_to_delete; + }); + if (list_infos_.end() == it) { + const base::FilePath store_path = + base_path_.AppendASCII(store_filename_to_delete); + bool path_exists = base::PathExists(store_path); + base::UmaHistogramBoolean("SafeBrowsing.V4UnusedStoreFileExists" + + GetUmaSuffixForStore(store_path), + path_exists); + if (!path_exists) { + continue; + } + task_runner_->PostTask(FROM_HERE, + base::Bind(base::IgnoreResult(&base::DeleteFile), + store_path, false /* recursive */)); + } else { + NOTREACHED() << "Trying to delete a store file that's in use: " + << store_filename_to_delete; + } + } +} + bool V4LocalDatabaseManager::GetPrefixMatches( const std::unique_ptr<PendingCheck>& check, FullHashToStoreAndHashPrefixesMap* full_hash_to_store_and_hash_prefixes) { @@ -482,7 +575,6 @@ bool V4LocalDatabaseManager::GetPrefixMatches( DCHECK(v4_database_); const base::TimeTicks before = TimeTicks::Now(); - DCHECK(!check->full_hashes.empty()); full_hash_to_store_and_hash_prefixes->clear(); for (const auto& full_hash : check->full_hashes) { @@ -554,6 +646,34 @@ SBThreatType V4LocalDatabaseManager::GetSBThreatTypeForList( return it->sb_threat_type(); } +AsyncMatch V4LocalDatabaseManager::HandleWhitelistCheck( + std::unique_ptr<PendingCheck> check) { + // We don't bother queuing whitelist checks since the DB will + // normally be available already -- whitelists are used after page load, + // and navigations are blocked until the DB is ready and dequeues checks. + // The caller should have already checked that the DB is ready. + DCHECK(v4_database_); + + FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes; + if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) { + return AsyncMatch::NO_MATCH; + } + + // Look for any full-length hash in the matches. If there is one, + // there's no need for a full-hash check. This saves bandwidth for + // very popular sites since they'll have full-length hashes locally. + // These loops will have exactly 1 entry most of the time. + for (const auto& entry : full_hash_to_store_and_hash_prefixes) { + for (const auto& store_and_prefix : entry.second) { + if (store_and_prefix.hash_prefix.size() == kMaxHashPrefixLength) + return AsyncMatch::MATCH; + } + } + + ScheduleFullHashCheck(std::move(check), full_hash_to_store_and_hash_prefixes); + return AsyncMatch::ASYNC; +} + bool V4LocalDatabaseManager::HandleCheck(std::unique_ptr<PendingCheck> check) { if (!v4_database_) { queued_checks_.push_back(std::move(check)); @@ -565,6 +685,14 @@ bool V4LocalDatabaseManager::HandleCheck(std::unique_ptr<PendingCheck> check) { return true; } + ScheduleFullHashCheck(std::move(check), full_hash_to_store_and_hash_prefixes); + return false; +} + +void V4LocalDatabaseManager::ScheduleFullHashCheck( + std::unique_ptr<PendingCheck> check, + const FullHashToStoreAndHashPrefixesMap& + full_hash_to_store_and_hash_prefixes) { // Add check to pending_checks_ before scheduling PerformFullHashCheck so that // even if the client calls CancelCheck before PerformFullHashCheck gets // called, the check can be found in pending_checks_. @@ -576,8 +704,6 @@ bool V4LocalDatabaseManager::HandleCheck(std::unique_ptr<PendingCheck> check) { base::Bind(&V4LocalDatabaseManager::PerformFullHashCheck, weak_factory_.GetWeakPtr(), base::Passed(std::move(check)), full_hash_to_store_and_hash_prefixes)); - - return false; } bool V4LocalDatabaseManager::HandleHashSynchronously( @@ -700,6 +826,16 @@ void V4LocalDatabaseManager::RespondToClient( check->matching_full_hash); break; + case ClientCallbackType::CHECK_CSD_WHITELIST: { + DCHECK_EQ(1u, check->urls.size()); + bool did_match_whitelist = + check->most_severe_threat_type == SB_THREAT_TYPE_CSD_WHITELIST; + DCHECK(did_match_whitelist || + check->most_severe_threat_type == SB_THREAT_TYPE_SAFE); + check->client->OnCheckWhitelistUrlResult(did_match_whitelist); + break; + } + case ClientCallbackType::CHECK_EXTENSION_IDS: { DCHECK_EQ(check->full_hash_threat_types.size(), check->full_hashes.size()); @@ -722,14 +858,6 @@ void V4LocalDatabaseManager::SetupDatabase() { DCHECK(!list_infos_.empty()); DCHECK_CURRENTLY_ON(BrowserThread::IO); - // Only get a new task runner if there isn't one already. If the service has - // previously been started and stopped, a task runner could already exist. - if (!task_runner_) { - base::SequencedWorkerPool* pool = BrowserThread::GetBlockingPool(); - task_runner_ = pool->GetSequencedTaskRunnerWithShutdownBehavior( - pool->GetSequenceToken(), base::SequencedWorkerPool::SKIP_ON_SHUTDOWN); - } - // Do not create the database on the IO thread since this may be an expensive // operation. Instead, do that on the task_runner and when the new database // has been created, swap it out on the IO thread. diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager.h b/chromium/components/safe_browsing_db/v4_local_database_manager.h index 9e3ea4fa00f..c35296e0f18 100644 --- a/chromium/components/safe_browsing_db/v4_local_database_manager.h +++ b/chromium/components/safe_browsing_db/v4_local_database_manager.h @@ -40,9 +40,13 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { void CancelCheck(Client* client) override; bool CanCheckResourceType(content::ResourceType resource_type) const override; + bool CanCheckSubresourceFilter() const override; bool CanCheckUrl(const GURL& url) const override; bool ChecksAreAlwaysAsync() const override; - bool CheckBrowseUrl(const GURL& url, Client* client) override; + bool CheckBrowseUrl(const GURL& url, + const SBThreatTypeSet& threat_types, + Client* client) override; + AsyncMatch CheckCsdWhitelistUrl(const GURL& url, Client* client) override; bool CheckDownloadUrl(const std::vector<GURL>& url_chain, Client* client) override; // TODO(vakh): |CheckExtensionIDs| in the base class accepts a set of @@ -88,28 +92,32 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { enum class ClientCallbackType { // This represents the case when we're trying to determine if a URL is // unsafe from the following perspectives: Malware, Phishing, UwS. - CHECK_BROWSE_URL = 0, + CHECK_BROWSE_URL, // This represents the case when we're trying to determine if any of the // URLs in a vector of URLs is unsafe for downloading binaries. - CHECK_DOWNLOAD_URLS = 1, + CHECK_DOWNLOAD_URLS, // This represents the case when we're trying to determine if a URL is an // unsafe resource. - CHECK_RESOURCE_URL = 2, + CHECK_RESOURCE_URL, // This represents the case when we're trying to determine if a Chrome // extension is a unsafe. - CHECK_EXTENSION_IDS = 3, + CHECK_EXTENSION_IDS, // This respresents the case when we're trying to determine if a URL belongs // to the list where subresource filter should be active. - CHECK_URL_FOR_SUBRESOURCE_FILTER = 4, + CHECK_URL_FOR_SUBRESOURCE_FILTER, + + // This respresents the case when we're trying to determine if a URL is + // part of the CSD whitelist. + CHECK_CSD_WHITELIST, // This represents the other cases when a check is being performed // synchronously so a client callback isn't required. For instance, when // trying to determing if an IP address is unsafe due to hosting Malware. - CHECK_OTHER = 5, + CHECK_OTHER, }; // The information we need to process a URL safety reputation request and @@ -193,6 +201,9 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { // Called when the database has been updated and schedules the next update. void DatabaseUpdated(); + // Delete any *.store files from disk that are no longer used. + void DeleteUnusedStoreFiles(); + // Identifies the prefixes and the store they matched in, for a given |check|. // Returns true if one or more hash prefix matches are found; false otherwise. bool GetPrefixMatches( @@ -221,6 +232,15 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { // schedules a task to perform full hash check and returns false. bool HandleCheck(std::unique_ptr<PendingCheck> check); + // Like HandleCheck, but for whitelists that have both full-hashes and + // partial hashes in the DB. Returns MATCH, NO_MATCH, or ASYNC. + AsyncMatch HandleWhitelistCheck(std::unique_ptr<PendingCheck> check); + + // Schedules a full-hash check for a given set of prefixes. + void ScheduleFullHashCheck(std::unique_ptr<PendingCheck> check, + const FullHashToStoreAndHashPrefixesMap& + full_hash_to_store_and_hash_prefixes); + // Checks |stores_to_check| in database synchronously for hash prefixes // matching |hash|. Returns true if there's a match; false otherwise. This is // used for lists that have full hash information in the database. @@ -314,7 +334,6 @@ class V4LocalDatabaseManager : public SafeBrowsingDatabaseManager { base::WeakPtrFactory<V4LocalDatabaseManager> weak_factory_; - friend class base::RefCountedThreadSafe<V4LocalDatabaseManager>; DISALLOW_COPY_AND_ASSIGN(V4LocalDatabaseManager); }; // class V4LocalDatabaseManager diff --git a/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc b/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc index 34cb4ac5887..f40d3410470 100644 --- a/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc +++ b/chromium/components/safe_browsing_db/v4_local_database_manager_unittest.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "components/safe_browsing_db/v4_local_database_manager.h" +#include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" #include "base/memory/ptr_util.h" #include "base/memory/ref_counted.h" @@ -11,6 +12,7 @@ #include "base/test/test_simple_task_runner.h" #include "base/threading/thread_task_runner_handle.h" #include "components/safe_browsing_db/v4_database.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" #include "components/safe_browsing_db/v4_test_util.h" #include "content/public/test/test_browser_thread_bundle.h" #include "crypto/sha2.h" @@ -31,7 +33,7 @@ FullHash HashForUrl(const GURL& url) { return full_hashes[0]; } -// Always returns misses from GetFullHashes(). +// Use this if you want GetFullHashes() to always return prescribed results. class FakeGetHashProtocolManager : public V4GetHashProtocolManager { public: FakeGetHashProtocolManager( @@ -74,6 +76,7 @@ class FakeGetHashProtocolManagerFactory }; // Use FakeGetHashProtocolManagerFactory in scope, then reset. +// You should make sure the DatabaseManager is created _after_ this. class ScopedFakeGetHashProtocolManagerFactory { public: ScopedFakeGetHashProtocolManagerFactory( @@ -113,6 +116,8 @@ class FakeV4Database : public V4Database { StoreAndHashPrefixes* store_and_hash_prefixes) override { store_and_hash_prefixes->clear(); for (const StoreAndHashPrefix& stored_sahp : store_and_hash_prefixes_) { + if (stores_to_check.count(stored_sahp.list_id) == 0) + continue; const PrefixSize& prefix_size = stored_sahp.hash_prefix.size(); if (!full_hash.compare(0, prefix_size, stored_sahp.hash_prefix)) { store_and_hash_prefixes->push_back(stored_sahp); @@ -159,6 +164,8 @@ class FakeV4Database : public V4Database { const bool stores_available_; }; +// TODO(nparker): This might be simpler with a mock and EXPECT calls. +// That would also catch unexpected calls. class TestClient : public SafeBrowsingDatabaseManager::Client { public: TestClient(SBThreatType sb_threat_type, @@ -166,17 +173,10 @@ class TestClient : public SafeBrowsingDatabaseManager::Client { V4LocalDatabaseManager* manager_to_cancel = nullptr) : expected_sb_threat_type(sb_threat_type), expected_urls(1, url), - on_check_browse_url_result_called_(false), - on_check_download_urls_result_called_(false), - on_check_resource_url_result_called_(false), manager_to_cancel_(manager_to_cancel) {} TestClient(SBThreatType sb_threat_type, const std::vector<GURL>& url_chain) - : expected_sb_threat_type(sb_threat_type), - expected_urls(url_chain), - on_check_browse_url_result_called_(false), - on_check_download_urls_result_called_(false), - on_check_resource_url_result_called_(false) {} + : expected_sb_threat_type(sb_threat_type), expected_urls(url_chain) {} void OnCheckBrowseUrlResult(const GURL& url, SBThreatType threat_type, @@ -197,6 +197,7 @@ class TestClient : public SafeBrowsingDatabaseManager::Client { ASSERT_EQ(threat_type == SB_THREAT_TYPE_SAFE, threat_hash.empty()); on_check_resource_url_result_called_ = true; } + void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain, SBThreatType threat_type) override { ASSERT_EQ(expected_urls, url_chain); @@ -206,12 +207,26 @@ class TestClient : public SafeBrowsingDatabaseManager::Client { SBThreatType expected_sb_threat_type; std::vector<GURL> expected_urls; - bool on_check_browse_url_result_called_; - bool on_check_download_urls_result_called_; - bool on_check_resource_url_result_called_; + bool on_check_browse_url_result_called_ = false; + bool on_check_download_urls_result_called_ = false; + bool on_check_resource_url_result_called_ = false; V4LocalDatabaseManager* manager_to_cancel_; }; +class TestWhitelistClient : public SafeBrowsingDatabaseManager::Client { + public: + explicit TestWhitelistClient(bool whitelist_expected) + : whitelist_expected_(whitelist_expected) {} + + void OnCheckWhitelistUrlResult(bool is_whitelisted) override { + EXPECT_EQ(whitelist_expected_, is_whitelisted); + callback_called_ = true; + } + + const bool whitelist_expected_; + bool callback_called_ = false; +}; + class TestExtensionClient : public SafeBrowsingDatabaseManager::Client { public: TestExtensionClient(const std::set<FullHash>& expected_bad_crxs) @@ -229,18 +244,19 @@ class TestExtensionClient : public SafeBrowsingDatabaseManager::Client { class FakeV4LocalDatabaseManager : public V4LocalDatabaseManager { public: - void PerformFullHashCheck(std::unique_ptr<PendingCheck> check, - const FullHashToStoreAndHashPrefixesMap& - full_hash_to_store_and_hash_prefixes) override { - perform_full_hash_check_called_ = true; - } - FakeV4LocalDatabaseManager( const base::FilePath& base_path, ExtendedReportingLevelCallback extended_reporting_level_callback) : V4LocalDatabaseManager(base_path, extended_reporting_level_callback), perform_full_hash_check_called_(false) {} + // V4LocalDatabaseManager impl: + void PerformFullHashCheck(std::unique_ptr<PendingCheck> check, + const FullHashToStoreAndHashPrefixesMap& + full_hash_to_store_and_hash_prefixes) override { + perform_full_hash_check_called_ = true; + } + static bool PerformFullHashCheckCalled( scoped_refptr<safe_browsing::V4LocalDatabaseManager>& v4_ldbm) { FakeV4LocalDatabaseManager* fake = @@ -369,6 +385,10 @@ class V4LocalDatabaseManagerTest : public PlatformTest { WaitForTasksOnTaskRunner(); } + const SBThreatTypeSet usual_threat_types_ = CreateSBThreatTypeSet( + {SB_THREAT_TYPE_URL_PHISHING, SB_THREAT_TYPE_URL_MALWARE, + SB_THREAT_TYPE_URL_UNWANTED}); + base::ScopedTempDir base_dir_; ExtendedReportingLevel extended_reporting_level_; ExtendedReportingLevelCallback erl_callback_; @@ -405,7 +425,7 @@ TEST_F(V4LocalDatabaseManagerTest, WaitForTasksOnTaskRunner(); // Both the stores are empty right now so CheckBrowseUrl should return true. EXPECT_TRUE(v4_local_database_manager_->CheckBrowseUrl( - GURL("http://example.com/a/"), nullptr)); + GURL("http://example.com/a/"), usual_threat_types_, nullptr)); } TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithFakeDbReturnsMatch) { @@ -420,10 +440,132 @@ TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithFakeDbReturnsMatch) { ReplaceV4Database(store_and_hash_prefixes); const GURL url_bad("https://" + url_bad_no_scheme); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url_bad, nullptr)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url_bad, usual_threat_types_, nullptr)); + + // Wait for PerformFullHashCheck to complete. + WaitForTasksOnTaskRunner(); +} + +TEST_F(V4LocalDatabaseManagerTest, TestCheckCsdWhitelistWithPrefixMatch) { + // Setup to receive full-hash misses. We won't make URL requests. + ScopedFakeGetHashProtocolManagerFactory pin(FullHashInfos({})); + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + std::string url_white_no_scheme("example.com/white/"); + FullHash white_full_hash(crypto::SHA256HashString(url_white_no_scheme)); + const HashPrefix white_hash_prefix(white_full_hash.substr(0, 5)); + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlCsdWhitelistId(), + white_hash_prefix); + ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */); + + TestWhitelistClient client(false /* whitelist_expected */); + const GURL url_check("https://" + url_white_no_scheme); + EXPECT_EQ(AsyncMatch::ASYNC, v4_local_database_manager_->CheckCsdWhitelistUrl( + url_check, &client)); + + EXPECT_FALSE(client.callback_called_); // Wait for PerformFullHashCheck to complete. WaitForTasksOnTaskRunner(); + EXPECT_TRUE(client.callback_called_); +} + +// This is like CsdWhitelistWithPrefixMatch, but we also verify the +// full-hash-match results in an appropriate callback value. +TEST_F(V4LocalDatabaseManagerTest, + TestCheckCsdWhitelistWithPrefixTheFullMatch) { + std::string url_white_no_scheme("example.com/white/"); + FullHash white_full_hash(crypto::SHA256HashString(url_white_no_scheme)); + + // Setup to receive full-hash hit. We won't make URL requests. + FullHashInfos infos( + {{white_full_hash, GetUrlCsdWhitelistId(), base::Time::Now()}}); + ScopedFakeGetHashProtocolManagerFactory pin(infos); + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + const HashPrefix white_hash_prefix(white_full_hash.substr(0, 5)); + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlCsdWhitelistId(), + white_hash_prefix); + ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */); + + TestWhitelistClient client(true /* whitelist_expected */); + const GURL url_check("https://" + url_white_no_scheme); + EXPECT_EQ(AsyncMatch::ASYNC, v4_local_database_manager_->CheckCsdWhitelistUrl( + url_check, &client)); + + EXPECT_FALSE(client.callback_called_); + + // Wait for PerformFullHashCheck to complete. + WaitForTasksOnTaskRunner(); + EXPECT_TRUE(client.callback_called_); +} + +TEST_F(V4LocalDatabaseManagerTest, TestCheckCsdWhitelistWithFullMatch) { + // Setup to receive full-hash misses. We won't make URL requests. + ScopedFakeGetHashProtocolManagerFactory pin(FullHashInfos({})); + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + std::string url_white_no_scheme("example.com/white/"); + FullHash white_full_hash(crypto::SHA256HashString(url_white_no_scheme)); + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlCsdWhitelistId(), white_full_hash); + ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */); + + TestWhitelistClient client(false /* whitelist_expected */); + const GURL url_check("https://" + url_white_no_scheme); + EXPECT_EQ(AsyncMatch::MATCH, v4_local_database_manager_->CheckCsdWhitelistUrl( + url_check, &client)); + + WaitForTasksOnTaskRunner(); + EXPECT_FALSE(client.callback_called_); +} + +TEST_F(V4LocalDatabaseManagerTest, TestCheckCsdWhitelistWithNoMatch) { + // Setup to receive full-hash misses. We won't make URL requests. + ScopedFakeGetHashProtocolManagerFactory pin(FullHashInfos({})); + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + // Add a full hash that won't match the URL we check. + std::string url_white_no_scheme("example.com/white/"); + FullHash white_full_hash(crypto::SHA256HashString(url_white_no_scheme)); + StoreAndHashPrefixes store_and_hash_prefixes; + store_and_hash_prefixes.emplace_back(GetUrlMalwareId(), white_full_hash); + ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */); + + TestWhitelistClient client(true /* whitelist_expected */); + const GURL url_check("https://other.com/"); + EXPECT_EQ( + AsyncMatch::NO_MATCH, + v4_local_database_manager_->CheckCsdWhitelistUrl(url_check, &client)); + + WaitForTasksOnTaskRunner(); + EXPECT_FALSE(client.callback_called_); +} + +// When whitelist is unavailable, all URLS should be whitelisted. +TEST_F(V4LocalDatabaseManagerTest, TestCheckCsdWhitelistUnavailable) { + // Setup to receive full-hash misses. We won't make URL requests. + ScopedFakeGetHashProtocolManagerFactory pin(FullHashInfos({})); + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + + StoreAndHashPrefixes store_and_hash_prefixes; + ReplaceV4Database(store_and_hash_prefixes, false /* stores_available */); + + TestWhitelistClient client(false /* whitelist_expected */); + const GURL url_check("https://other.com/"); + EXPECT_EQ(AsyncMatch::MATCH, v4_local_database_manager_->CheckCsdWhitelistUrl( + url_check, &client)); + + WaitForTasksOnTaskRunner(); + EXPECT_FALSE(client.callback_called_); } TEST_F(V4LocalDatabaseManagerTest, @@ -435,7 +577,7 @@ TEST_F(V4LocalDatabaseManagerTest, ForceDisableLocalDatabaseManager(); EXPECT_TRUE(v4_local_database_manager_->CheckBrowseUrl( - GURL("http://example.com/a/"), nullptr)); + GURL("http://example.com/a/"), usual_threat_types_, nullptr)); } TEST_F(V4LocalDatabaseManagerTest, TestGetSeverestThreatTypeAndMetadata) { @@ -489,7 +631,7 @@ TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) { const GURL url("https://www.example.com/"); TestClient client(SB_THREAT_TYPE_SAFE, url); EXPECT_TRUE(GetQueuedChecks().empty()); - v4_local_database_manager_->CheckBrowseUrl(url, &client); + v4_local_database_manager_->CheckBrowseUrl(url, usual_threat_types_, &client); // The database is unavailable so the check should get queued. EXPECT_EQ(1ul, GetQueuedChecks().size()); @@ -498,7 +640,7 @@ TEST_F(V4LocalDatabaseManagerTest, TestChecksAreQueued) { EXPECT_TRUE(GetQueuedChecks().empty()); ResetV4Database(); - v4_local_database_manager_->CheckBrowseUrl(url, &client); + v4_local_database_manager_->CheckBrowseUrl(url, usual_threat_types_, &client); // The database is unavailable so the check should get queued. EXPECT_EQ(1ul, GetQueuedChecks().size()); @@ -527,7 +669,8 @@ TEST_F(V4LocalDatabaseManagerTest, CancelPending) { // Test that a request flows through to the callback. { TestClient client(SB_THREAT_TYPE_SAFE, url_bad); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url_bad, &client)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url_bad, usual_threat_types_, &client)); EXPECT_FALSE(client.on_check_browse_url_result_called_); WaitForTasksOnTaskRunner(); EXPECT_TRUE(client.on_check_browse_url_result_called_); @@ -536,7 +679,8 @@ TEST_F(V4LocalDatabaseManagerTest, CancelPending) { // Test that cancel prevents the callback from being called. { TestClient client(SB_THREAT_TYPE_SAFE, url_bad); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url_bad, &client)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url_bad, usual_threat_types_, &client)); v4_local_database_manager_->CancelCheck(&client); EXPECT_FALSE(client.on_check_browse_url_result_called_); WaitForTasksOnTaskRunner(); @@ -552,8 +696,10 @@ TEST_F(V4LocalDatabaseManagerTest, CancelQueued) { TestClient client1(SB_THREAT_TYPE_SAFE, url, v4_local_database_manager_.get()); TestClient client2(SB_THREAT_TYPE_SAFE, url); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client1)); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url, &client2)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url, usual_threat_types_, &client1)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url, usual_threat_types_, &client2)); EXPECT_EQ(2ul, GetQueuedChecks().size()); EXPECT_FALSE(client1.on_check_browse_url_result_called_); EXPECT_FALSE(client2.on_check_browse_url_result_called_); @@ -578,7 +724,8 @@ TEST_F(V4LocalDatabaseManagerTest, PerformFullHashCheckCalledAsync) { const GURL url_bad("https://" + url_bad_no_scheme); // The fake database returns a matched hash prefix. - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url_bad, nullptr)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url_bad, usual_threat_types_, nullptr)); EXPECT_FALSE(FakeV4LocalDatabaseManager::PerformFullHashCheckCalled( v4_local_database_manager_)); @@ -602,7 +749,8 @@ TEST_F(V4LocalDatabaseManagerTest, UsingWeakPtrDropsCallback) { ReplaceV4Database(store_and_hash_prefixes); const GURL url_bad("https://" + url_bad_no_scheme); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(url_bad, nullptr)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + url_bad, usual_threat_types_, nullptr)); v4_local_database_manager_->StopOnIOThread(true); // Release the V4LocalDatabaseManager object right away before the callback @@ -671,7 +819,7 @@ TEST_F(V4LocalDatabaseManagerTest, TestMatchDownloadWhitelistUrl) { GURL other_url("http://iffy.com"); StoreAndHashPrefixes store_and_hash_prefixes; - store_and_hash_prefixes.emplace_back(GetCertCsdDownloadWhitelistId(), + store_and_hash_prefixes.emplace_back(GetUrlCsdDownloadWhitelistId(), HashForUrl(good_url)); ReplaceV4Database(store_and_hash_prefixes, false /* not available */); @@ -753,7 +901,8 @@ TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithSameClientAndCancel) { GURL second_url("http://example.com/"); TestClient client(SB_THREAT_TYPE_SAFE, first_url); // The fake database returns a matched hash prefix. - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(first_url, &client)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + first_url, usual_threat_types_, &client)); // That check gets queued. Now, let's cancel the check. After this, we should // not receive a call for |OnCheckBrowseUrlResult| with |first_url|. @@ -761,7 +910,8 @@ TEST_F(V4LocalDatabaseManagerTest, TestCheckBrowseUrlWithSameClientAndCancel) { // Now, re-use that client but for |second_url|. client.expected_urls.assign(1, second_url); - EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl(second_url, &client)); + EXPECT_FALSE(v4_local_database_manager_->CheckBrowseUrl( + second_url, usual_threat_types_, &client)); // Wait for PerformFullHashCheck to complete. WaitForTasksOnTaskRunner(); @@ -958,7 +1108,7 @@ TEST_F(V4LocalDatabaseManagerTest, TestCheckDownloadUrlWithOneBlacklisted) { store_and_hash_prefixes.emplace_back(GetUrlMalBinId(), bad_hash_prefix); ReplaceV4Database(store_and_hash_prefixes, true /* stores_available */); - TestClient client(SB_THREAT_TYPE_BINARY_MALWARE_URL, url_chain); + TestClient client(SB_THREAT_TYPE_URL_BINARY_MALWARE, url_chain); EXPECT_FALSE( v4_local_database_manager_->CheckDownloadUrl(url_chain, &client)); EXPECT_FALSE(client.on_check_download_urls_result_called_); @@ -966,4 +1116,45 @@ TEST_F(V4LocalDatabaseManagerTest, TestCheckDownloadUrlWithOneBlacklisted) { EXPECT_TRUE(client.on_check_download_urls_result_called_); } +TEST_F(V4LocalDatabaseManagerTest, DeleteUnusedStoreFileDoesNotExist) { + auto store_file_path = base_dir_.GetPath().AppendASCII("AnyIpMalware.store"); + ASSERT_FALSE(base::PathExists(store_file_path)); + + // Reset the database manager so that DeleteUnusedStoreFiles is called. + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + ASSERT_FALSE(base::PathExists(store_file_path)); +} + +TEST_F(V4LocalDatabaseManagerTest, DeleteUnusedStoreFileSuccess) { + auto store_file_path = base_dir_.GetPath().AppendASCII("AnyIpMalware.store"); + ASSERT_FALSE(base::PathExists(store_file_path)); + + // Now write an empty file. + base::WriteFile(store_file_path, "", 0); + ASSERT_TRUE(base::PathExists(store_file_path)); + + // Reset the database manager so that DeleteUnusedStoreFiles is called. + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + ASSERT_FALSE(base::PathExists(store_file_path)); +} + +TEST_F(V4LocalDatabaseManagerTest, DeleteUnusedStoreFileRandomFileNotDeleted) { + auto random_store_file_path = base_dir_.GetPath().AppendASCII("random.store"); + ASSERT_FALSE(base::PathExists(random_store_file_path)); + + // Now write an empty file. + base::WriteFile(random_store_file_path, "", 0); + ASSERT_TRUE(base::PathExists(random_store_file_path)); + + // Reset the database manager so that DeleteUnusedStoreFiles is called. + ResetLocalDatabaseManager(); + WaitForTasksOnTaskRunner(); + ASSERT_TRUE(base::PathExists(random_store_file_path)); + + // Cleanup + base::DeleteFile(random_store_file_path, false /* recursive */); +} + } // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc b/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc index bd097a72b58..5153d4a5a98 100644 --- a/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc +++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util.cc @@ -21,6 +21,10 @@ using base::Time; using base::TimeDelta; namespace safe_browsing { +const base::FilePath::CharType kStoreSuffix[] = FILE_PATH_LITERAL(".store"); + +// The Safe Browsing V4 server URL prefix. +const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4"; namespace { @@ -141,8 +145,11 @@ ListIdentifier GetUrlUwsId() { return ListIdentifier(GetCurrentPlatformType(), URL, UNWANTED_SOFTWARE); } -// The Safe Browsing V4 server URL prefix. -const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4"; +std::string GetUmaSuffixForStore(const base::FilePath& file_path) { + DCHECK_EQ(kStoreSuffix, file_path.BaseName().Extension()); + return base::StringPrintf( + ".%" PRIsFP, file_path.BaseName().RemoveExtension().value().c_str()); +} StoreAndHashPrefix::StoreAndHashPrefix(ListIdentifier list_id, const HashPrefix& hash_prefix) @@ -165,6 +172,21 @@ size_t StoreAndHashPrefix::hash() const { return base::HashInts(first, second); } +bool SBThreatTypeSetIsValidForCheckBrowseUrl(const SBThreatTypeSet& set) { + for (SBThreatType type : set) { + switch (type) { + case SB_THREAT_TYPE_URL_PHISHING: + case SB_THREAT_TYPE_URL_MALWARE: + case SB_THREAT_TYPE_URL_UNWANTED: + break; + + default: + return false; + } + } + return true; +} + bool ListIdentifier::operator==(const ListIdentifier& other) const { return platform_type_ == other.platform_type_ && threat_entry_type_ == other.threat_entry_type_ && diff --git a/chromium/components/safe_browsing_db/v4_protocol_manager_util.h b/chromium/components/safe_browsing_db/v4_protocol_manager_util.h index 882fb4a45cc..9c083542bdd 100644 --- a/chromium/components/safe_browsing_db/v4_protocol_manager_util.h +++ b/chromium/components/safe_browsing_db/v4_protocol_manager_util.h @@ -8,6 +8,7 @@ // A class that implements the stateless methods used by the GetHashUpdate and // GetFullHash stubby calls made by Chrome using the SafeBrowsing V4 protocol. +#include <initializer_list> #include <memory> #include <ostream> #include <string> @@ -15,6 +16,7 @@ #include <unordered_set> #include <vector> +#include "base/containers/flat_set.h" #include "base/gtest_prod_util.h" #include "base/strings/string_piece.h" #include "components/safe_browsing_db/safebrowsing.pb.h" @@ -75,6 +77,8 @@ struct V4ProtocolConfig { // Different types of threats that SafeBrowsing protects against. This is the // type that's returned to the clients of SafeBrowsing in Chromium. +// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.components.safe_browsing +// GENERATED_JAVA_PREFIX_TO_STRIP: SB_THREAT_TYPE_ enum SBThreatType { // This type can be used for lists that can be checked synchronously so a // client callback isn't required, or for whitelists. @@ -93,18 +97,18 @@ enum SBThreatType { SB_THREAT_TYPE_URL_UNWANTED, // The download URL is malware. - SB_THREAT_TYPE_BINARY_MALWARE_URL, + SB_THREAT_TYPE_URL_BINARY_MALWARE, // Url detected by the client-side phishing model. Note that unlike the // above values, this does not correspond to a downloaded list. - SB_THREAT_TYPE_CLIENT_SIDE_PHISHING_URL, + SB_THREAT_TYPE_URL_CLIENT_SIDE_PHISHING, // The Chrome extension or app (given by its ID) is malware. SB_THREAT_TYPE_EXTENSION, // Url detected by the client-side malware IP list. This IP list is part // of the client side detection model. - SB_THREAT_TYPE_CLIENT_SIDE_MALWARE_URL, + SB_THREAT_TYPE_URL_CLIENT_SIDE_MALWARE, // Url leads to a blacklisted resource script. Note that no warnings should be // shown on this threat type, but an incident report might be sent. @@ -115,8 +119,28 @@ enum SBThreatType { // Activation patterns for the Subresource Filter. SB_THREAT_TYPE_SUBRESOURCE_FILTER, + + // CSD Phishing whitelist. This "threat" means a URL matched the whitelist. + SB_THREAT_TYPE_CSD_WHITELIST, + + // Url detected by password protection service. + SB_THREAT_TYPE_URL_PASSWORD_PROTECTION_PHISHING, }; +using SBThreatTypeSet = base::flat_set<SBThreatType>; + +// Return true if |set| only contains types that are valid for CheckBrowseUrl(). +// Intended for use in DCHECK(). +bool SBThreatTypeSetIsValidForCheckBrowseUrl(const SBThreatTypeSet& set); + +// Shorthand for creating an SBThreatTypeSet from a list of SBThreatTypes. Use +// like CreateSBThreatTypeSet({SB_THREAT_TYPE_URL_PHISHING, +// SB_THREAT_TYPE_URL_MALWARE}) +inline SBThreatTypeSet CreateSBThreatTypeSet( + std::initializer_list<SBThreatType> set) { + return SBThreatTypeSet(set, base::KEEP_FIRST_OF_DUPES); +} + // The information required to uniquely identify each list the client is // interested in maintaining and downloading from the SafeBrowsing servers. // For example, for digests of Malware binaries on Windows: @@ -163,6 +187,9 @@ ListIdentifier GetUrlSocEngId(); ListIdentifier GetUrlSubresourceFilterId(); ListIdentifier GetUrlUwsId(); +// Returns the basename of the store file, without the ".store" extension. +std::string GetUmaSuffixForStore(const base::FilePath& file_path); + // Represents the state of each store. using StoreStateMap = std::unordered_map<ListIdentifier, std::string>; diff --git a/chromium/components/safe_browsing_db/v4_store.cc b/chromium/components/safe_browsing_db/v4_store.cc index fe3220a1f2b..a1bc3070143 100644 --- a/chromium/components/safe_browsing_db/v4_store.cc +++ b/chromium/components/safe_browsing_db/v4_store.cc @@ -46,11 +46,6 @@ const char kTime[] = ".Time"; const uint32_t kFileMagic = 0x600D71FE; const uint32_t kFileVersion = 9; -std::string GetUmaSuffixForStore(const base::FilePath& file_path) { - return base::StringPrintf( - ".%" PRIsFP, file_path.BaseName().RemoveExtension().value().c_str()); -} - void RecordTimeWithAndWithoutSuffix(const std::string& metric, base::TimeDelta time, const base::FilePath& file_path) { @@ -208,7 +203,7 @@ V4Store::V4Store(const scoped_refptr<base::SequencedTaskRunner>& task_runner, task_runner_(task_runner) {} V4Store::~V4Store() { - DCHECK(task_runner_->RunsTasksOnCurrentThread()); + DCHECK(task_runner_->RunsTasksInCurrentSequence()); } std::string V4Store::DebugString() const { @@ -533,7 +528,7 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, const HashPrefixMap& additions_map, const RepeatedField<int32>* raw_removals, const std::string& expected_checksum) { - DCHECK(task_runner_->RunsTasksOnCurrentThread()); + DCHECK(task_runner_->RunsTasksInCurrentSequence()); DCHECK(hash_prefix_map_.empty()); bool calculate_checksum = !expected_checksum.empty(); @@ -665,7 +660,7 @@ ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map, } StoreReadResult V4Store::ReadFromDisk() { - DCHECK(task_runner_->RunsTasksOnCurrentThread()); + DCHECK(task_runner_->RunsTasksInCurrentSequence()); V4StoreFileFormat file_format; int64_t file_size; @@ -801,7 +796,7 @@ bool V4Store::HashPrefixMatches(const HashPrefix& hash_prefix, } bool V4Store::VerifyChecksum() { - DCHECK(task_runner_->RunsTasksOnCurrentThread()); + DCHECK(task_runner_->RunsTasksInCurrentSequence()); if (expected_checksum_.empty()) { // Nothing to check here folks! diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc b/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc index 3d485bdd6bf..ddd08f588b6 100644 --- a/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc +++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager.cc @@ -153,7 +153,9 @@ V4UpdateProtocolManager::V4UpdateProtocolManager( // when it is ready to process updates. } -V4UpdateProtocolManager::~V4UpdateProtocolManager() {} +V4UpdateProtocolManager::~V4UpdateProtocolManager() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); +} bool V4UpdateProtocolManager::IsUpdateScheduled() const { return update_timer_.IsRunning(); @@ -166,7 +168,7 @@ void V4UpdateProtocolManager::ScheduleNextUpdate( } void V4UpdateProtocolManager::ScheduleNextUpdateWithBackoff(bool back_off) { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); if (config_.disable_auto_update) { DCHECK(!IsUpdateScheduled()); @@ -181,7 +183,7 @@ void V4UpdateProtocolManager::ScheduleNextUpdateWithBackoff(bool back_off) { // According to section 5 of the SafeBrowsing protocol specification, we must // back off after a certain number of errors. base::TimeDelta V4UpdateProtocolManager::GetNextUpdateInterval(bool back_off) { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(next_update_interval_ > base::TimeDelta()); base::TimeDelta next = next_update_interval_; @@ -208,7 +210,7 @@ base::TimeDelta V4UpdateProtocolManager::GetNextUpdateInterval(bool back_off) { void V4UpdateProtocolManager::ScheduleNextUpdateAfterInterval( base::TimeDelta interval) { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(interval >= base::TimeDelta()); // Unschedule any current timer. @@ -299,7 +301,7 @@ bool V4UpdateProtocolManager::ParseUpdateResponse( } void V4UpdateProtocolManager::IssueUpdateRequest() { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); // If an update request is already pending, record and return silently. if (request_.get()) { @@ -370,7 +372,7 @@ void V4UpdateProtocolManager::HandleTimeout() { // SafeBrowsing request responses are handled here. void V4UpdateProtocolManager::OnURLFetchComplete( const net::URLFetcher* source) { - DCHECK(CalledOnValidThread()); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); timeout_timer_.Stop(); diff --git a/chromium/components/safe_browsing_db/v4_update_protocol_manager.h b/chromium/components/safe_browsing_db/v4_update_protocol_manager.h index 5c60a0c398d..706c7d4c331 100644 --- a/chromium/components/safe_browsing_db/v4_update_protocol_manager.h +++ b/chromium/components/safe_browsing_db/v4_update_protocol_manager.h @@ -18,7 +18,7 @@ #include "base/gtest_prod_util.h" #include "base/macros.h" -#include "base/threading/non_thread_safe.h" +#include "base/sequence_checker.h" #include "base/time/time.h" #include "base/timer/timer.h" #include "components/safe_browsing/common/safe_browsing_prefs.h" @@ -47,8 +47,7 @@ typedef base::Callback<void(std::unique_ptr<ParsedServerResponse>)> typedef base::Callback<ExtendedReportingLevel()> ExtendedReportingLevelCallback; -class V4UpdateProtocolManager : public net::URLFetcherDelegate, - public base::NonThreadSafe { +class V4UpdateProtocolManager : public net::URLFetcherDelegate { public: ~V4UpdateProtocolManager() override; @@ -192,6 +191,8 @@ class V4UpdateProtocolManager : public net::URLFetcherDelegate, ExtendedReportingLevelCallback extended_reporting_level_callback_; + SEQUENCE_CHECKER(sequence_checker_); + DISALLOW_COPY_AND_ASSIGN(V4UpdateProtocolManager); }; diff --git a/chromium/components/safe_browsing_db/whitelist_checker_client.cc b/chromium/components/safe_browsing_db/whitelist_checker_client.cc new file mode 100644 index 00000000000..4691ddd17e1 --- /dev/null +++ b/chromium/components/safe_browsing_db/whitelist_checker_client.cc @@ -0,0 +1,74 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/safe_browsing_db/whitelist_checker_client.h" + +#include "base/bind.h" + +namespace safe_browsing { + +// Static +void WhitelistCheckerClient::StartCheckCsdWhitelist( + scoped_refptr<SafeBrowsingDatabaseManager> database_manager, + const GURL& url, + base::Callback<void(bool)> callback_for_result) { + // TODO(nparker): Maybe also call SafeBrowsingDatabaseManager::CanCheckUrl() + if (!url.is_valid()) { + callback_for_result.Run(true /* is_whitelisted */); + return; + } + + // Make a client for each request. The caller could have several in + // flight at once. + std::unique_ptr<WhitelistCheckerClient> client = + base::MakeUnique<WhitelistCheckerClient>(callback_for_result, + database_manager); + AsyncMatch match = database_manager->CheckCsdWhitelistUrl(url, client.get()); + + switch (match) { + case AsyncMatch::MATCH: + callback_for_result.Run(true /* is_whitelisted */); + break; + case AsyncMatch::NO_MATCH: + callback_for_result.Run(false /* is_whitelisted */); + break; + case AsyncMatch::ASYNC: + // Client is now self-owned. When it gets called back with the result, + // it will delete itself. + client.release(); + break; + } +} + +WhitelistCheckerClient::WhitelistCheckerClient( + base::Callback<void(bool)> callback_for_result, + scoped_refptr<SafeBrowsingDatabaseManager> database_manager) + : callback_for_result_(callback_for_result), + database_manager_(database_manager), + weak_factory_(this) { + // Set a timer to fail open, i.e. call it "whitelisted", if the full + // check takes too long. + auto timeout_callback = + base::Bind(&WhitelistCheckerClient::OnCheckWhitelistUrlTimeout, + weak_factory_.GetWeakPtr()); + timer_.Start(FROM_HERE, base::TimeDelta::FromMilliseconds(kTimeoutMsec), + timeout_callback); +} + +WhitelistCheckerClient::~WhitelistCheckerClient() {} + +// SafeBrowsingDatabaseMananger::Client impl +void WhitelistCheckerClient::OnCheckWhitelistUrlResult(bool is_whitelisted) { + timer_.Stop(); + callback_for_result_.Run(is_whitelisted); + // This method is invoked only if we're already self-owned. + delete this; +} + +void WhitelistCheckerClient::OnCheckWhitelistUrlTimeout() { + database_manager_->CancelCheck(this); + this->OnCheckWhitelistUrlResult(true /* is_whitelisted */); +} + +} // namespace safe_browsing diff --git a/chromium/components/safe_browsing_db/whitelist_checker_client.h b/chromium/components/safe_browsing_db/whitelist_checker_client.h new file mode 100644 index 00000000000..c28782618d3 --- /dev/null +++ b/chromium/components/safe_browsing_db/whitelist_checker_client.h @@ -0,0 +1,55 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_SAFE_BROWSING_DB_WHITELIST_CHECKER_CLIENT_H_ +#define COMPONENTS_SAFE_BROWSING_DB_WHITELIST_CHECKER_CLIENT_H_ + +#include "base/memory/ptr_util.h" +#include "base/memory/weak_ptr.h" +#include "base/timer/timer.h" +#include "components/safe_browsing_db/database_manager.h" + +namespace safe_browsing { + +// This provides a simpler interface to +// SafeBrowsingDatabaseManager::CheckCsdWhitelistUrl() for callers that +// don't want to track their own clients. + +class WhitelistCheckerClient : public SafeBrowsingDatabaseManager::Client { + public: + using BoolCallback = base::Callback<void(bool /* is_whitelisted */)>; + + // Static method to instantiate and start a check. The callback will + // be invoked when it's done, times out, or if database_manager gets + // shut down. Must be called on IO thread. + static void StartCheckCsdWhitelist( + scoped_refptr<SafeBrowsingDatabaseManager> database_manager, + const GURL& url, + BoolCallback callback_for_result); + + WhitelistCheckerClient( + BoolCallback callback_for_result, + scoped_refptr<SafeBrowsingDatabaseManager> database_manager); + ~WhitelistCheckerClient() override; + + // SafeBrowsingDatabaseMananger::Client impl + void OnCheckWhitelistUrlResult(bool is_whitelisted) override; + + protected: + static const int kTimeoutMsec = 5000; + base::OneShotTimer timer_; + BoolCallback callback_for_result_; + scoped_refptr<SafeBrowsingDatabaseManager> database_manager_; + base::WeakPtrFactory<WhitelistCheckerClient> weak_factory_; + + private: + WhitelistCheckerClient(); + + // Called when the call to CheckCsdWhitelistUrl times out. + void OnCheckWhitelistUrlTimeout(); +}; + +} // namespace safe_browsing + +#endif // COMPONENTS_SAFE_BROWSING_DB_WHITELIST_CHECKER_CLIENT_H_ diff --git a/chromium/components/safe_browsing_db/whitelist_checker_client_unittest.cc b/chromium/components/safe_browsing_db/whitelist_checker_client_unittest.cc new file mode 100644 index 00000000000..0b78df22d9a --- /dev/null +++ b/chromium/components/safe_browsing_db/whitelist_checker_client_unittest.cc @@ -0,0 +1,132 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +#include "components/safe_browsing_db/whitelist_checker_client.h" + +#include "base/bind.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/test/mock_callback.h" +#include "base/test/test_mock_time_task_runner.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/safe_browsing_db/test_database_manager.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/test/test_browser_thread.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace safe_browsing { + +using base::TimeDelta; +using testing::_; +using testing::Return; +using testing::SaveArg; + +using BoolCallback = base::Callback<void(bool /* is_whitelisted */)>; +using MockBoolCallback = testing::StrictMock<base::MockCallback<BoolCallback>>; + +namespace { +class MockSafeBrowsingDatabaseManager : public TestSafeBrowsingDatabaseManager { + public: + MockSafeBrowsingDatabaseManager() {} + + MOCK_METHOD1(CancelCheck, void(SafeBrowsingDatabaseManager::Client*)); + + MOCK_METHOD2(CheckCsdWhitelistUrl, + AsyncMatch(const GURL&, SafeBrowsingDatabaseManager::Client*)); + + protected: + ~MockSafeBrowsingDatabaseManager() override {} + + private: + DISALLOW_COPY_AND_ASSIGN(MockSafeBrowsingDatabaseManager); +}; +} // namespace + +class WhitelistCheckerClientTest : public testing::Test { + public: + WhitelistCheckerClientTest() : target_url_("http://foo.bar") {} + + void SetUp() override { + database_manager_ = new MockSafeBrowsingDatabaseManager; + task_runner_ = new base::TestMockTimeTaskRunner(base::Time::Now(), + base::TimeTicks::Now()); + message_loop_.reset(new base::MessageLoop); + io_thread_ = base::MakeUnique<content::TestBrowserThread>( + content::BrowserThread::IO, base::MessageLoop::current()); + message_loop_->SetTaskRunner(task_runner_); + } + + void TearDown() override { + database_manager_ = nullptr; + base::RunLoop().RunUntilIdle(); + + // Verify no callback is remaining. + // TODO(nparker): We should somehow EXPECT that no entry is remaining, + // rather than just invoking it. + task_runner_->FastForwardUntilNoTasksRemain(); + } + + protected: + GURL target_url_; + scoped_refptr<MockSafeBrowsingDatabaseManager> database_manager_; + + // Needed for |database_manager_| teardown tasks. + std::unique_ptr<content::TestBrowserThread> io_thread_; + + std::unique_ptr<base::MessageLoop> message_loop_; + scoped_refptr<base::TestMockTimeTaskRunner> task_runner_; +}; + +TEST_F(WhitelistCheckerClientTest, TestMatch) { + EXPECT_CALL(*database_manager_.get(), CheckCsdWhitelistUrl(target_url_, _)) + .WillOnce(Return(AsyncMatch::MATCH)); + + MockBoolCallback callback; + EXPECT_CALL(callback, Run(true /* is_whitelisted */)); + WhitelistCheckerClient::StartCheckCsdWhitelist(database_manager_, target_url_, + callback.Get()); +} + +TEST_F(WhitelistCheckerClientTest, TestNoMatch) { + EXPECT_CALL(*database_manager_.get(), CheckCsdWhitelistUrl(target_url_, _)) + .WillOnce(Return(AsyncMatch::NO_MATCH)); + + MockBoolCallback callback; + EXPECT_CALL(callback, Run(false /* is_whitelisted */)); + WhitelistCheckerClient::StartCheckCsdWhitelist(database_manager_, target_url_, + callback.Get()); +} + +TEST_F(WhitelistCheckerClientTest, TestAsyncNoMatch) { + SafeBrowsingDatabaseManager::Client* client; + EXPECT_CALL(*database_manager_.get(), CheckCsdWhitelistUrl(target_url_, _)) + .WillOnce(DoAll(SaveArg<1>(&client), Return(AsyncMatch::ASYNC))); + + MockBoolCallback callback; + WhitelistCheckerClient::StartCheckCsdWhitelist(database_manager_, target_url_, + callback.Get()); + // Callback should not be called yet. + + EXPECT_CALL(callback, Run(false /* is_whitelisted */)); + // The self-owned client deletes itself here. + client->OnCheckWhitelistUrlResult(false); +} + +TEST_F(WhitelistCheckerClientTest, TestAsyncTimeout) { + SafeBrowsingDatabaseManager::Client* client; + EXPECT_CALL(*database_manager_.get(), CheckCsdWhitelistUrl(target_url_, _)) + .WillOnce(DoAll(SaveArg<1>(&client), Return(AsyncMatch::ASYNC))); + EXPECT_CALL(*database_manager_.get(), CancelCheck(_)).Times(1); + + MockBoolCallback callback; + WhitelistCheckerClient::StartCheckCsdWhitelist(database_manager_, target_url_, + callback.Get()); + task_runner_->FastForwardBy(base::TimeDelta::FromSeconds(1)); + // No callback yet. + + EXPECT_CALL(callback, Run(true /* is_whitelisted */)); + task_runner_->FastForwardBy(base::TimeDelta::FromSeconds(5)); +} + +} // namespace safe_browsing |