From 31ccca0778db85c159634478b4ec7997f6704860 Mon Sep 17 00:00:00 2001 From: Allan Sandfeld Jensen Date: Wed, 11 Mar 2020 11:32:04 +0100 Subject: BASELINE: Update Chromium to 80.0.3987.136 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I98e1649aafae85ba3a83e67af00bb27ef301db7b Reviewed-by: Jüri Valdmann --- chromium/components/optimization_guide/BUILD.gn | 12 +- .../components/optimization_guide/hint_cache.cc | 63 +- .../components/optimization_guide/hint_cache.h | 38 +- .../optimization_guide/hint_cache_store.cc | 776 -------- .../optimization_guide/hint_cache_store.h | 340 ---- .../hint_cache_store_unittest.cc | 1534 --------------- .../optimization_guide/hint_cache_unittest.cc | 54 +- .../optimization_guide/hint_update_data.cc | 109 -- .../optimization_guide/hint_update_data.h | 88 - .../hint_update_data_unittest.cc | 72 - .../components/optimization_guide/hints_fetcher.cc | 58 +- .../components/optimization_guide/hints_fetcher.h | 37 +- .../optimization_guide/hints_fetcher_unittest.cc | 28 +- .../optimization_guide/hints_processing_util.cc | 8 +- .../optimization_guide/hints_processing_util.h | 8 +- .../hints_processing_util_unittest.cc | 10 +- .../optimization_guide_constants.cc | 10 +- .../optimization_guide_constants.h | 13 +- .../optimization_guide_decider.h | 11 +- .../optimization_guide/optimization_guide_enums.h | 38 +- .../optimization_guide_features.cc | 106 +- .../optimization_guide_features.h | 51 +- .../optimization_guide_features_unittest.cc | 18 +- .../optimization_guide/optimization_guide_prefs.cc | 57 +- .../optimization_guide/optimization_guide_prefs.h | 12 +- .../optimization_guide/optimization_guide_store.cc | 1102 +++++++++++ .../optimization_guide/optimization_guide_store.h | 481 +++++ .../optimization_guide_store_unittest.cc | 2070 ++++++++++++++++++++ .../optimization_guide_switches.cc | 48 +- .../optimization_guide_switches.h | 29 +- .../optimization_guide/proto/hint_cache.proto | 22 +- .../optimization_guide/proto/models.proto | 4 + .../optimization_guide/store_update_data.cc | 211 ++ .../optimization_guide/store_update_data.h | 110 ++ .../store_update_data_unittest.cc | 121 ++ 35 files changed, 4615 insertions(+), 3134 deletions(-) delete mode 100644 chromium/components/optimization_guide/hint_cache_store.cc delete mode 100644 chromium/components/optimization_guide/hint_cache_store.h delete mode 100644 chromium/components/optimization_guide/hint_cache_store_unittest.cc delete mode 100644 chromium/components/optimization_guide/hint_update_data.cc delete mode 100644 chromium/components/optimization_guide/hint_update_data.h delete mode 100644 chromium/components/optimization_guide/hint_update_data_unittest.cc create mode 100644 chromium/components/optimization_guide/optimization_guide_store.cc create mode 100644 chromium/components/optimization_guide/optimization_guide_store.h create mode 100644 chromium/components/optimization_guide/optimization_guide_store_unittest.cc create mode 100644 chromium/components/optimization_guide/store_update_data.cc create mode 100644 chromium/components/optimization_guide/store_update_data.h create mode 100644 chromium/components/optimization_guide/store_update_data_unittest.cc (limited to 'chromium/components/optimization_guide') diff --git a/chromium/components/optimization_guide/BUILD.gn b/chromium/components/optimization_guide/BUILD.gn index 972ec5d8279..7782d880738 100644 --- a/chromium/components/optimization_guide/BUILD.gn +++ b/chromium/components/optimization_guide/BUILD.gn @@ -12,10 +12,6 @@ static_library("optimization_guide") { "command_line_top_host_provider.h", "hint_cache.cc", "hint_cache.h", - "hint_cache_store.cc", - "hint_cache_store.h", - "hint_update_data.cc", - "hint_update_data.h", "hints_component_info.h", "hints_component_util.cc", "hints_component_util.h", @@ -36,8 +32,12 @@ static_library("optimization_guide") { "optimization_guide_service.cc", "optimization_guide_service.h", "optimization_guide_service_observer.h", + "optimization_guide_store.cc", + "optimization_guide_store.h", "optimization_guide_switches.cc", "optimization_guide_switches.h", + "store_update_data.cc", + "store_update_data.h", "top_host_provider.h", "url_pattern_with_wildcards.cc", "url_pattern_with_wildcards.h", @@ -83,16 +83,16 @@ source_set("unit_tests") { sources = [ "bloom_filter_unittest.cc", "command_line_top_host_provider_unittest.cc", - "hint_cache_store_unittest.cc", "hint_cache_unittest.cc", - "hint_update_data_unittest.cc", "hints_component_util_unittest.cc", "hints_fetcher_unittest.cc", "hints_processing_util_unittest.cc", "optimization_filter_unittest.cc", "optimization_guide_features_unittest.cc", "optimization_guide_service_unittest.cc", + "optimization_guide_store_unittest.cc", "optimization_guide_switches_unittest.cc", + "store_update_data_unittest.cc", "url_pattern_with_wildcards_unittest.cc", ] diff --git a/chromium/components/optimization_guide/hint_cache.cc b/chromium/components/optimization_guide/hint_cache.cc index cb65e15d3e6..e2f8c7204da 100644 --- a/chromium/components/optimization_guide/hint_cache.cc +++ b/chromium/components/optimization_guide/hint_cache.cc @@ -7,9 +7,9 @@ #include #include "base/bind.h" -#include "components/optimization_guide/hint_update_data.h" #include "components/optimization_guide/hints_processing_util.h" #include "components/optimization_guide/optimization_guide_features.h" +#include "components/optimization_guide/store_update_data.h" #include "url/gurl.h" namespace optimization_guide { @@ -23,13 +23,13 @@ const size_t kDefaultMaxMemoryCacheHints = 20; } // namespace HintCache::HintCache( - std::unique_ptr hint_store, + std::unique_ptr optimization_guide_store, base::Optional max_memory_cache_hints /*= base::Optional()*/) - : hint_store_(std::move(hint_store)), + : optimization_guide_store_(std::move(optimization_guide_store)), memory_cache_( std::max(max_memory_cache_hints.value_or(kDefaultMaxMemoryCacheHints), 1)) { - DCHECK(hint_store_); + DCHECK(optimization_guide_store_); } HintCache::~HintCache() = default; @@ -37,28 +37,30 @@ HintCache::~HintCache() = default; void HintCache::Initialize(bool purge_existing_data, base::OnceClosure callback) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - hint_store_->Initialize( + optimization_guide_store_->Initialize( purge_existing_data, base::BindOnce(&HintCache::OnStoreInitialized, base::Unretained(this), std::move(callback))); } -std::unique_ptr +std::unique_ptr HintCache::MaybeCreateUpdateDataForComponentHints( const base::Version& version) const { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - return hint_store_->MaybeCreateUpdateDataForComponentHints(version); + return optimization_guide_store_->MaybeCreateUpdateDataForComponentHints( + version); } -std::unique_ptr HintCache::CreateUpdateDataForFetchedHints( +std::unique_ptr HintCache::CreateUpdateDataForFetchedHints( base::Time update_time, base::Time expiry_time) const { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - return hint_store_->CreateUpdateDataForFetchedHints(update_time, expiry_time); + return optimization_guide_store_->CreateUpdateDataForFetchedHints( + update_time, expiry_time); } void HintCache::UpdateComponentHints( - std::unique_ptr component_data, + std::unique_ptr component_data, base::OnceClosure callback) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(component_data); @@ -67,8 +69,8 @@ void HintCache::UpdateComponentHints( // data. memory_cache_.Clear(); - hint_store_->UpdateComponentHints(std::move(component_data), - std::move(callback)); + optimization_guide_store_->UpdateComponentHints(std::move(component_data), + std::move(callback)); } void HintCache::UpdateFetchedHints( @@ -82,33 +84,33 @@ void HintCache::UpdateFetchedHints( } else { expiry_time += features::StoredFetchedHintsFreshnessDuration(); } - std::unique_ptr fetched_hints_update_data = + std::unique_ptr fetched_hints_update_data = CreateUpdateDataForFetchedHints(update_time, expiry_time); ProcessHints(get_hints_response.get()->mutable_hints(), fetched_hints_update_data.get()); - hint_store_->UpdateFetchedHints(std::move(fetched_hints_update_data), - std::move(callback)); + optimization_guide_store_->UpdateFetchedHints( + std::move(fetched_hints_update_data), std::move(callback)); } void HintCache::ClearFetchedHints() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(hint_store_); + DCHECK(optimization_guide_store_); // TODO(mcrouse): Update to remove only fetched hints from |memory_cache_|. memory_cache_.Clear(); - hint_store_->ClearFetchedHintsFromDatabase(); + optimization_guide_store_->ClearFetchedHintsFromDatabase(); } bool HintCache::HasHint(const std::string& host) const { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - HintCacheStore::EntryKey hint_entry_key; - return hint_store_->FindHintEntryKey(host, &hint_entry_key); + OptimizationGuideStore::EntryKey hint_entry_key; + return optimization_guide_store_->FindHintEntryKey(host, &hint_entry_key); } void HintCache::LoadHint(const std::string& host, HintLoadedCallback callback) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store_->FindHintEntryKey(host, &hint_entry_key)) { + OptimizationGuideStore::EntryKey hint_entry_key; + if (!optimization_guide_store_->FindHintEntryKey(host, &hint_entry_key)) { std::move(callback).Run(nullptr); return; } @@ -117,7 +119,7 @@ void HintCache::LoadHint(const std::string& host, HintLoadedCallback callback) { // then asynchronously load it from the store and return. auto hint_it = memory_cache_.Get(hint_entry_key); if (hint_it == memory_cache_.end()) { - hint_store_->LoadHint( + optimization_guide_store_->LoadHint( hint_entry_key, base::BindOnce(&HintCache::OnLoadStoreHint, base::Unretained(this), std::move(callback))); @@ -133,8 +135,8 @@ const proto::Hint* HintCache::GetHintIfLoaded(const std::string& host) { // Try to retrieve the hint entry key for the host. If no hint exists for the // host, then simply return. - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store_->FindHintEntryKey(host, &hint_entry_key)) { + OptimizationGuideStore::EntryKey hint_entry_key; + if (!optimization_guide_store_->FindHintEntryKey(host, &hint_entry_key)) { return nullptr; } @@ -148,11 +150,11 @@ const proto::Hint* HintCache::GetHintIfLoaded(const std::string& host) { return nullptr; } -base::Time HintCache::FetchedHintsUpdateTime() const { - if (!hint_store_) { +base::Time HintCache::GetFetchedHintsUpdateTime() const { + if (!optimization_guide_store_) { return base::Time(); } - return hint_store_->FetchedHintsUpdateTime(); + return optimization_guide_store_->GetFetchedHintsUpdateTime(); } void HintCache::OnStoreInitialized(base::OnceClosure callback) { @@ -160,9 +162,10 @@ void HintCache::OnStoreInitialized(base::OnceClosure callback) { std::move(callback).Run(); } -void HintCache::OnLoadStoreHint(HintLoadedCallback callback, - const HintCacheStore::EntryKey& hint_entry_key, - std::unique_ptr hint) { +void HintCache::OnLoadStoreHint( + HintLoadedCallback callback, + const OptimizationGuideStore::EntryKey& hint_entry_key, + std::unique_ptr hint) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); if (!hint) { std::move(callback).Run(nullptr); diff --git a/chromium/components/optimization_guide/hint_cache.h b/chromium/components/optimization_guide/hint_cache.h index d9e03f3bd65..ae66a82faed 100644 --- a/chromium/components/optimization_guide/hint_cache.h +++ b/chromium/components/optimization_guide/hint_cache.h @@ -12,11 +12,11 @@ #include "base/macros.h" #include "base/optional.h" #include "base/sequence_checker.h" -#include "components/optimization_guide/hint_cache_store.h" +#include "components/optimization_guide/optimization_guide_store.h" #include "components/optimization_guide/proto/hints.pb.h" namespace optimization_guide { -class HintUpdateData; +class StoreUpdateData; using HintLoadedCallback = base::OnceCallback; @@ -29,10 +29,11 @@ using HintLoadedCallback = base::OnceCallback; class HintCache { public: // Construct the HintCache with a backing store and an optional max memory - // cache size. While |hint_store| is required, |max_memory_cache_hints| is - // optional and the default max size will be used if it is not provided. + // cache size. While |optimization_guide_store| is required, + // |max_memory_cache_hints| is optional and the default max size will be used + // if it is not provided. explicit HintCache( - std::unique_ptr hint_store, + std::unique_ptr optimization_guide_store, base::Optional max_memory_cache_hints = base::Optional()); ~HintCache(); @@ -42,13 +43,13 @@ class HintCache { // pre-existing data and begin in a clean state. void Initialize(bool purge_existing_data, base::OnceClosure callback); - // Returns a HintUpdateData. During component processing, hints from the - // component are moved into the HintUpdateData. After component + // Returns a StoreUpdateData. During component processing, hints from the + // component are moved into the StoreUpdateData. After component // processing completes, the component update data is provided to the backing // store in UpdateComponentHints() and used to update its component hints. In // the case the provided component version is not newer than the store's // version, nullptr will be returned by the call. - std::unique_ptr MaybeCreateUpdateDataForComponentHints( + std::unique_ptr MaybeCreateUpdateDataForComponentHints( const base::Version& version) const; // Returns an UpdateData created by the store to hold updates for fetched @@ -59,13 +60,13 @@ class HintCache { // created update data will be scheduled to be updated. |expiry_time| // specifies when the hints within the created update data will be expired // from the store. - std::unique_ptr CreateUpdateDataForFetchedHints( + std::unique_ptr CreateUpdateDataForFetchedHints( base::Time update_time, base::Time expiry_time) const; - // Updates the store's component data using the provided HintUpdateData + // Updates the store's component data using the provided StoreUpdateData // and asynchronously runs the provided callback after the update finishes. - void UpdateComponentHints(std::unique_ptr component_data, + void UpdateComponentHints(std::unique_ptr component_data, base::OnceClosure callback); // Process |get_hints_response| to be stored in the hint cache store. @@ -77,7 +78,7 @@ class HintCache { base::Time update_time, base::OnceClosure callback); - // Purge fetched hints from the owned |hint_store_| and reset + // Purge fetched hints from the owned |optimization_guide_store_| and reset // the |memory_cache_|. void ClearFetchedHints(); @@ -92,14 +93,14 @@ class HintCache { // Returns the update time provided by |hint_store_|, which specifies when the // fetched hints within the store are ready to be updated. If |hint_store_| is // not initialized, base::Time() is returned. - base::Time FetchedHintsUpdateTime() const; + base::Time GetFetchedHintsUpdateTime() const; // Returns the hint data for |host| if found in memory, otherwise nullptr. const proto::Hint* GetHintIfLoaded(const std::string& host); private: using StoreHintMemoryCache = - base::HashingMRUCache>; // The callback run after the store finishes initialization. This then runs @@ -110,12 +111,13 @@ class HintCache { // loaded hint to |memory_cache_|, potentially purging the least recently // used element, and then runs the callback initially provided by the // LoadHint() call. - void OnLoadStoreHint(HintLoadedCallback callback, - const HintCacheStore::EntryKey& store_hint_entry_key, - std::unique_ptr hint); + void OnLoadStoreHint( + HintLoadedCallback callback, + const OptimizationGuideStore::EntryKey& store_hint_entry_key, + std::unique_ptr hint); // The backing store used with this hint cache. Set during construction. - const std::unique_ptr hint_store_; + const std::unique_ptr optimization_guide_store_; // The in-memory cache of hints loaded from the store. Maps store EntryKey to // Hint proto. This servers two purposes: diff --git a/chromium/components/optimization_guide/hint_cache_store.cc b/chromium/components/optimization_guide/hint_cache_store.cc deleted file mode 100644 index 3b1a0078d6b..00000000000 --- a/chromium/components/optimization_guide/hint_cache_store.cc +++ /dev/null @@ -1,776 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/optimization_guide/hint_cache_store.h" - -#include "base/bind.h" -#include "base/metrics/histogram_functions.h" -#include "base/metrics/histogram_macros.h" -#include "base/strings/strcat.h" -#include "base/strings/string_number_conversions.h" -#include "base/strings/string_util.h" -#include "components/leveldb_proto/public/proto_database.h" -#include "components/leveldb_proto/public/proto_database_provider.h" -#include "components/leveldb_proto/public/shared_proto_database_client_list.h" -#include "components/optimization_guide/optimization_guide_prefs.h" -#include "components/optimization_guide/proto/hint_cache.pb.h" - -namespace optimization_guide { - -namespace { - -// Enforce that StoreEntryType enum is synced with the StoreEntryType proto -// (components/previews/content/proto/hint_cache.proto) -static_assert(proto::StoreEntryType_MAX == - static_cast(HintCacheStore::StoreEntryType::kMaxValue), - "mismatched StoreEntryType enums"); - -// The folder where the data will be stored on disk. -constexpr char kHintCacheStoreFolder[] = "previews_hint_cache_store"; - -// The amount of data to build up in memory before converting to a sorted on- -// disk file. -constexpr size_t kDatabaseWriteBufferSizeBytes = 128 * 1024; - -// Delimiter that appears between the sections of a store entry key. -// Examples: -// "[StoreEntryType::kMetadata]_[MetadataType]" -// "[StoreEntryType::kComponentHint]_[component_version]_[host]" -constexpr char kKeySectionDelimiter = '_'; - -// Realistic minimum length of a host suffix. -const int kMinHostSuffix = 6; // eg., abc.tv - -// Enumerates the possible outcomes of loading metadata. Used in UMA histograms, -// so the order of enumerators should not be changed. -// -// Keep in sync with OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult -// in tools/metrics/histograms/enums.xml. -enum class OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult { - kSuccess = 0, - kLoadMetadataFailed = 1, - kSchemaMetadataMissing = 2, - kSchemaMetadataWrongVersion = 3, - kComponentMetadataMissing = 4, - kFetchedMetadataMissing = 5, - kComponentAndFetchedMetadataMissing = 6, - kMaxValue = kComponentAndFetchedMetadataMissing, -}; - -// Util class for recording the result of loading the metadata. The result is -// recorded when it goes out of scope and its destructor is called. -class ScopedLoadMetadataResultRecorder { - public: - ScopedLoadMetadataResultRecorder() - : result_(OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kSuccess) {} - ~ScopedLoadMetadataResultRecorder() { - UMA_HISTOGRAM_ENUMERATION( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", result_); - } - - void set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult result) { - result_ = result; - } - - private: - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult result_; -}; - -void RecordStatusChange(HintCacheStore::Status status) { - UMA_HISTOGRAM_ENUMERATION("OptimizationGuide.HintCacheLevelDBStore.Status", - status); -} - -// Returns true if |key_prefix| is a prefix of |key|. -bool DatabasePrefixFilter(const std::string& key_prefix, - const std::string& key) { - return base::StartsWith(key, key_prefix, base::CompareCase::SENSITIVE); -} - -} // namespace - -HintCacheStore::HintCacheStore( - leveldb_proto::ProtoDatabaseProvider* database_provider, - const base::FilePath& database_dir, - scoped_refptr store_task_runner) { - base::FilePath hint_store_dir = - database_dir.AppendASCII(kHintCacheStoreFolder); - database_ = database_provider->GetDB( - leveldb_proto::ProtoDbType::HINT_CACHE_STORE, hint_store_dir, - store_task_runner); - - RecordStatusChange(status_); -} - -HintCacheStore::HintCacheStore( - std::unique_ptr> database) - : database_(std::move(database)) { - RecordStatusChange(status_); -} - -HintCacheStore::~HintCacheStore() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); -} - -void HintCacheStore::Initialize(bool purge_existing_data, - base::OnceClosure callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - UpdateStatus(Status::kInitializing); - - // Asynchronously initialize the store and run the callback once - // initialization completes. Initialization consists of the following steps: - // 1. Initialize the database. - // 2. If |purge_existing_data| is set to true, unconditionally purge - // database and skip to step 6. - // 3. Otherwise, retrieve the metadata entries (e.g. Schema and Component). - // 4. If schema is the wrong version, purge database and skip to step 6. - // 5. Otherwise, load all hint entry keys. - // 6. Run callback after purging database or retrieving hint entry keys. - - leveldb_env::Options options = leveldb_proto::CreateSimpleOptions(); - options.write_buffer_size = kDatabaseWriteBufferSizeBytes; - database_->Init(options, - base::BindOnce(&HintCacheStore::OnDatabaseInitialized, - weak_ptr_factory_.GetWeakPtr(), - purge_existing_data, std::move(callback))); -} - -std::unique_ptr -HintCacheStore::MaybeCreateUpdateDataForComponentHints( - const base::Version& version) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(version.IsValid()); - - if (!IsAvailable()) { - return nullptr; - } - - // Component updates are only permitted when the update version is newer than - // the store's current one. - if (component_version_.has_value() && version <= component_version_) { - return nullptr; - } - - // Create and return a HintUpdateData object. This object has - // hints from the component moved into it and organizes them in a format - // usable by the store; the object will returned to the store in - // StoreComponentHints(). - return HintUpdateData::CreateComponentHintUpdateData(version); -} - -std::unique_ptr HintCacheStore::CreateUpdateDataForFetchedHints( - base::Time update_time, - base::Time expiry_time) const { - // Create and returns a HintUpdateData object. This object has has hints - // from the GetHintsResponse moved into and organizes them in a format - // usable by the store. The object will be store with UpdateFetchedData(). - return HintUpdateData::CreateFetchedHintUpdateData(update_time, expiry_time); -} - -void HintCacheStore::UpdateComponentHints( - std::unique_ptr component_data, - base::OnceClosure callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(component_data); - DCHECK(!data_update_in_flight_); - DCHECK(component_data->component_version()); - - if (!IsAvailable()) { - std::move(callback).Run(); - return; - } - - // If this isn't a newer component version than the store's current one, then - // the simply return. There's nothing to update. - if (component_version_.has_value() && - component_data->component_version() <= component_version_) { - std::move(callback).Run(); - return; - } - - // Mark that there's now a component data update in-flight. While this is - // true, keys and hints will not be returned by the store. - data_update_in_flight_ = true; - - // Set the component version prior to requesting the update. This ensures that - // a second update request for the same component version won't be allowed. In - // the case where the update fails, the store will become unavailable, so it's - // safe to treat component version in the update as the new one. - SetComponentVersion(*component_data->component_version()); - - // The current keys are about to be removed, so clear out the keys available - // within the store. The keys will be populated after the component data - // update completes. - hint_entry_keys_.reset(); - - // Purge any component hints that are missing the new component version - // prefix. - EntryKeyPrefix retain_prefix = - GetComponentHintEntryKeyPrefix(component_version_.value()); - EntryKeyPrefix filter_prefix = GetComponentHintEntryKeyPrefixWithoutVersion(); - - // Add the new component data and purge any old component hints from the db. - // After processing finishes, OnUpdateHints() is called, which loads - // the updated hint entry keys from the database. - database_->UpdateEntriesWithRemoveFilter( - component_data->TakeUpdateEntries(), - base::BindRepeating( - [](const EntryKeyPrefix& retain_prefix, - const EntryKeyPrefix& filter_prefix, const std::string& key) { - return key.compare(0, retain_prefix.length(), retain_prefix) != 0 && - key.compare(0, filter_prefix.length(), filter_prefix) == 0; - }, - retain_prefix, filter_prefix), - base::BindOnce(&HintCacheStore::OnUpdateHints, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void HintCacheStore::UpdateFetchedHints( - std::unique_ptr fetched_hints_data, - base::OnceClosure callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(fetched_hints_data); - DCHECK(!data_update_in_flight_); - DCHECK(fetched_hints_data->fetch_update_time()); - - if (!IsAvailable()) { - std::move(callback).Run(); - return; - } - - fetched_update_time_ = *fetched_hints_data->fetch_update_time(); - - data_update_in_flight_ = true; - - hint_entry_keys_.reset(); - - // This will remove the fetched metadata entry and insert all the entries - // currently in |leveldb_fetched_hints_data|. - database_->UpdateEntriesWithRemoveFilter( - fetched_hints_data->TakeUpdateEntries(), - base::BindRepeating(&DatabasePrefixFilter, - GetMetadataTypeEntryKey(MetadataType::kFetched)), - base::BindOnce(&HintCacheStore::OnUpdateHints, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void HintCacheStore::PurgeExpiredFetchedHints() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!IsAvailable()) { - return; - } - - // Load all the fetched hints to check their expiry times. - database_->LoadKeysAndEntriesWithFilter( - base::BindRepeating(&DatabasePrefixFilter, - GetFetchedHintEntryKeyPrefix()), - base::BindOnce(&HintCacheStore::OnLoadFetchedHintsToPurgeExpired, - weak_ptr_factory_.GetWeakPtr())); -} - -void HintCacheStore::OnLoadFetchedHintsToPurgeExpired( - bool success, - std::unique_ptr fetched_entries) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!success) { - return; - } - - auto keys_to_remove = std::make_unique(); - int64_t now_since_epoch = - base::Time::Now().ToDeltaSinceWindowsEpoch().InSeconds(); - - for (const auto& entry : *fetched_entries) { - if (entry.second.expiry_time_secs() <= now_since_epoch) { - keys_to_remove->insert(entry.first); - } - } - - // TODO(mcrouse): Record the number of hints that will be expired from the - // store. - - data_update_in_flight_ = true; - hint_entry_keys_.reset(); - - auto empty_entries = std::make_unique(); - - database_->UpdateEntriesWithRemoveFilter( - std::move(empty_entries), - base::BindRepeating( - [](EntryKeySet* keys_to_remove, const std::string& key) { - return keys_to_remove->find(key) != keys_to_remove->end(); - }, - keys_to_remove.get()), - base::BindOnce(&HintCacheStore::OnUpdateHints, - weak_ptr_factory_.GetWeakPtr(), base::DoNothing::Once())); -} - -bool HintCacheStore::FindHintEntryKey(const std::string& host, - EntryKey* out_hint_entry_key) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // Search for kFetched hint entry keys first, fetched hints should be - // fresher and preferred. - if (FindHintEntryKeyForHostWithPrefix(host, out_hint_entry_key, - GetFetchedHintEntryKeyPrefix())) { - return true; - } - - // Search for kComponent hint entry keys next. - DCHECK(!component_version_.has_value() || - component_hint_entry_key_prefix_ == - GetComponentHintEntryKeyPrefix(component_version_.value())); - if (FindHintEntryKeyForHostWithPrefix(host, out_hint_entry_key, - component_hint_entry_key_prefix_)) - return true; - - return false; -} - -bool HintCacheStore::FindHintEntryKeyForHostWithPrefix( - const std::string& host, - EntryKey* out_hint_entry_key, - const EntryKeyPrefix& hint_entry_key_prefix) const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(out_hint_entry_key); - - // Look for longest host name suffix that has a hint. No need to continue - // lookups and substring work once get to a root domain like ".com" or - // ".co.in" (MinHostSuffix length check is a heuristic for that). - std::string host_suffix(host); - while (host_suffix.length() >= kMinHostSuffix) { - // Attempt to find a hint entry key associated with the current host suffix. - *out_hint_entry_key = hint_entry_key_prefix + host_suffix; - if (hint_entry_keys_ && hint_entry_keys_->find(*out_hint_entry_key) != - hint_entry_keys_->end()) { - return true; - } - - size_t pos = host_suffix.find_first_of('.'); - if (pos == std::string::npos) { - break; - } - host_suffix = host_suffix.substr(pos + 1); - } - return false; -} - -void HintCacheStore::LoadHint(const EntryKey& hint_entry_key, - HintLoadedCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!IsAvailable()) { - std::move(callback).Run(hint_entry_key, nullptr); - return; - } - - database_->GetEntry(hint_entry_key, - base::BindOnce(&HintCacheStore::OnLoadHint, - weak_ptr_factory_.GetWeakPtr(), - hint_entry_key, std::move(callback))); -} - -base::Time HintCacheStore::FetchedHintsUpdateTime() const { - // If the store is not available, the metadata entries have not been loaded - // so there are no fetched hints. - if (!IsAvailable()) - return base::Time(); - return fetched_update_time_; -} - -// static -const char HintCacheStore::kStoreSchemaVersion[] = "1"; - -// static -HintCacheStore::EntryKeyPrefix HintCacheStore::GetMetadataEntryKeyPrefix() { - return base::NumberToString( - static_cast(HintCacheStore::StoreEntryType::kMetadata)) + - kKeySectionDelimiter; -} - -// static -HintCacheStore::EntryKey HintCacheStore::GetMetadataTypeEntryKey( - MetadataType metadata_type) { - return GetMetadataEntryKeyPrefix() + - base::NumberToString(static_cast(metadata_type)); -} - -// static -HintCacheStore::EntryKeyPrefix -HintCacheStore::GetComponentHintEntryKeyPrefixWithoutVersion() { - return base::NumberToString( - static_cast(HintCacheStore::StoreEntryType::kComponentHint)) + - kKeySectionDelimiter; -} - -// static -HintCacheStore::EntryKeyPrefix HintCacheStore::GetComponentHintEntryKeyPrefix( - const base::Version& component_version) { - return GetComponentHintEntryKeyPrefixWithoutVersion() + - component_version.GetString() + kKeySectionDelimiter; -} - -// static -HintCacheStore::EntryKeyPrefix HintCacheStore::GetFetchedHintEntryKeyPrefix() { - return base::NumberToString( - static_cast(HintCacheStore::StoreEntryType::kFetchedHint)) + - kKeySectionDelimiter; -} - -void HintCacheStore::UpdateStatus(Status new_status) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // Status::kUninitialized can only transition to Status::kInitializing. - DCHECK(status_ != Status::kUninitialized || - new_status == Status::kInitializing); - // Status::kInitializing can only transition to Status::kAvailable or - // Status::kFailed. - DCHECK(status_ != Status::kInitializing || new_status == Status::kAvailable || - new_status == Status::kFailed); - // Status::kAvailable can only transition to kStatus::Failed. - DCHECK(status_ != Status::kAvailable || new_status == Status::kFailed); - // The status can never transition from Status::kFailed. - DCHECK(status_ != Status::kFailed || new_status == Status::kFailed); - - // If the status is not changing, simply return; the remaining logic handles - // status changes. - if (status_ == new_status) { - return; - } - - status_ = new_status; - RecordStatusChange(status_); - - if (status_ == Status::kFailed) { - database_->Destroy(base::BindOnce(&HintCacheStore::OnDatabaseDestroyed, - weak_ptr_factory_.GetWeakPtr())); - ClearComponentVersion(); - hint_entry_keys_.reset(); - } -} - -bool HintCacheStore::IsAvailable() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - return status_ == Status::kAvailable; -} - -void HintCacheStore::PurgeDatabase(base::OnceClosure callback) { - // When purging the database, update the schema version to the current one. - EntryKey schema_entry_key = GetMetadataTypeEntryKey(MetadataType::kSchema); - proto::StoreEntry schema_entry; - schema_entry.set_version(kStoreSchemaVersion); - - auto entries_to_save = std::make_unique(); - entries_to_save->emplace_back(schema_entry_key, schema_entry); - - database_->UpdateEntriesWithRemoveFilter( - std::move(entries_to_save), - base::BindRepeating( - [](const std::string& schema_entry_key, const std::string& key) { - return key.compare(0, schema_entry_key.length(), - schema_entry_key) != 0; - }, - schema_entry_key), - base::BindOnce(&HintCacheStore::OnPurgeDatabase, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void HintCacheStore::SetComponentVersion( - const base::Version& component_version) { - DCHECK(component_version.IsValid()); - component_version_ = component_version; - component_hint_entry_key_prefix_ = - GetComponentHintEntryKeyPrefix(component_version_.value()); -} - -void HintCacheStore::ClearComponentVersion() { - component_version_.reset(); - component_hint_entry_key_prefix_.clear(); -} - -void HintCacheStore::ClearFetchedHintsFromDatabase() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(!data_update_in_flight_); - - if (!IsAvailable()) { - return; - } - - data_update_in_flight_ = true; - auto entries_to_save = std::make_unique(); - - // TODO(mcrouse): Add histogram to record the number of hints being removed. - hint_entry_keys_.reset(); - - // Removes all |kFetchedHint| store entries. OnUpdateHints will handle - // updating status and re-filling hint_entry_keys with the hints still in the - // store. - database_->UpdateEntriesWithRemoveFilter( - std::move(entries_to_save), // this should be empty. - base::BindRepeating(&DatabasePrefixFilter, - GetFetchedHintEntryKeyPrefix()), - base::BindOnce(&HintCacheStore::OnUpdateHints, - weak_ptr_factory_.GetWeakPtr(), base::DoNothing::Once())); -} - -void HintCacheStore::MaybeLoadHintEntryKeys(base::OnceClosure callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // If the database is unavailable or if there's an in-flight component data - // update, then don't load the hint keys. Simply run the callback. - if (!IsAvailable() || data_update_in_flight_) { - std::move(callback).Run(); - return; - } - - // Create a new KeySet object. This is populated by the store's keys as the - // filter is run with each key on the DB's background thread. The filter - // itself always returns false, ensuring that no entries are ever actually - // loaded by the DB. Ownership of the KeySet is passed into the - // LoadKeysAndEntriesCallback callback, guaranteeing that the KeySet has a - // lifespan longer than the filter calls. - std::unique_ptr hint_entry_keys(std::make_unique()); - EntryKeySet* raw_hint_entry_keys_pointer = hint_entry_keys.get(); - database_->LoadKeysAndEntriesWithFilter( - base::BindRepeating( - [](EntryKeySet* hint_entry_keys, const std::string& filter_prefix, - const std::string& entry_key) { - if (entry_key.compare(0, filter_prefix.length(), filter_prefix) != - 0) { - hint_entry_keys->insert(entry_key); - } - return false; - }, - raw_hint_entry_keys_pointer, GetMetadataEntryKeyPrefix()), - base::BindOnce(&HintCacheStore::OnLoadHintEntryKeys, - weak_ptr_factory_.GetWeakPtr(), std::move(hint_entry_keys), - std::move(callback))); -} - -size_t HintCacheStore::GetHintEntryKeyCount() const { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - return hint_entry_keys_ ? hint_entry_keys_->size() : 0; -} - -void HintCacheStore::OnDatabaseInitialized( - bool purge_existing_data, - base::OnceClosure callback, - leveldb_proto::Enums::InitStatus status) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (status != leveldb_proto::Enums::InitStatus::kOK) { - UpdateStatus(Status::kFailed); - std::move(callback).Run(); - return; - } - - // If initialization is set to purge all existing data, then simply trigger - // the purge and return. There's no need to load metadata entries that'll - // immediately be purged. - if (purge_existing_data) { - PurgeDatabase(std::move(callback)); - return; - } - - // Load all entries from the DB with the metadata key prefix. - database_->LoadKeysAndEntriesWithFilter( - leveldb_proto::KeyFilter(), leveldb::ReadOptions(), - GetMetadataEntryKeyPrefix(), - base::BindOnce(&HintCacheStore::OnLoadMetadata, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void HintCacheStore::OnDatabaseDestroyed(bool /*success*/) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); -} - -void HintCacheStore::OnLoadMetadata( - base::OnceClosure callback, - bool success, - std::unique_ptr metadata_entries) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(metadata_entries); - - // Create a scoped load metadata result recorder. It records the result when - // its destructor is called. - ScopedLoadMetadataResultRecorder result_recorder; - - if (!success) { - result_recorder.set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kLoadMetadataFailed); - - UpdateStatus(Status::kFailed); - std::move(callback).Run(); - return; - } - - // If the schema version in the DB is not the current version, then purge - // the database. - auto schema_entry = - metadata_entries->find(GetMetadataTypeEntryKey(MetadataType::kSchema)); - if (schema_entry == metadata_entries->end() || - !schema_entry->second.has_version() || - schema_entry->second.version() != kStoreSchemaVersion) { - if (schema_entry == metadata_entries->end()) { - result_recorder.set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kSchemaMetadataMissing); - } else { - result_recorder.set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kSchemaMetadataWrongVersion); - } - - PurgeDatabase(std::move(callback)); - return; - } - - // If the component metadata entry exists, then use it to set the component - // version. - bool component_metadata_missing = false; - auto component_entry = - metadata_entries->find(GetMetadataTypeEntryKey(MetadataType::kComponent)); - if (component_entry != metadata_entries->end()) { - DCHECK(component_entry->second.has_version()); - SetComponentVersion(base::Version(component_entry->second.version())); - } else { - result_recorder.set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kComponentMetadataMissing); - component_metadata_missing = true; - } - - auto fetched_entry = - metadata_entries->find(GetMetadataTypeEntryKey(MetadataType::kFetched)); - if (fetched_entry != metadata_entries->end()) { - DCHECK(fetched_entry->second.has_update_time_secs()); - fetched_update_time_ = base::Time::FromDeltaSinceWindowsEpoch( - base::TimeDelta::FromSeconds(fetched_entry->second.update_time_secs())); - } else { - if (component_metadata_missing) { - result_recorder.set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kComponentAndFetchedMetadataMissing); - } else { - result_recorder.set_result( - OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: - kFetchedMetadataMissing); - } - fetched_update_time_ = base::Time(); - } - - UpdateStatus(Status::kAvailable); - MaybeLoadHintEntryKeys(std::move(callback)); -} - -void HintCacheStore::OnPurgeDatabase(base::OnceClosure callback, bool success) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - // The database can only be purged during initialization. - DCHECK_EQ(status_, Status::kInitializing); - - UpdateStatus(success ? Status::kAvailable : Status::kFailed); - std::move(callback).Run(); -} - -void HintCacheStore::OnUpdateHints(base::OnceClosure callback, bool success) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(data_update_in_flight_); - - data_update_in_flight_ = false; - if (!success) { - UpdateStatus(Status::kFailed); - std::move(callback).Run(); - return; - } - MaybeLoadHintEntryKeys(std::move(callback)); -} - -void HintCacheStore::OnLoadHintEntryKeys( - std::unique_ptr hint_entry_keys, - base::OnceClosure callback, - bool success, - std::unique_ptr /*unused*/) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(!hint_entry_keys_); - - if (!success) { - UpdateStatus(Status::kFailed); - std::move(callback).Run(); - return; - } - - // If the store was set to unavailable after the request was started, or if - // there's an in-flight component data update, which means the keys are - // about to be invalidated, then the loaded keys should not be considered - // valid. Reset the keys so that they are cleared. - if (!IsAvailable() || data_update_in_flight_) { - hint_entry_keys.reset(); - } - - hint_entry_keys_ = std::move(hint_entry_keys); - std::move(callback).Run(); -} - -void HintCacheStore::OnLoadHint(const std::string& entry_key, - HintLoadedCallback callback, - bool success, - std::unique_ptr entry) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // If either the request failed, the store was set to unavailable after the - // request was started, or there's an in-flight component data update, which - // means the entry is about to be invalidated, then the loaded hint should - // not be considered valid. Reset the entry so that no hint is returned to - // the requester. - if (!success || !IsAvailable() || data_update_in_flight_) { - entry.reset(); - } - - if (!entry || !entry->has_hint()) { - std::unique_ptr loaded_hint(nullptr); - std::move(callback).Run(entry_key, std::move(loaded_hint)); - return; - } - - if (entry->has_expiry_time_secs() && - entry->expiry_time_secs() <= - base::Time::Now().ToDeltaSinceWindowsEpoch().InSeconds()) { - // An expired hint should be loaded rarely if the user is regularly fetching - // and storing fresh hints. Expired fetched hints are removed each time - // fresh hints are fetched and placed into the store. In the future, the - // expired hints could be asynchronously removed if necessary. - // An empty hint is returned instead of the expired one. - UMA_HISTOGRAM_BOOLEAN( - "OptimizationGuide.HintCacheStore.OnLoadHint.FetchedHintExpired", true); - std::unique_ptr loaded_hint(nullptr); - std::move(callback).Run(entry_key, std::move(loaded_hint)); - return; - } - - // Release the Hint into a Hint unique_ptr. This eliminates the need for any - // copies of the entry's hint. - std::unique_ptr loaded_hint(entry->release_hint()); - - StoreEntryType store_entry_type = - static_cast(entry->entry_type()); - UMA_HISTOGRAM_ENUMERATION("OptimizationGuide.HintCache.HintType.Loaded", - store_entry_type); - - if (store_entry_type == StoreEntryType::kFetchedHint) { - UMA_HISTOGRAM_CUSTOM_TIMES( - "OptimizationGuide.HintCache.FetchedHint.TimeToExpiration", - base::Time::FromDeltaSinceWindowsEpoch( - base::TimeDelta::FromSeconds(entry->expiry_time_secs())) - - base::Time::Now(), - base::TimeDelta::FromHours(1), base::TimeDelta::FromDays(15), 50); - } - std::move(callback).Run(entry_key, std::move(loaded_hint)); -} - -} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/hint_cache_store.h b/chromium/components/optimization_guide/hint_cache_store.h deleted file mode 100644 index 9470a2110ab..00000000000 --- a/chromium/components/optimization_guide/hint_cache_store.h +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_OPTIMIZATION_GUIDE_HINT_CACHE_STORE_H_ -#define COMPONENTS_OPTIMIZATION_GUIDE_HINT_CACHE_STORE_H_ - -#include -#include - -#include "base/callback.h" -#include "base/containers/flat_set.h" -#include "base/macros.h" -#include "base/memory/scoped_refptr.h" -#include "base/memory/weak_ptr.h" -#include "base/optional.h" -#include "base/sequence_checker.h" -#include "base/version.h" -#include "components/leveldb_proto/public/proto_database.h" -#include "components/leveldb_proto/public/proto_database_provider.h" -#include "components/optimization_guide/hint_update_data.h" - -namespace base { -class SequencedTaskRunner; -} // namespace base - -namespace optimization_guide { -namespace proto { -class Hint; -class StoreEntry; -} // namespace proto - -// The HintCache backing store, which is responsible for storing all hints that -// are locally available. While the HintCache itself may retain some hints in a -// memory cache, all of its hints are initially loaded asynchronously by the -// store. All calls to this store must be made from the same thread. -class HintCacheStore { - public: - using HintLoadedCallback = - base::OnceCallback)>; - using EntryKey = std::string; - using StoreEntryProtoDatabase = - leveldb_proto::ProtoDatabase; - - // Status of the store. The store begins in kUninitialized, transitions to - // kInitializing after Initialize() is called, and transitions to kAvailable - // if initialization successfully completes. In the case where anything fails, - // the store transitions to kFailed, at which point it is fully purged and - // becomes unusable. - // - // Keep in sync with OptimizationGuideHintCacheLevelDBStoreStatus in - // tools/metrics/histograms/enums.xml. - enum class Status { - kUninitialized = 0, - kInitializing = 1, - kAvailable = 2, - kFailed = 3, - kMaxValue = kFailed, - }; - - // Store entry types within the store appear at the start of the keys of - // entries. These values are converted into strings within the key: a key - // starting with "1_" signifies a metadata entry and one starting with "2_" - // signifies a component hint entry. Adding this to the start of the key - // allows the store to quickly perform operations on all entries of a specific - // key type. Given that store entry type comparisons may be performed many - // times, the entry type string is kept as small as possible. - // Example metadata entry type key: - // "[StoreEntryType::kMetadata]_[MetadataType::kSchema]" ==> "1_1" - // Example component hint store entry type key: - // "[StoreEntryType::kComponentHint]_[component_version]_[host]" - // ==> "2_55_foo.com" - // NOTE: The order and value of the existing store entry types within the enum - // cannot be changed, but new types can be added to the end. - // StoreEntryType should remain synchronized with the - // HintCacheStoreEntryType in enums.xml. - // Also ensure to add to the OptimizationGuide.StoreEntryTypes histogram - // suffixes if adding a new one. - enum class StoreEntryType { - kEmpty = 0, - kMetadata = 1, - kComponentHint = 2, - kFetchedHint = 3, - kMaxValue = kFetchedHint - }; - - HintCacheStore(leveldb_proto::ProtoDatabaseProvider* database_provider, - const base::FilePath& database_dir, - scoped_refptr store_task_runner); - // For tests only. - explicit HintCacheStore(std::unique_ptr database); - ~HintCacheStore(); - - // Initializes the hint cache store. If |purge_existing_data| is set to true, - // then the cache is purged during initialization and starts in a fresh state. - // When initialization completes, the provided callback is run asynchronously. - void Initialize(bool purge_existing_data, base::OnceClosure callback); - - // Creates and returns a HintUpdateData object for component hints. This - // object is used to collect hints within a component in a format usable on a - // background thread and is later returned to the store in - // UpdateComponentHints(). The HintUpdateData object is only created when the - // provided component version is newer than the store's version, indicating - // fresh hints. If the component's version is not newer than the store's - // version, then no HintUpdateData is created and nullptr is returned. This - // prevents unnecessary processing of the component's hints by the caller. - std::unique_ptr MaybeCreateUpdateDataForComponentHints( - const base::Version& version) const; - - // Creates and returns a HintsUpdateData object for Fetched Hints. - // This object is used to collect a batch of hints in a format that is usable - // to update the store on a background thread. This is always created when - // hints have been successfully fetched from the remote Optimization Guide - // Service so the store can expire old hints, remove hints specified by the - // server, and store the fresh hints. - std::unique_ptr CreateUpdateDataForFetchedHints( - base::Time update_time, - base::Time expiry_time) const; - - // Updates the component hints and version contained within the store. When - // this is called, all pre-existing component hints within the store is purged - // and only the new hints are retained. After the store is fully updated with - // the new component hints, the callback is run asynchronously. - void UpdateComponentHints(std::unique_ptr component_data, - base::OnceClosure callback); - - // Updates the fetched hints contained in the store, including the - // metadata entry. The callback is run asynchronously after the database - // stores the hints. - void UpdateFetchedHints(std::unique_ptr fetched_hints_data, - base::OnceClosure callback); - - // Removes fetched hint store entries from |this|. |hint_entry_keys_| is - // updated after the fetched hint entries are removed. - void ClearFetchedHintsFromDatabase(); - - // Finds the most specific hint entry key for the specified host. Returns - // true if a hint entry key is found, in which case |out_hint_entry_key| is - // populated with the key. All keys for kFetched hints are considered before - // kComponent hints as they are updated more frequently. - bool FindHintEntryKey(const std::string& host, - EntryKey* out_hint_entry_key) const; - - // Loads the hint specified by |hint_entry_key|. - // After the load finishes, the hint data is passed to |callback|. In the case - // where the hint cannot be loaded, the callback is run with a nullptr. - // Depending on the load result, the callback may be synchronous or - // asynchronous. - void LoadHint(const EntryKey& hint_entry_key, HintLoadedCallback callback); - - // Returns the time that the fetched hints in the store can be updated. If - // |this| is not available, base::Time() is returned. - base::Time FetchedHintsUpdateTime() const; - - // Removes all fetched hints that have expired from the store. - // |hint_entry_keys| is updated after the expired fetched hints are - // removed. - void PurgeExpiredFetchedHints(); - - private: - friend class HintCacheStoreTest; - friend class HintUpdateData; - - using EntryKeyPrefix = std::string; - using EntryKeySet = base::flat_set; - - using EntryVector = - leveldb_proto::ProtoDatabase::KeyEntryVector; - using EntryMap = std::map; - - // Metadata types within the store. The metadata type appears at the end of - // metadata entry keys. These values are converted into strings within the - // key. - // Example metadata type keys: - // "[StoreEntryType::kMetadata]_[MetadataType::kSchema]" ==> "1_1" - // "[StoreEntryType::kMetadata]_[MetadataType::kComponent]" ==> "1_2" - // NOTE: The order and value of the existing metadata types within the enum - // cannot be changed, but new types can be added to the end. - enum class MetadataType { - kSchema = 1, - kComponent = 2, - kFetched = 3, - }; - - // Current schema version of the hint cache store. When this is changed, - // pre-existing store data from an earlier version is purged. - static const char kStoreSchemaVersion[]; - - // Returns prefix in the key of every metadata entry type entry: "1_" - static EntryKeyPrefix GetMetadataEntryKeyPrefix(); - - // Returns entry key for the specified metadata type entry: "1_[MetadataType]" - static EntryKey GetMetadataTypeEntryKey(MetadataType metadata_type); - - // Returns prefix in the key of every component hint entry: "2_" - static EntryKeyPrefix GetComponentHintEntryKeyPrefixWithoutVersion(); - - // Returns prefix in the key of component hint entries for the specified - // component version: "2_[component_version]_" - static EntryKeyPrefix GetComponentHintEntryKeyPrefix( - const base::Version& component_version); - - // Returns prefix of the key of every fetched hint entry: "3_". - static EntryKeyPrefix GetFetchedHintEntryKeyPrefix(); - - // Updates the status of the store to the specified value, validates the - // transition, and destroys the database in the case where the status - // transitions to Status::kFailed. - void UpdateStatus(Status new_status); - - // Returns true if the current status is Status::kAvailable. - bool IsAvailable() const; - - // Asynchronously purges all existing entries from the database and runs the - // callback after it completes. This should only be run during initialization. - void PurgeDatabase(base::OnceClosure callback); - - // Updates |component_version_| and |component_hint_entry_key_prefix_| for - // the new component version. This must be called rather than directly - // modifying |component_version_|, as it ensures that both member variables - // are updated in sync. - void SetComponentVersion(const base::Version& component_version); - - // Resets |component_version_| and |component_hint_entry_key_prefix_| back to - // their default state. Called after the database is destroyed. - void ClearComponentVersion(); - - // Asynchronously loads the hint entry keys from the store, populates - // |hint_entry_keys_| with them, and runs the provided callback after they - // finish loading. In the case where there is currently an in-flight component - // update, this does nothing, as the hint entry keys will be loaded after the - // component update completes. - void MaybeLoadHintEntryKeys(base::OnceClosure callback); - - // Returns the total hint entry keys contained within the store. - size_t GetHintEntryKeyCount() const; - - // Finds the most specific host suffix of the host name that the store has an - // hint with the provided prefix, |hint_entry_key_prefix|. |out_entry_key| is - // populated with the entry key for the corresponding hint. Returns true if a - // hint was successsfully found. - bool FindHintEntryKeyForHostWithPrefix( - const std::string& host, - EntryKey* out_entry_key, - const EntryKeyPrefix& hint_entry_key_prefix) const; - - // Callback that identifies any expired hints from |fetched_entries| and - // asynchronously removes them from the store. - void OnLoadFetchedHintsToPurgeExpired( - bool success, - std::unique_ptr fetched_entries); - - // Callback that runs after the database finishes being initialized. If - // |purge_existing_data| is true, then unconditionally purges the database; - // otherwise, triggers loading of the metadata. - void OnDatabaseInitialized(bool purge_existing_data, - base::OnceClosure callback, - leveldb_proto::Enums::InitStatus status); - - // Callback that is run after the database finishes being destroyed. - void OnDatabaseDestroyed(bool success); - - // Callback that runs after the metadata finishes being loaded. This - // validates the schema version, sets the component version, and either purges - // the store (on a schema version mismatch) or loads all hint entry keys (on - // a schema version match). - void OnLoadMetadata(base::OnceClosure callback, - bool success, - std::unique_ptr metadata_entries); - - // Callback that runs after the database is purged during initialization. - void OnPurgeDatabase(base::OnceClosure callback, bool success); - - // Callback that runs after the hints data within the store is fully - // updated. If the update was successful, it attempts to load all of the hint - // entry keys contained within the database. - void OnUpdateHints(base::OnceClosure callback, bool success); - - // Callback that runs after the hint entry keys are fully loaded. If there's - // currently an in-flight component update, then the hint entry keys will be - // loaded again after the component update completes, so the results are - // tossed; otherwise, |hint_entry_keys| is moved into |hint_entry_keys_|. - // Regardless of the outcome of loading the keys, the callback always runs. - void OnLoadHintEntryKeys(std::unique_ptr hint_entry_keys, - base::OnceClosure callback, - bool success, - std::unique_ptr unused); - - // Callback that runs after a hint entry is loaded from the database. If - // there's currently an in-flight component update, then the hint is about to - // be invalidated, so results are tossed; otherwise, the hint is released into - // the callback, allowing the caller to own the hint without copying it. - // Regardless of the success or failure of retrieving the key, the callback - // always runs (it simply runs with a nullptr on failure). - void OnLoadHint(const EntryKey& entry_key, - HintLoadedCallback callback, - bool success, - std::unique_ptr entry); - - // Proto database used by the store. - std::unique_ptr database_; - - // The current status of the store. It should only be updated through - // UpdateStatus(), which validates status transitions and triggers - // accompanying logic. - Status status_ = Status::kUninitialized; - - // The current component version of the store. This should only be updated - // via SetComponentVersion(), which ensures that both |component_version_| - // and |component_hint_key_prefix_| are updated at the same time. - base::Optional component_version_; - - // The current entry key prefix shared by all component hints containd within - // the store. While this could be generated on the fly using - // |component_version_|, it is retaind separately as an optimization, as it - // is needed often. - EntryKeyPrefix component_hint_entry_key_prefix_; - - // If a component data update is in the middle of being processed; when this - // is true, keys and hints will not be returned by the store. - bool data_update_in_flight_ = false; - - // The next update time for the fetched hints that are currently in the - // store. - base::Time fetched_update_time_; - - // The keys of the hints available within the store. - std::unique_ptr hint_entry_keys_; - - SEQUENCE_CHECKER(sequence_checker_); - - base::WeakPtrFactory weak_ptr_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(HintCacheStore); -}; - -} // namespace optimization_guide - -#endif // COMPONENTS_OPTIMIZATION_GUIDE_HINT_CACHE_STORE_H_ diff --git a/chromium/components/optimization_guide/hint_cache_store_unittest.cc b/chromium/components/optimization_guide/hint_cache_store_unittest.cc deleted file mode 100644 index 2af289e0b49..00000000000 --- a/chromium/components/optimization_guide/hint_cache_store_unittest.cc +++ /dev/null @@ -1,1534 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/optimization_guide/hint_cache_store.h" - -#include -#include - -#include "base/bind.h" -#include "base/macros.h" -#include "base/optional.h" -#include "base/strings/string_number_conversions.h" -#include "base/test/metrics/histogram_tester.h" -#include "components/leveldb_proto/testing/fake_db.h" -#include "components/optimization_guide/hint_update_data.h" -#include "components/optimization_guide/optimization_guide_features.h" -#include "components/optimization_guide/proto/hint_cache.pb.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" -#include "url/gurl.h" - -using leveldb_proto::test::FakeDB; -using testing::Mock; - -namespace optimization_guide { - -namespace { - -constexpr char kDefaultComponentVersion[] = "1.0.0"; -constexpr char kUpdateComponentVersion[] = "2.0.0"; - -std::string GetHostSuffix(size_t id) { - // Host suffix alternates between two different domain types depending on - // whether the id is odd or even. - if (id % 2 == 0) { - return "domain" + base::NumberToString(id) + ".org"; - } else { - return "different.domain" + base::NumberToString(id) + ".co.in"; - } -} - -enum class MetadataSchemaState { - kMissing, - kInvalid, - kValid, -}; - -} // namespace - -class HintCacheStoreTest : public testing::Test { - public: - using StoreEntry = proto::StoreEntry; - using StoreEntryMap = std::map; - - HintCacheStoreTest() : db_(nullptr) {} - - void TearDown() override { last_loaded_hint_.reset(); } - - // Initializes the entries contained within the database on startup. - void SeedInitialData( - MetadataSchemaState state, - base::Optional component_hint_count = base::Optional(), - base::Optional fetched_hints_update = - base::Optional()) { - db_store_.clear(); - - // Add a metadata schema entry if its state isn't kMissing. The version - // entry version is set to the store's current version if the state is - // kValid; otherwise, it's set to the invalid version of "0". - if (state == MetadataSchemaState::kValid) { - db_store_[HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kSchema)] - .set_version(HintCacheStore::kStoreSchemaVersion); - } else if (state == MetadataSchemaState::kInvalid) { - db_store_[HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kSchema)] - .set_version("0"); - } - - // If the database is being seeded with component hints, it is indicated - // with a provided count. Add the component metadata with the default - // component version and then add the indicated number of component hints. - // if (component_hint_count && component_hint_count > - // static_cast(0)) { - if (component_hint_count && component_hint_count > 0u) { - db_store_[HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kComponent)] - .set_version(kDefaultComponentVersion); - HintCacheStore::EntryKeyPrefix component_hint_key_prefix = - HintCacheStore::GetComponentHintEntryKeyPrefix( - base::Version(kDefaultComponentVersion)); - for (size_t i = 0; i < component_hint_count.value(); ++i) { - std::string host_suffix = GetHostSuffix(i); - StoreEntry& entry = db_store_[component_hint_key_prefix + host_suffix]; - entry.set_entry_type(static_cast( - HintCacheStore::StoreEntryType::kComponentHint)); - proto::Hint* hint = entry.mutable_hint(); - hint->set_key(host_suffix); - hint->set_key_representation(proto::HOST_SUFFIX); - proto::PageHint* page_hint = hint->add_page_hints(); - page_hint->set_page_pattern("page pattern " + base::NumberToString(i)); - } - } - if (fetched_hints_update) { - db_store_[HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kFetched)] - .set_update_time_secs( - fetched_hints_update->ToDeltaSinceWindowsEpoch().InSeconds()); - } - } - - // Moves the specified number of component hints into the update data. - void SeedComponentUpdateData(HintUpdateData* update_data, - size_t component_hint_count) { - for (size_t i = 0; i < component_hint_count; ++i) { - std::string host_suffix = GetHostSuffix(i); - proto::Hint hint; - hint.set_key(host_suffix); - hint.set_key_representation(proto::HOST_SUFFIX); - proto::PageHint* page_hint = hint.add_page_hints(); - page_hint->set_page_pattern("page pattern " + base::NumberToString(i)); - update_data->MoveHintIntoUpdateData(std::move(hint)); - } - } - // Moves the specified number of component hints into the update data. - void SeedFetchedUpdateData(HintUpdateData* update_data, - size_t fetched_hint_count) { - for (size_t i = 0; i < fetched_hint_count; ++i) { - std::string host_suffix = GetHostSuffix(i); - proto::Hint hint; - hint.set_key(host_suffix); - hint.set_key_representation(proto::HOST_SUFFIX); - proto::PageHint* page_hint = hint.add_page_hints(); - page_hint->set_page_pattern("page pattern " + base::NumberToString(i)); - update_data->MoveHintIntoUpdateData(std::move(hint)); - } - } - - void CreateDatabase() { - // Reset everything. - db_ = nullptr; - hint_store_.reset(); - - // Setup the fake db and the class under test. - auto db = std::make_unique>(&db_store_); - db_ = db.get(); - - hint_store_ = std::make_unique(std::move(db)); - } - - void InitializeDatabase(bool success, bool purge_existing_data = false) { - EXPECT_CALL(*this, OnInitialized()); - hint_store()->Initialize(purge_existing_data, - base::BindOnce(&HintCacheStoreTest::OnInitialized, - base::Unretained(this))); - // OnDatabaseInitialized callback - db()->InitStatusCallback(success ? leveldb_proto::Enums::kOK - : leveldb_proto::Enums::kError); - } - - void InitializeStore(MetadataSchemaState state, - bool purge_existing_data = false) { - InitializeDatabase(true /*=success*/, purge_existing_data); - - if (purge_existing_data) { - // OnPurgeDatabase callback - db()->UpdateCallback(true); - return; - } - - // OnLoadMetadata callback - db()->LoadCallback(true); - if (state == MetadataSchemaState::kValid) { - // OnLoadHintEntryKeys callback - db()->LoadCallback(true); - } else { - // OnPurgeDatabase callback - db()->UpdateCallback(true); - } - } - - void UpdateComponentHints(std::unique_ptr component_data, - bool update_success = true, - bool load_hint_entry_keys_success = true) { - EXPECT_CALL(*this, OnUpdateHints()); - hint_store()->UpdateComponentHints( - std::move(component_data), - base::BindOnce(&HintCacheStoreTest::OnUpdateHints, - base::Unretained(this))); - // OnUpdateHints callback - db()->UpdateCallback(update_success); - if (update_success) { - // OnLoadHintEntryKeys callback - db()->LoadCallback(load_hint_entry_keys_success); - } - } - - void UpdateFetchedHints(std::unique_ptr fetched_data, - bool update_success = true, - bool load_hint_entry_keys_success = true) { - EXPECT_CALL(*this, OnUpdateHints()); - hint_store()->UpdateFetchedHints( - std::move(fetched_data), - base::BindOnce(&HintCacheStoreTest::OnUpdateHints, - base::Unretained(this))); - // OnUpdateHints callback - db()->UpdateCallback(update_success); - if (update_success) { - // OnLoadHintEntryKeys callback - db()->LoadCallback(load_hint_entry_keys_success); - } - } - - void ClearFetchedHintsFromDatabase() { - hint_store()->ClearFetchedHintsFromDatabase(); - db()->UpdateCallback(true); - db()->LoadCallback(true); - } - - void PurgeExpiredFetchedHints() { - hint_store()->PurgeExpiredFetchedHints(); - - // OnFetchedHintsLoadedToMaybePurge - db()->LoadCallback(true); - // OnUpdateHints - db()->UpdateCallback(true); - // OnLoadHintEntryKeys callback - db()->LoadCallback(true); - } - - bool IsMetadataSchemaEntryKeyPresent() const { - return IsKeyPresent(HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kSchema)); - } - - // Verifies that the fetched metadata has the expected next update time. - void ExpectFetchedMetadata(base::Time update_time) const { - const auto& metadata_entry = - db_store_.find(HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kFetched)); - if (metadata_entry != db_store_.end()) { - // The next update time should have same time up to the second as the - // metadata entry is stored in seconds. - EXPECT_TRUE( - base::Time::FromDeltaSinceWindowsEpoch(base::TimeDelta::FromSeconds( - metadata_entry->second.update_time_secs())) - - update_time < - base::TimeDelta::FromSeconds(1)); - } else { - FAIL() << "No fetched metadata found"; - } - } - // Verifies that the component metadata has the expected version and all - // expected component hints are present. - void ExpectComponentHintsPresent(const std::string& version, - int count) const { - const auto& metadata_entry = - db_store_.find(HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kComponent)); - if (metadata_entry != db_store_.end()) { - EXPECT_EQ(metadata_entry->second.version(), version); - } else { - FAIL() << "No component metadata found"; - } - - HintCacheStore::EntryKeyPrefix component_hint_entry_key_prefix = - HintCacheStore::GetComponentHintEntryKeyPrefix(base::Version(version)); - for (int i = 0; i < count; ++i) { - std::string host_suffix = GetHostSuffix(i); - HintCacheStore::EntryKey hint_entry_key = - component_hint_entry_key_prefix + host_suffix; - const auto& hint_entry = db_store_.find(hint_entry_key); - if (hint_entry == db_store_.end()) { - FAIL() << "No entry found for component hint: " << hint_entry_key; - continue; - } - - if (!hint_entry->second.has_hint()) { - FAIL() << "Component hint entry does not have hint: " << hint_entry_key; - continue; - } - - EXPECT_EQ(hint_entry->second.hint().key(), host_suffix); - } - } - - // Returns true if the data is present for the given key. - bool IsKeyPresent(const HintCacheStore::EntryKey& entry_key) const { - return db_store_.find(entry_key) != db_store_.end(); - } - - size_t GetDBStoreEntryCount() const { return db_store_.size(); } - size_t GetStoreHintEntryKeyCount() const { - return hint_store_->GetHintEntryKeyCount(); - } - - HintCacheStore* hint_store() { return hint_store_.get(); } - FakeDB* db() { return db_; } - - const HintCacheStore::EntryKey& last_loaded_hint_entry_key() const { - return last_loaded_hint_entry_key_; - } - - proto::Hint* last_loaded_hint() { return last_loaded_hint_.get(); } - - void OnHintLoaded(const HintCacheStore::EntryKey& hint_entry_key, - std::unique_ptr loaded_hint) { - last_loaded_hint_entry_key_ = hint_entry_key; - last_loaded_hint_ = std::move(loaded_hint); - } - - MOCK_METHOD0(OnInitialized, void()); - MOCK_METHOD0(OnUpdateHints, void()); - - private: - FakeDB* db_; - StoreEntryMap db_store_; - std::unique_ptr hint_store_; - - HintCacheStore::EntryKey last_loaded_hint_entry_key_; - std::unique_ptr last_loaded_hint_; - - DISALLOW_COPY_AND_ASSIGN(HintCacheStoreTest); -}; - -TEST_F(HintCacheStoreTest, NoInitialization) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kMissing); - CreateDatabase(); - - histogram_tester.ExpectTotalCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, InitializeFailedOnInitializeWithNoInitialData) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kMissing); - CreateDatabase(); - InitializeDatabase(false /*=success*/); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectTotalCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, InitializeFailedOnLoadMetadataWithNoInitialData) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kMissing); - CreateDatabase(); - InitializeDatabase(true /*=success*/); - - // OnLoadMetadata callback - db()->LoadCallback(false); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 1 /* kLoadMetadataFailed */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, InitializeFailedOnUpdateMetadataNoInitialData) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kMissing); - CreateDatabase(); - - InitializeDatabase(true /*=success*/); - - // OnLoadMetadata callback - db()->LoadCallback(true); - // OnPurgeDatabase callback - db()->UpdateCallback(false); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 2 /* kSchemaMetadataMissing */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, InitializeFailedOnInitializeWithInitialData) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kValid, 10); - CreateDatabase(); - InitializeDatabase(false /*=success*/); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectTotalCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, InitializeFailedOnLoadMetadataWithInitialData) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kValid, 10); - CreateDatabase(); - InitializeDatabase(true /*=success*/); - - // OnLoadMetadata callback - db()->LoadCallback(false); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 1 /* kLoadMetadataFailed */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, - InitializeFailedOnUpdateMetadataWithInvalidSchemaEntry) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kInvalid, 10); - CreateDatabase(); - InitializeDatabase(true /*=success*/); - - // OnLoadMetadata callback - db()->LoadCallback(true); - // OnPurgeDatabase callback - db()->UpdateCallback(false); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 3 /* kSchemaMetadataWrongVersion */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, InitializeFailedOnLoadHintEntryKeysWithInitialData) { - base::HistogramTester histogram_tester; - - SeedInitialData(MetadataSchemaState::kValid, 10, base::Time().Now()); - CreateDatabase(); - InitializeDatabase(true /*=success*/); - - // OnLoadMetadata callback - db()->LoadCallback(true); - // OnLoadHintEntryKeys callback - db()->LoadCallback(false); - - // In the case where initialization fails, the store should be fully purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 0 /* kSuccess */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); -} - -TEST_F(HintCacheStoreTest, InitializeSucceededWithoutSchemaEntry) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kMissing; - SeedInitialData(schema_state); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry and nothing else. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 2 /* kSchemaMetadataMissing */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, InitializeSucceededWithInvalidSchemaEntry) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kInvalid; - SeedInitialData(schema_state); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry and nothing else. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 3 /* kSchemaMetadataWrongVersion */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, InitializeSucceededWithValidSchemaEntry) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry and nothing else. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 4 /* kComponentMetadataMissing*/, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 5 /* kFetchedMetadataMissing*/, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 6 /* kComponentAndFetchedMetadataMissing*/, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, - InitializeSucceededWithInvalidSchemaEntryAndInitialData) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kInvalid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry and nothing else, as - // the initial component hints are all purged. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 3 /* kSchemaMetadataWrongVersion */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, InitializeSucceededWithPurgeExistingData) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state, true /*=purge_existing_data*/); - - // The store should contain the schema metadata entry and nothing else. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - - histogram_tester.ExpectTotalCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, - InitializeSucceededWithValidSchemaEntryAndInitialData) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t component_hint_count = 10; - SeedInitialData(schema_state, component_hint_count, base::Time().Now()); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry, the component metadata - // entry, and all of the initial component hints. - EXPECT_EQ(GetDBStoreEntryCount(), - static_cast(component_hint_count + 3)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), component_hint_count); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - ExpectComponentHintsPresent(kDefaultComponentVersion, component_hint_count); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 0 /* kSuccess */, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, - InitializeSucceededWithValidSchemaEntryAndComponentDataOnly) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t component_hint_count = 10; - SeedInitialData(schema_state, component_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry, the component metadata - // entry, and all of the initial component hints. - EXPECT_EQ(GetDBStoreEntryCount(), - static_cast(component_hint_count + 2)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), component_hint_count); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - ExpectComponentHintsPresent(kDefaultComponentVersion, component_hint_count); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 4 /* kComponentMetadataMissing*/, 0); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 5 /* kFetchedMetadataMissing*/, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 6 /* kComponentAndFetchedMetadataMissing*/, 0); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, - InitializeSucceededWithValidSchemaEntryAndFetchedMetaData) { - base::HistogramTester histogram_tester; - - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t component_hint_count = 0; - SeedInitialData(schema_state, component_hint_count, base::Time().Now()); - CreateDatabase(); - InitializeStore(schema_state); - - // The store should contain the schema metadata entry, the component metadata - // entry, and all of the initial component hints. - EXPECT_EQ(GetDBStoreEntryCount(), - static_cast(component_hint_count + 2)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), component_hint_count); - - EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", - 4 /* kComponentMetadataMissing*/, 1); - - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, - 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); -} - -TEST_F(HintCacheStoreTest, - CreateComponentUpdateDataFailsForUninitializedStore) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - - // HintUpdateData for a component update should only be created if the store - // is initialized. - EXPECT_FALSE(hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion))); -} - -TEST_F(HintCacheStoreTest, CreateComponentUpdateDataFailsForEarlierVersion) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state); - - // No HintUpdateData for a component update should be created when the - // component version of the update is older than the store's component - // version. - EXPECT_FALSE(hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version("0.0.0"))); -} - -TEST_F(HintCacheStoreTest, CreateComponentUpdateDataFailsForCurrentVersion) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state); - - // No HintUpdateData should be created when the component version of the - // update is the same as the store's component version. - EXPECT_FALSE(hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kDefaultComponentVersion))); -} - -TEST_F(HintCacheStoreTest, - CreateComponentUpdateDataSucceedsWithNoPreexistingVersion) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state); - CreateDatabase(); - InitializeStore(schema_state); - - // HintUpdateData for a component update should be created when there is no - // pre-existing component in the store. - EXPECT_TRUE(hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kDefaultComponentVersion))); -} - -TEST_F(HintCacheStoreTest, CreateComponentUpdateDataSucceedsForNewerVersion) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state); - - // HintUpdateData for a component update should be created when the component - // version of the update is newer than the store's component version. - EXPECT_TRUE(hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion))); -} - -TEST_F(HintCacheStoreTest, UpdateComponentHintsUpdateEntriesFails) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), 5); - - UpdateComponentHints(std::move(update_data), false /*update_success*/); - - // The store should be purged if the component data update fails. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); -} - -TEST_F(HintCacheStoreTest, UpdateComponentHintsGetKeysFails) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - SeedInitialData(schema_state, 10); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), 5); - - UpdateComponentHints(std::move(update_data), true /*update_success*/, - false /*load_hints_keys_success*/); - - // The store should be purged if loading the keys after the component update - // fails. - EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); - EXPECT_EQ(GetStoreHintEntryKeyCount(), static_cast(0)); -} - -TEST_F(HintCacheStoreTest, UpdateComponentHints) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - size_t update_hint_count = 5; - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), update_hint_count); - UpdateComponentHints(std::move(update_data)); - - // When the component update succeeds, the store should contain the schema - // metadata entry, the component metadata entry, and all of the update's - // component hints. - EXPECT_EQ(GetDBStoreEntryCount(), update_hint_count + 2); - EXPECT_EQ(GetStoreHintEntryKeyCount(), update_hint_count); - ExpectComponentHintsPresent(kUpdateComponentVersion, update_hint_count); -} - -TEST_F(HintCacheStoreTest, UpdateComponentHintsAfterInitializationDataPurge) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - size_t update_hint_count = 5; - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state, true /*=purge_existing_data*/); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), update_hint_count); - UpdateComponentHints(std::move(update_data)); - - // When the component update succeeds, the store should contain the schema - // metadata entry, the component metadata entry, and all of the update's - // component hints. - EXPECT_EQ(GetDBStoreEntryCount(), update_hint_count + 2); - EXPECT_EQ(GetStoreHintEntryKeyCount(), update_hint_count); - ExpectComponentHintsPresent(kUpdateComponentVersion, update_hint_count); -} - -TEST_F(HintCacheStoreTest, CreateComponentDataWithAlreadyUpdatedVersionFails) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - size_t update_hint_count = 5; - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), update_hint_count); - UpdateComponentHints(std::move(update_data)); - - // HintUpdateData for the component update should not be created for a second - // component update with the same version as the first component update. - EXPECT_FALSE(hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion))); -} - -TEST_F(HintCacheStoreTest, UpdateComponentHintsWithUpdatedVersionFails) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - size_t update_hint_count_1 = 5; - size_t update_hint_count_2 = 15; - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - // Create two updates for the same component version with different counts. - std::unique_ptr update_data_1 = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - std::unique_ptr update_data_2 = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data_1); - SeedComponentUpdateData(update_data_1.get(), update_hint_count_1); - ASSERT_TRUE(update_data_2); - SeedComponentUpdateData(update_data_2.get(), update_hint_count_2); - - // Update the component data with the same component version twice: - // first with |update_data_1| and then with |update_data_2|. - UpdateComponentHints(std::move(update_data_1)); - - EXPECT_CALL(*this, OnUpdateHints()); - hint_store()->UpdateComponentHints( - std::move(update_data_2), - base::BindOnce(&HintCacheStoreTest::OnUpdateHints, - base::Unretained(this))); - - // Verify that the store is populated with the component data from - // |update_data_1| and not |update_data_2|. - EXPECT_EQ(GetDBStoreEntryCount(), update_hint_count_1 + 2); - EXPECT_EQ(GetStoreHintEntryKeyCount(), update_hint_count_1); - ExpectComponentHintsPresent(kUpdateComponentVersion, update_hint_count_1); -} - -TEST_F(HintCacheStoreTest, LoadHintOnUnavailableStore) { - size_t initial_hint_count = 10; - SeedInitialData(MetadataSchemaState::kValid, initial_hint_count); - CreateDatabase(); - - const HintCacheStore::EntryKey kInvalidEntryKey = "invalid"; - hint_store()->LoadHint(kInvalidEntryKey, - base::BindOnce(&HintCacheStoreTest::OnHintLoaded, - base::Unretained(this))); - - // Verify that the OnHintLoaded callback runs when the store is unavailable - // and that both the key and the hint were correctly set in it. - EXPECT_EQ(last_loaded_hint_entry_key(), kInvalidEntryKey); - EXPECT_FALSE(last_loaded_hint()); -} - -TEST_F(HintCacheStoreTest, LoadHintFailure) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t hint_count = 10; - SeedInitialData(schema_state, hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - const HintCacheStore::EntryKey kInvalidEntryKey = "invalid"; - hint_store()->LoadHint(kInvalidEntryKey, - base::BindOnce(&HintCacheStoreTest::OnHintLoaded, - base::Unretained(this))); - - // OnLoadHint callback - db()->GetCallback(false); - - // Verify that the OnHintLoaded callback runs when the store is unavailable - // and that both the key and the hint were correctly set in it. - EXPECT_EQ(last_loaded_hint_entry_key(), kInvalidEntryKey); - EXPECT_FALSE(last_loaded_hint()); -} - -TEST_F(HintCacheStoreTest, LoadHintSuccessInitialData) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t hint_count = 10; - SeedInitialData(schema_state, hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - // Verify that all component hints in the initial data can successfully be - // loaded from the store. - for (size_t i = 0; i < hint_count; ++i) { - std::string host_suffix = GetHostSuffix(i); - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - continue; - } - - hint_store()->LoadHint(hint_entry_key, - base::BindOnce(&HintCacheStoreTest::OnHintLoaded, - base::Unretained(this))); - - // OnLoadHint callback - db()->GetCallback(true); - - EXPECT_EQ(last_loaded_hint_entry_key(), hint_entry_key); - if (!last_loaded_hint()) { - FAIL() << "Loaded hint was null for entry key: " << hint_entry_key; - continue; - } - - EXPECT_EQ(last_loaded_hint()->key(), host_suffix); - } -} - -TEST_F(HintCacheStoreTest, LoadHintSuccessUpdateData) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - size_t update_hint_count = 5; - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), update_hint_count); - UpdateComponentHints(std::move(update_data)); - - // Verify that all component hints within a successful component update can - // be loaded from the store. - for (size_t i = 0; i < update_hint_count; ++i) { - std::string host_suffix = GetHostSuffix(i); - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - continue; - } - - hint_store()->LoadHint(hint_entry_key, - base::BindOnce(&HintCacheStoreTest::OnHintLoaded, - base::Unretained(this))); - - // OnLoadHint callback - db()->GetCallback(true); - - EXPECT_EQ(last_loaded_hint_entry_key(), hint_entry_key); - if (!last_loaded_hint()) { - FAIL() << "Loaded hint was null for entry key: " << hint_entry_key; - continue; - } - - EXPECT_EQ(last_loaded_hint()->key(), host_suffix); - } -} - -TEST_F(HintCacheStoreTest, FindHintEntryKeyOnUnavailableStore) { - size_t initial_hint_count = 10; - SeedInitialData(MetadataSchemaState::kValid, initial_hint_count); - CreateDatabase(); - - std::string host_suffix = GetHostSuffix(0); - HintCacheStore::EntryKey hint_entry_key; - - // Verify that hint entry keys can't be found when the store is unavailable. - EXPECT_FALSE(hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); -} - -TEST_F(HintCacheStoreTest, FindHintEntryKeyInitialData) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t hint_count = 10; - SeedInitialData(schema_state, hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - // Verify that all hints contained within the initial store data are reported - // as being found and hints that are not containd within the initial data are - // properly reported as not being found. - for (size_t i = 0; i < hint_count * 2; ++i) { - std::string host_suffix = GetHostSuffix(i); - HintCacheStore::EntryKey hint_entry_key; - bool success = hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key); - EXPECT_EQ(success, i < hint_count); - } -} - -TEST_F(HintCacheStoreTest, FindHintEntryKeyUpdateData) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - size_t update_hint_count = 5; - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - SeedComponentUpdateData(update_data.get(), update_hint_count); - UpdateComponentHints(std::move(update_data)); - - // Verify that all hints contained within the component update are reported - // by the store as being found and hints that are not containd within the - // component update are properly reported as not being found. - for (size_t i = 0; i < update_hint_count * 2; ++i) { - std::string host_suffix = GetHostSuffix(i); - HintCacheStore::EntryKey hint_entry_key; - bool success = hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key); - EXPECT_EQ(success, i < update_hint_count); - } -} - -TEST_F(HintCacheStoreTest, FetchedHintsMetadataStored) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - base::Time update_time = base::Time().Now(); - SeedInitialData(schema_state, 10, update_time); - CreateDatabase(); - InitializeStore(schema_state); - - ExpectFetchedMetadata(update_time); -} - -TEST_F(HintCacheStoreTest, FindHintEntryKeyForFetchedHints) { - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t update_hint_count = 5; - base::Time update_time = base::Time().Now(); - SeedInitialData(schema_state, 0); - CreateDatabase(); - InitializeStore(schema_state); - - std::unique_ptr update_data = - hint_store()->CreateUpdateDataForFetchedHints( - update_time, update_time + optimization_guide::features:: - StoredFetchedHintsFreshnessDuration()); - ASSERT_TRUE(update_data); - SeedFetchedUpdateData(update_data.get(), update_hint_count); - UpdateFetchedHints(std::move(update_data)); - - for (size_t i = 0; i < update_hint_count; ++i) { - std::string host_suffix = GetHostSuffix(i); - HintCacheStore::EntryKey hint_entry_key; - bool success = hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key); - EXPECT_EQ(success, i < update_hint_count); - } -} - -TEST_F(HintCacheStoreTest, FindHintEntryKeyCheckFetchedBeforeComponentHints) { - base::HistogramTester histogram_tester; - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - base::Time update_time = base::Time().Now(); - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - base::Version version("2.0.0"); - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - - proto::Hint hint1; - hint1.set_key("domain1.org"); - hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint1)); - proto::Hint hint2; - hint2.set_key("host.domain2.org"); - hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint2)); - - UpdateComponentHints(std::move(update_data)); - - // Add fetched hints to the store that overlap with the same hosts as the - // initial set. - update_data = hint_store()->CreateUpdateDataForFetchedHints( - update_time, - update_time + - optimization_guide::features::StoredFetchedHintsFreshnessDuration()); - - proto::Hint hint; - hint.set_key("domain2.org"); - hint.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint)); - - UpdateFetchedHints(std::move(update_data)); - - // Hint for host.domain2.org should be a fetched hint ("3_" prefix) - // as fetched hints take priority. - std::string host_suffix = "host.domain2.org"; - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - } - - EXPECT_EQ(hint_entry_key, "3_domain2.org"); - - host_suffix = "subdomain.domain1.org"; - - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - } - - EXPECT_EQ(hint_entry_key, "2_2.0.0_domain1.org"); -} - -TEST_F(HintCacheStoreTest, ClearFetchedHints) { - base::HistogramTester histogram_tester; - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - base::Time update_time = base::Time().Now(); - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - base::Version version("2.0.0"); - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - - proto::Hint hint1; - hint1.set_key("domain1.org"); - hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint1)); - proto::Hint hint2; - hint2.set_key("host.domain2.org"); - hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint2)); - - UpdateComponentHints(std::move(update_data)); - - // Add fetched hints to the store that overlap with the same hosts as the - // initial set. - update_data = hint_store()->CreateUpdateDataForFetchedHints( - update_time, update_time + base::TimeDelta().FromDays(7)); - - proto::Hint fetched_hint1; - fetched_hint1.set_key("domain2.org"); - fetched_hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); - proto::Hint fetched_hint2; - fetched_hint2.set_key("domain3.org"); - fetched_hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); - - UpdateFetchedHints(std::move(update_data)); - - // Hint for host.domain2.org should be a fetched hint ("3_" prefix) - // as fetched hints take priority. - std::string host_suffix = "host.domain2.org"; - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - } - - EXPECT_EQ(hint_entry_key, "3_domain2.org"); - - host_suffix = "subdomain.domain1.org"; - - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - } - - EXPECT_EQ(hint_entry_key, "2_2.0.0_domain1.org"); - - // Remove the fetched hints from the HintCacheStore. - ClearFetchedHintsFromDatabase(); - - host_suffix = "domain1.org"; - // Component hint should still exist. - EXPECT_TRUE(hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); - - host_suffix = "domain3.org"; - // Fetched hint should not still exist. - EXPECT_FALSE(hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); - - // Add Components back - newer version. - base::Version version3("3.0.0"); - std::unique_ptr update_data2 = - hint_store()->MaybeCreateUpdateDataForComponentHints(version3); - - ASSERT_TRUE(update_data2); - - proto::Hint new_hint2; - new_hint2.set_key("domain2.org"); - new_hint2.set_key_representation(proto::HOST_SUFFIX); - update_data2->MoveHintIntoUpdateData(std::move(new_hint2)); - - UpdateComponentHints(std::move(update_data2)); - - host_suffix = "host.domain2.org"; - EXPECT_TRUE(hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); - - update_data = hint_store()->CreateUpdateDataForFetchedHints( - update_time, - update_time + - optimization_guide::features::StoredFetchedHintsFreshnessDuration()); - proto::Hint new_hint; - new_hint.set_key("domain1.org"); - new_hint.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(new_hint)); - - UpdateFetchedHints(std::move(update_data)); - - // Add fetched hints to the store that overlap with the same hosts as the - // initial set. - host_suffix = "subdomain.domain1.org"; - - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - } - - EXPECT_EQ(hint_entry_key, "3_domain1.org"); -} - -TEST_F(HintCacheStoreTest, FetchHintsPurgeExpiredFetchedHints) { - base::HistogramTester histogram_tester; - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - base::Time update_time = base::Time().Now(); - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - base::Version version("2.0.0"); - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - - proto::Hint hint1; - hint1.set_key("domain1.org"); - hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint1)); - proto::Hint hint2; - hint2.set_key("host.domain2.org"); - hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint2)); - - UpdateComponentHints(std::move(update_data)); - - // Add fetched hints to the store that overlap with the same hosts as the - // initial set. - update_data = hint_store()->CreateUpdateDataForFetchedHints( - update_time, update_time + base::TimeDelta().FromDays(7)); - - proto::Hint fetched_hint1; - fetched_hint1.set_key("domain2.org"); - fetched_hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); - proto::Hint fetched_hint2; - fetched_hint2.set_key("domain3.org"); - fetched_hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); - - UpdateFetchedHints(std::move(update_data)); - - // Add expired fetched hints to the store. - update_data = hint_store()->CreateUpdateDataForFetchedHints( - update_time, update_time - base::TimeDelta().FromDays(7)); - - proto::Hint fetched_hint3; - fetched_hint1.set_key("domain4.org"); - fetched_hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); - proto::Hint fetched_hint4; - fetched_hint2.set_key("domain5.org"); - fetched_hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); - - UpdateFetchedHints(std::move(update_data)); - - PurgeExpiredFetchedHints(); - - HintCacheStore::EntryKey hint_entry_key; - EXPECT_FALSE(hint_store()->FindHintEntryKey("domain4.org", &hint_entry_key)); - EXPECT_FALSE(hint_store()->FindHintEntryKey("domain5.org", &hint_entry_key)); - EXPECT_TRUE(hint_store()->FindHintEntryKey("domain2.org", &hint_entry_key)); - EXPECT_TRUE(hint_store()->FindHintEntryKey("domain3.org", &hint_entry_key)); -} - -TEST_F(HintCacheStoreTest, FetchedHintsLoadExpiredHint) { - base::HistogramTester histogram_tester; - MetadataSchemaState schema_state = MetadataSchemaState::kValid; - size_t initial_hint_count = 10; - base::Time update_time = base::Time().Now(); - SeedInitialData(schema_state, initial_hint_count); - CreateDatabase(); - InitializeStore(schema_state); - - base::Version version("2.0.0"); - std::unique_ptr update_data = - hint_store()->MaybeCreateUpdateDataForComponentHints( - base::Version(kUpdateComponentVersion)); - ASSERT_TRUE(update_data); - - proto::Hint hint1; - hint1.set_key("domain1.org"); - hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint1)); - proto::Hint hint2; - hint2.set_key("host.domain2.org"); - hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(hint2)); - - UpdateComponentHints(std::move(update_data)); - - // Add fetched hints to the store that expired. - update_data = hint_store()->CreateUpdateDataForFetchedHints( - update_time, update_time - base::TimeDelta().FromDays(10)); - - proto::Hint fetched_hint1; - fetched_hint1.set_key("domain2.org"); - fetched_hint1.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); - proto::Hint fetched_hint2; - fetched_hint2.set_key("domain3.org"); - fetched_hint2.set_key_representation(proto::HOST_SUFFIX); - update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); - - UpdateFetchedHints(std::move(update_data)); - - // Hint for host.domain2.org should be a fetched hint ("3_" prefix) - // as fetched hints take priority. - std::string host_suffix = "host.domain2.org"; - HintCacheStore::EntryKey hint_entry_key; - if (!hint_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { - FAIL() << "Hint entry not found for host suffix: " << host_suffix; - } - EXPECT_EQ(hint_entry_key, "3_domain2.org"); - hint_store()->LoadHint(hint_entry_key, - base::BindOnce(&HintCacheStoreTest::OnHintLoaded, - base::Unretained(this))); - - // OnLoadHint callback - db()->GetCallback(true); - - // |hint_entry_key| will be a fetched hint but the entry will be empty. - EXPECT_EQ(last_loaded_hint_entry_key(), hint_entry_key); - EXPECT_FALSE(last_loaded_hint()); - histogram_tester.ExpectBucketCount( - "OptimizationGuide.HintCacheStore.OnLoadHint.FetchedHintExpired", true, - 1); -} - -} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/hint_cache_unittest.cc b/chromium/components/optimization_guide/hint_cache_unittest.cc index 6dd078d04e0..71a52b5ebec 100644 --- a/chromium/components/optimization_guide/hint_cache_unittest.cc +++ b/chromium/components/optimization_guide/hint_cache_unittest.cc @@ -14,8 +14,8 @@ #include "base/strings/string_number_conversions.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/task_environment.h" -#include "components/optimization_guide/hint_cache_store.h" #include "components/optimization_guide/optimization_guide_features.h" +#include "components/optimization_guide/optimization_guide_store.h" #include "components/optimization_guide/proto_database_provider_test_base.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" @@ -41,15 +41,15 @@ class HintCacheTest : public ProtoDatabaseProviderTestBase { } protected: - // Creates and initializes the hint cache and hint cache store and waits for - // the callback indicating that initialization is complete. + // Creates and initializes the hint cache and optimization guide store and + // waits for the callback indicating that initialization is complete. void CreateAndInitializeHintCache(int memory_cache_size, bool purge_existing_data = false) { auto database_path = temp_dir_.GetPath(); auto database_task_runner = task_environment_.GetMainThreadTaskRunner(); hint_cache_ = std::make_unique( - std::make_unique(db_provider_.get(), database_path, - database_task_runner), + std::make_unique( + db_provider_.get(), database_path, database_task_runner), memory_cache_size); is_store_initialized_ = false; hint_cache_->Initialize(purge_existing_data, @@ -77,7 +77,7 @@ class HintCacheTest : public ProtoDatabaseProviderTestBase { // Updates the cache with |component_data| and waits for callback indicating // that the update is complete. - void UpdateComponentHints(std::unique_ptr component_data) { + void UpdateComponentHints(std::unique_ptr component_data) { are_component_hints_updated_ = false; hint_cache_->UpdateComponentHints( std::move(component_data), @@ -147,7 +147,7 @@ TEST_F(HintCacheTest, ComponentUpdate) { CreateAndInitializeHintCache(kMemoryCacheSize); base::Version version("2.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); ASSERT_TRUE(update_data); @@ -182,7 +182,7 @@ TEST_F(HintCacheTest, ComponentUpdateWithSameVersionIgnored) { CreateAndInitializeHintCache(kMemoryCacheSize); base::Version version("2.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); ASSERT_TRUE(update_data); @@ -198,7 +198,7 @@ TEST_F(HintCacheTest, ComponentUpdateWithEarlierVersionIgnored) { base::Version version_1("1.0.0"); base::Version version_2("2.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version_2); ASSERT_TRUE(update_data); @@ -214,7 +214,7 @@ TEST_F(HintCacheTest, ComponentUpdateWithLaterVersionProcessed) { base::Version version_1("1.0.0"); base::Version version_2("2.0.0"); - std::unique_ptr update_data_1 = + std::unique_ptr update_data_1 = hint_cache()->MaybeCreateUpdateDataForComponentHints(version_1); ASSERT_TRUE(update_data_1); @@ -243,7 +243,7 @@ TEST_F(HintCacheTest, ComponentUpdateWithLaterVersionProcessed) { EXPECT_TRUE(hint_cache()->HasHint("host.subdomain.domain.org")); EXPECT_TRUE(hint_cache()->HasHint("subhost.host.subdomain.domain.org")); - std::unique_ptr update_data_2 = + std::unique_ptr update_data_2 = hint_cache()->MaybeCreateUpdateDataForComponentHints(version_2); ASSERT_TRUE(update_data_2); @@ -284,7 +284,7 @@ TEST_F(HintCacheTest, ComponentHintsAvailableAfterRestart) { base::Version version("2.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); if (i == 0) { ASSERT_TRUE(update_data); @@ -329,7 +329,7 @@ TEST_F(HintCacheTest, ComponentHintsUpdatableAfterRestartWithPurge) { base::Version version("2.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); ASSERT_TRUE(update_data); @@ -370,7 +370,7 @@ TEST_F(HintCacheTest, ComponentHintsNotRetainedAfterRestartWithPurge) { base::Version version("2.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); if (i == 0) { ASSERT_TRUE(update_data); @@ -416,7 +416,7 @@ TEST_F(HintCacheTest, TestMemoryCacheLeastRecentlyUsedPurge) { CreateAndInitializeHintCache(kMemoryCacheSize); base::Version version("1.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); ASSERT_TRUE(update_data); @@ -456,7 +456,7 @@ TEST_F(HintCacheTest, TestHostNotInCache) { CreateAndInitializeHintCache(kMemoryCacheSize); base::Version version("1.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); ASSERT_TRUE(update_data); @@ -477,7 +477,7 @@ TEST_F(HintCacheTest, TestMemoryCacheLoadCallback) { CreateAndInitializeHintCache(kMemoryCacheSize); base::Version version("1.0.0"); - std::unique_ptr update_data = + std::unique_ptr update_data = hint_cache()->MaybeCreateUpdateDataForComponentHints(version); ASSERT_TRUE(update_data); @@ -501,8 +501,8 @@ TEST_F(HintCacheTest, StoreValidFetchedHints) { const int kMemoryCacheSize = 5; CreateAndInitializeHintCache(kMemoryCacheSize); - // Default update time for empty hint cache store is base::Time(). - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), base::Time()); + // Default update time for empty optimization guide store is base::Time(). + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), base::Time()); std::unique_ptr get_hints_response = std::make_unique(); @@ -518,7 +518,7 @@ TEST_F(HintCacheTest, StoreValidFetchedHints) { EXPECT_TRUE(are_fetched_hints_updated()); // Next update time for hints should be updated. - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), stored_time); + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), stored_time); } TEST_F(HintCacheTest, ParseEmptyFetchedHints) { @@ -532,7 +532,7 @@ TEST_F(HintCacheTest, ParseEmptyFetchedHints) { UpdateFetchedHintsAndWait(std::move(get_hints_response), stored_time); // Empty Fetched Hints causes the metadata entry to be updated. EXPECT_TRUE(are_fetched_hints_updated()); - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), stored_time); + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), stored_time); } TEST_F(HintCacheTest, StoreValidFetchedHintsWithServerProvidedExpiryTime) { @@ -541,8 +541,8 @@ TEST_F(HintCacheTest, StoreValidFetchedHintsWithServerProvidedExpiryTime) { const int kFetchedHintExpirationSecs = 60; CreateAndInitializeHintCache(kMemoryCacheSize); - // Default update time for empty hint cache store is base::Time(). - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), base::Time()); + // Default update time for empty optimization guide store is base::Time(). + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), base::Time()); std::unique_ptr get_hints_response = std::make_unique(); @@ -562,7 +562,7 @@ TEST_F(HintCacheTest, StoreValidFetchedHintsWithServerProvidedExpiryTime) { EXPECT_TRUE(are_fetched_hints_updated()); // Next update time for hints should be updated. - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), stored_time); + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), stored_time); LoadHint("host.domain.org"); // HISTOGRAM TEST! @@ -576,8 +576,8 @@ TEST_F(HintCacheTest, StoreValidFetchedHintsWithDefaultExpiryTime) { const int kMemoryCacheSize = 5; CreateAndInitializeHintCache(kMemoryCacheSize); - // Default update time for empty hint cache store is base::Time(). - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), base::Time()); + // Default update time for empty optimization guide store is base::Time(). + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), base::Time()); std::unique_ptr get_hints_response = std::make_unique(); @@ -593,7 +593,7 @@ TEST_F(HintCacheTest, StoreValidFetchedHintsWithDefaultExpiryTime) { EXPECT_TRUE(are_fetched_hints_updated()); // Next update time for hints should be updated. - EXPECT_EQ(hint_cache()->FetchedHintsUpdateTime(), stored_time); + EXPECT_EQ(hint_cache()->GetFetchedHintsUpdateTime(), stored_time); LoadHint("host.domain.org"); histogram_tester.ExpectTimeBucketCount( diff --git a/chromium/components/optimization_guide/hint_update_data.cc b/chromium/components/optimization_guide/hint_update_data.cc deleted file mode 100644 index fac8de39d1f..00000000000 --- a/chromium/components/optimization_guide/hint_update_data.cc +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/optimization_guide/hint_update_data.h" - -#include "components/optimization_guide/hint_cache_store.h" -#include "components/optimization_guide/proto/hint_cache.pb.h" -#include "components/optimization_guide/proto/hints.pb.h" - -namespace optimization_guide { - -// static -std::unique_ptr HintUpdateData::CreateComponentHintUpdateData( - const base::Version& component_version) { - std::unique_ptr update_data(new HintUpdateData( - base::Optional(component_version), - base::Optional(), base::Optional())); - return update_data; -} - -// static -std::unique_ptr HintUpdateData::CreateFetchedHintUpdateData( - base::Time fetch_update_time, - base::Time expiry_time) { - std::unique_ptr update_data( - new HintUpdateData(base::Optional(), - base::Optional(fetch_update_time), - base::Optional(expiry_time))); - return update_data; -} - -HintUpdateData::HintUpdateData(base::Optional component_version, - base::Optional fetch_update_time, - base::Optional expiry_time) - : component_version_(component_version), - fetch_update_time_(fetch_update_time), - expiry_time_(expiry_time), - entries_to_save_(std::make_unique()) { - DCHECK_NE(!component_version_, !fetch_update_time_); - - if (component_version_.has_value()) { - hint_entry_key_prefix_ = - HintCacheStore::GetComponentHintEntryKeyPrefix(*component_version_); - - // Add a component metadata entry for the component's version. - proto::StoreEntry metadata_component_entry; - - metadata_component_entry.set_entry_type(static_cast( - HintCacheStore::StoreEntryType::kMetadata)); - metadata_component_entry.set_version(component_version_->GetString()); - entries_to_save_->emplace_back( - HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kComponent), - std::move(metadata_component_entry)); - } else if (fetch_update_time_.has_value()) { - hint_entry_key_prefix_ = HintCacheStore::GetFetchedHintEntryKeyPrefix(); - proto::StoreEntry metadata_fetched_entry; - metadata_fetched_entry.set_entry_type(static_cast( - HintCacheStore::StoreEntryType::kMetadata)); - metadata_fetched_entry.set_update_time_secs( - fetch_update_time_->ToDeltaSinceWindowsEpoch().InSeconds()); - entries_to_save_->emplace_back(HintCacheStore::GetMetadataTypeEntryKey( - HintCacheStore::MetadataType::kFetched), - std::move(metadata_fetched_entry)); - } else { - NOTREACHED(); - } - // |this| may be modified on another thread after construction but all - // future modifications, from that call forward, must be made on the same - // thread. - DETACH_FROM_SEQUENCE(sequence_checker_); -} - -HintUpdateData::~HintUpdateData() {} - -void HintUpdateData::MoveHintIntoUpdateData(proto::Hint&& hint) { - // All future modifications must be made by the same thread. Note, |this| may - // have been constructed on another thread. - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK(!hint_entry_key_prefix_.empty()); - - // To avoid any unnecessary copying, the hint is moved into proto::StoreEntry. - HintCacheStore::EntryKey hint_entry_key = hint_entry_key_prefix_ + hint.key(); - proto::StoreEntry entry_proto; - if (component_version()) { - entry_proto.set_entry_type(static_cast( - HintCacheStore::StoreEntryType::kComponentHint)); - } else if (fetch_update_time()) { - DCHECK(expiry_time()); - entry_proto.set_expiry_time_secs( - expiry_time_->ToDeltaSinceWindowsEpoch().InSeconds()); - entry_proto.set_entry_type(static_cast( - HintCacheStore::StoreEntryType::kFetchedHint)); - } - entry_proto.set_allocated_hint(new proto::Hint(std::move(hint))); - entries_to_save_->emplace_back(std::move(hint_entry_key), - std::move(entry_proto)); -} - -std::unique_ptr HintUpdateData::TakeUpdateEntries() { - // TakeUpdateEntries is not be sequence checked as it only gives up ownership - // of the entries_to_save_ and does not modify any state. - DCHECK(entries_to_save_); - - return std::move(entries_to_save_); -} - -} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/hint_update_data.h b/chromium/components/optimization_guide/hint_update_data.h deleted file mode 100644 index efe175029cc..00000000000 --- a/chromium/components/optimization_guide/hint_update_data.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_OPTIMIZATION_GUIDE_HINT_UPDATE_DATA_H_ -#define COMPONENTS_OPTIMIZATION_GUIDE_HINT_UPDATE_DATA_H_ - -#include - -#include "base/macros.h" -#include "base/optional.h" -#include "base/sequence_checker.h" -#include "base/time/time.h" -#include "base/version.h" -#include "components/leveldb_proto/public/proto_database.h" - -namespace optimization_guide { -namespace proto { -class Hint; -class StoreEntry; -} // namespace proto - -using EntryVector = - leveldb_proto::ProtoDatabase::KeyEntryVector; - -// Holds hint data for updating the HintCacheStore. -class HintUpdateData { - public: - ~HintUpdateData(); - - // Creates an update data object for a component hint update. - static std::unique_ptr CreateComponentHintUpdateData( - const base::Version& component_version); - - // Creates an update data object for a fetched hint update. - static std::unique_ptr CreateFetchedHintUpdateData( - base::Time fetch_update_time, - base::Time expiry_time); - - // Returns the component version of a component hint update. - const base::Optional component_version() const { - return component_version_; - } - - // Returns the next update time for a fetched hint update. - const base::Optional fetch_update_time() const { - return fetch_update_time_; - } - - // Returns the expiry time for the hints in a fetched hint update. - const base::Optional expiry_time() const { return expiry_time_; } - - // Moves |hint| into this update data. After MoveHintIntoUpdateData() is - // called, |hint| is no longer valid. - void MoveHintIntoUpdateData(proto::Hint&& hint); - - // Returns the store entry updates along with ownership to them. - std::unique_ptr TakeUpdateEntries(); - - private: - HintUpdateData(base::Optional component_version, - base::Optional fetch_update_time, - base::Optional expiry_time); - - // The component version of the update data for a component update. - base::Optional component_version_; - - // The time when hints in this update need to be updated for a fetch update. - base::Optional fetch_update_time_; - - // The time when hints in this update expire. - base::Optional expiry_time_; - - // The prefix to add to the key of every hint entry. It is set - // during construction for appropriate type of update. - std::string hint_entry_key_prefix_; - - // The vector of entries to save. - std::unique_ptr entries_to_save_; - - SEQUENCE_CHECKER(sequence_checker_); - - DISALLOW_COPY_AND_ASSIGN(HintUpdateData); -}; - -} // namespace optimization_guide - -#endif // COMPONENTS_OPTIMIZATION_GUIDE_HINT_UPDATE_DATA_H_ diff --git a/chromium/components/optimization_guide/hint_update_data_unittest.cc b/chromium/components/optimization_guide/hint_update_data_unittest.cc deleted file mode 100644 index 4ab091a27ab..00000000000 --- a/chromium/components/optimization_guide/hint_update_data_unittest.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/optimization_guide/hint_update_data.h" - -#include -#include - -#include "base/macros.h" -#include "base/time/time.h" -#include "base/version.h" -#include "components/optimization_guide/optimization_guide_features.h" -#include "components/optimization_guide/proto/hint_cache.pb.h" -#include "components/optimization_guide/proto/hints.pb.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace optimization_guide { - -namespace { - -TEST(HintUpdateDataTest, BuildComponentHintUpdateData) { - // Verify creating a Component Hint update package. - base::Version v1("1.2.3.4"); - proto::Hint hint1; - hint1.set_key("foo.org"); - hint1.set_key_representation(proto::HOST_SUFFIX); - proto::PageHint* page_hint1 = hint1.add_page_hints(); - page_hint1->set_page_pattern("slowpage"); - proto::Hint hint2; - hint2.set_key("bar.com"); - hint2.set_key_representation(proto::HOST_SUFFIX); - proto::PageHint* page_hint2 = hint2.add_page_hints(); - page_hint2->set_page_pattern("slowpagealso"); - - std::unique_ptr component_update = - HintUpdateData::CreateComponentHintUpdateData(v1); - component_update->MoveHintIntoUpdateData(std::move(hint1)); - component_update->MoveHintIntoUpdateData(std::move(hint2)); - EXPECT_TRUE(component_update->component_version().has_value()); - EXPECT_FALSE(component_update->fetch_update_time().has_value()); - EXPECT_EQ(v1, *component_update->component_version()); - // Verify there are 3 store entries: 1 for the metadata entry plus - // the 2 added hint entries. - EXPECT_EQ(3ul, component_update->TakeUpdateEntries()->size()); -} - -TEST(HintUpdateDataTest, BuildFetchUpdateData) { - // Verify creating a Fetched Hint update package. - base::Time update_time = base::Time::Now(); - proto::Hint hint1; - hint1.set_key("foo.org"); - hint1.set_key_representation(proto::HOST_SUFFIX); - proto::PageHint* page_hint1 = hint1.add_page_hints(); - page_hint1->set_page_pattern("slowpage"); - - std::unique_ptr fetch_update = - HintUpdateData::CreateFetchedHintUpdateData( - update_time, update_time + optimization_guide::features:: - StoredFetchedHintsFreshnessDuration()); - fetch_update->MoveHintIntoUpdateData(std::move(hint1)); - EXPECT_FALSE(fetch_update->component_version().has_value()); - EXPECT_TRUE(fetch_update->fetch_update_time().has_value()); - EXPECT_EQ(update_time, *fetch_update->fetch_update_time()); - // Verify there are 2 store entries: 1 for the metadata entry plus - // the 1 added hint entries. - EXPECT_EQ(2ul, fetch_update->TakeUpdateEntries()->size()); -} - -} // namespace - -} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/hints_fetcher.cc b/chromium/components/optimization_guide/hints_fetcher.cc index 3e693015f06..7dc91a330a1 100644 --- a/chromium/components/optimization_guide/hints_fetcher.cc +++ b/chromium/components/optimization_guide/hints_fetcher.cc @@ -50,7 +50,7 @@ HintsFetcher::HintsFetcher( time_clock_(base::DefaultClock::GetInstance()) { url_loader_factory_ = std::move(url_loader_factory); CHECK(optimization_guide_service_url_.SchemeIs(url::kHttpsScheme)); - CHECK(features::IsHintsFetchingEnabled()); + DCHECK(features::IsRemoteFetchingEnabled()); } HintsFetcher::~HintsFetcher() {} @@ -96,17 +96,28 @@ bool HintsFetcher::FetchOptimizationGuideServiceHints( SEQUENCE_CHECKER(sequence_checker_); if (content::GetNetworkConnectionTracker()->IsOffline()) { - std::move(hints_fetched_callback).Run(request_context, base::nullopt); + std::move(hints_fetched_callback) + .Run(request_context, HintsFetcherRequestStatus::kNetworkOffline, + base::nullopt); return false; } - if (url_loader_) + if (active_url_loader_) { + std::move(hints_fetched_callback) + .Run(request_context, HintsFetcherRequestStatus::kFetcherBusy, + base::nullopt); return false; + } std::vector filtered_hosts = GetSizeLimitedHostsDueForHintsRefresh(hosts); - if (filtered_hosts.empty()) + if (filtered_hosts.empty()) { + std::move(hints_fetched_callback) + .Run(request_context, HintsFetcherRequestStatus::kNoHostsToFetch, + base::nullopt); return false; + } + DCHECK_GE(features::MaxHostsForOptimizationGuideServiceHintsFetch(), filtered_hosts.size()); @@ -161,27 +172,27 @@ bool HintsFetcher::FetchOptimizationGuideServiceHints( resource_request->url = optimization_guide_service_url_; resource_request->method = "POST"; - resource_request->load_flags = net::LOAD_BYPASS_PROXY; resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit; - url_loader_ = network::SimpleURLLoader::Create(std::move(resource_request), - traffic_annotation); + active_url_loader_ = network::SimpleURLLoader::Create( + std::move(resource_request), traffic_annotation); - url_loader_->AttachStringForUpload(serialized_request, - "application/x-protobuf"); + active_url_loader_->AttachStringForUpload(serialized_request, + "application/x-protobuf"); UMA_HISTOGRAM_COUNTS_100( "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", filtered_hosts.size()); - // |url_loader_| should not retry on 5xx errors since the server may already - // be overloaded. |url_loader_| should retry on network changes since the - // network stack may receive the connection change event later than |this|. + // |active_url_loader_| should not retry on 5xx errors since the server may + // already be overloaded. |active_url_loader_| should retry on network changes + // since the network stack may receive the connection change event later than + // |this|. static const int kMaxRetries = 1; - url_loader_->SetRetryOptions( + active_url_loader_->SetRetryOptions( kMaxRetries, network::SimpleURLLoader::RETRY_ON_NETWORK_CHANGE); - url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( + active_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( url_loader_factory_.get(), base::BindOnce(&HintsFetcher::OnURLLoadComplete, base::Unretained(this))); @@ -217,9 +228,12 @@ void HintsFetcher::HandleResponse(const std::string& get_hints_response_data, base::TimeTicks::Now() - hints_fetch_start_time_); UpdateHostsSuccessfullyFetched(); std::move(hints_fetched_callback_) - .Run(request_context_, std::move(get_hints_response)); + .Run(request_context_, HintsFetcherRequestStatus::kSuccess, + std::move(get_hints_response)); } else { - std::move(hints_fetched_callback_).Run(request_context_, base::nullopt); + std::move(hints_fetched_callback_) + .Run(request_context_, HintsFetcherRequestStatus::kResponseError, + base::nullopt); } } @@ -278,12 +292,14 @@ void HintsFetcher::OnURLLoadComplete( SEQUENCE_CHECKER(sequence_checker_); int response_code = -1; - if (url_loader_->ResponseInfo() && url_loader_->ResponseInfo()->headers) { - response_code = url_loader_->ResponseInfo()->headers->response_code(); + if (active_url_loader_->ResponseInfo() && + active_url_loader_->ResponseInfo()->headers) { + response_code = + active_url_loader_->ResponseInfo()->headers->response_code(); } - HandleResponse(response_body ? *response_body : "", url_loader_->NetError(), - response_code); - url_loader_.reset(); + HandleResponse(response_body ? *response_body : "", + active_url_loader_->NetError(), response_code); + active_url_loader_.reset(); } std::vector HintsFetcher::GetSizeLimitedHostsDueForHintsRefresh( diff --git a/chromium/components/optimization_guide/hints_fetcher.h b/chromium/components/optimization_guide/hints_fetcher.h index a222218b181..282da8842fb 100644 --- a/chromium/components/optimization_guide/hints_fetcher.h +++ b/chromium/components/optimization_guide/hints_fetcher.h @@ -28,11 +28,34 @@ class SimpleURLLoader; namespace optimization_guide { +// Status of a request to fetch hints. +// This enum must remain synchronized with the enum +// |OptimizationGuideHintsFetcherRequestStatus| in +// tools/metrics/histograms/enums.xml. +enum class HintsFetcherRequestStatus { + // No fetch status known. Used in testing. + kUnknown, + // Fetch request was sent and a response received. + kSuccess, + // Fetch request was sent but no response received. + kResponseError, + // Fetch request not sent because of offline network status. + kNetworkOffline, + // Fetch request not sent because fetcher was busy with another request. + kFetcherBusy, + // Fetch request not sent because the host list was empty. + kNoHostsToFetch, + + // Insert new values before this line. + kMaxValue = kNoHostsToFetch +}; + // Callback to inform the caller that the remote hints have been fetched and // to pass back the fetched hints response from the remote Optimization Guide // Service. using HintsFetchedCallback = base::OnceCallback>)>; // A class to handle requests for optimization hints from a remote Optimization @@ -51,11 +74,11 @@ class HintsFetcher { // Requests hints from the Optimization Guide Service if a request for them is // not already in progress. Returns whether a new request was issued. - // |hints_fetched_callback| is run, passing a GetHintsResponse object, if a - // fetch was successful or passes nullopt if the fetch fails. Virtualized for - // testing. Hints fetcher may fetch hints for only a subset of the provided - // |hosts|. |hosts| should be an ordered list in descending order of - // probability that the hints are needed for that host. + // |hints_fetched_callback| is run once when the outcome of this request is + // determined (whether a request was actually sent or not). + // Virtualized for testing. Hints fetcher may fetch hints for only a subset + // of the provided |hosts|. |hosts| should be an ordered list in descending + // order of probability that the hints are needed for that host. virtual bool FetchOptimizationGuideServiceHints( const std::vector& hosts, optimization_guide::proto::RequestContext request_context, @@ -113,7 +136,7 @@ class HintsFetcher { const GURL optimization_guide_service_url_; // Holds the |URLLoader| for an active hints request. - std::unique_ptr url_loader_; + std::unique_ptr active_url_loader_; // Context of the fetch request. Opaque field that's returned back in the // callback and is also included in the requests to the hints server. @@ -128,7 +151,7 @@ class HintsFetcher { // Clock used for recording time that the hints fetch occurred. const base::Clock* time_clock_; - // Used for creating a |url_loader_| when needed for request hints. + // Used for creating an |active_url_loader_| when needed for request hints. scoped_refptr url_loader_factory_; // The start time of the current hints fetch, used to determine the latency in diff --git a/chromium/components/optimization_guide/hints_fetcher_unittest.cc b/chromium/components/optimization_guide/hints_fetcher_unittest.cc index 1467ffa60e1..5f1f7ebadc8 100644 --- a/chromium/components/optimization_guide/hints_fetcher_unittest.cc +++ b/chromium/components/optimization_guide/hints_fetcher_unittest.cc @@ -44,7 +44,7 @@ class HintsFetcherTest : public testing::Test { &test_url_loader_factory_)) { base::test::ScopedFeatureList scoped_list; scoped_list.InitAndEnableFeatureWithParameters( - features::kOptimizationHintsFetching, {}); + features::kRemoteOptimizationGuideFetching, {}); pref_service_ = std::make_unique(); prefs::RegisterProfilePrefs(pref_service_->registry()); @@ -57,13 +57,19 @@ class HintsFetcherTest : public testing::Test { ~HintsFetcherTest() override {} - void OnHintsFetched(optimization_guide::proto::RequestContext request_context, - base::Optional> - get_hints_response) { + void OnHintsFetched( + optimization_guide::proto::RequestContext request_context, + optimization_guide::HintsFetcherRequestStatus fetcher_request_status, + base::Optional> + get_hints_response) { + fetcher_request_status_ = fetcher_request_status; if (get_hints_response) hints_fetched_ = true; } + optimization_guide::HintsFetcherRequestStatus fetcher_request_status() { + return fetcher_request_status_; + } bool hints_fetched() { return hints_fetched_; } void SetConnectionOffline() { @@ -142,6 +148,8 @@ class HintsFetcherTest : public testing::Test { base::RunLoop().RunUntilIdle(); } + optimization_guide::HintsFetcherRequestStatus fetcher_request_status_ = + optimization_guide::HintsFetcherRequestStatus::kUnknown; bool hints_fetched_ = false; base::test::TaskEnvironment task_environment_; @@ -162,6 +170,8 @@ TEST_F(HintsFetcherTest, FetchOptimizationGuideServiceHints) { EXPECT_TRUE(FetchHints(std::vector{"foo.com"})); VerifyHasPendingFetchRequests(); EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK)); + EXPECT_EQ(optimization_guide::HintsFetcherRequestStatus::kSuccess, + fetcher_request_status()); EXPECT_TRUE(hints_fetched()); histogram_tester.ExpectTotalCount( @@ -179,10 +189,14 @@ TEST_F(HintsFetcherTest, FetchInProgress) { // |fetch_in_progress_| should cause early exit. EXPECT_TRUE(FetchHints(std::vector{"foo.com"})); EXPECT_FALSE(FetchHints(std::vector{"bar.com"})); + EXPECT_EQ(optimization_guide::HintsFetcherRequestStatus::kFetcherBusy, + fetcher_request_status()); // Once response arrives, check to make sure a new fetch can start. SimulateResponse(response_content, net::HTTP_OK); EXPECT_TRUE(FetchHints(std::vector{"bar.com"})); + EXPECT_EQ(optimization_guide::HintsFetcherRequestStatus::kSuccess, + fetcher_request_status()); } // Tests that the hints are refreshed again for hosts for whom hints were @@ -250,6 +264,8 @@ TEST_F(HintsFetcherTest, FetchReturned404) { // Send a 404 to HintsFetcher. SimulateResponse(response_content, net::HTTP_NOT_FOUND); EXPECT_FALSE(hints_fetched()); + EXPECT_EQ(optimization_guide::HintsFetcherRequestStatus::kResponseError, + fetcher_request_status()); // Make sure histogram not recorded on bad response. histogram_tester.ExpectTotalCount( @@ -264,6 +280,8 @@ TEST_F(HintsFetcherTest, FetchReturnBadResponse) { VerifyHasPendingFetchRequests(); EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK)); EXPECT_FALSE(hints_fetched()); + EXPECT_EQ(optimization_guide::HintsFetcherRequestStatus::kResponseError, + fetcher_request_status()); // Make sure histogram not recorded on bad response. histogram_tester.ExpectTotalCount( @@ -277,6 +295,8 @@ TEST_F(HintsFetcherTest, FetchAttemptWhenNetworkOffline) { std::string response_content; EXPECT_FALSE(FetchHints(std::vector{"foo.com"})); EXPECT_FALSE(hints_fetched()); + EXPECT_EQ(optimization_guide::HintsFetcherRequestStatus::kNetworkOffline, + fetcher_request_status()); // Make sure histogram not recorded on bad response. histogram_tester.ExpectTotalCount( diff --git a/chromium/components/optimization_guide/hints_processing_util.cc b/chromium/components/optimization_guide/hints_processing_util.cc index 2d5f690a83d..ef4969d7093 100644 --- a/chromium/components/optimization_guide/hints_processing_util.cc +++ b/chromium/components/optimization_guide/hints_processing_util.cc @@ -9,8 +9,8 @@ #include "base/containers/flat_set.h" #include "base/metrics/field_trial_params.h" #include "base/strings/stringprintf.h" -#include "components/optimization_guide/hint_update_data.h" #include "components/optimization_guide/optimization_guide_features.h" +#include "components/optimization_guide/store_update_data.h" #include "components/optimization_guide/url_pattern_with_wildcards.h" #include "url/gurl.h" @@ -86,9 +86,9 @@ std::string HashHostForDictionary(const std::string& host) { } bool ProcessHints(google::protobuf::RepeatedPtrField* hints, - optimization_guide::HintUpdateData* hint_update_data) { + optimization_guide::StoreUpdateData* update_data) { // If there's no update data, then there's nothing to do. - if (!hint_update_data) + if (!update_data) return false; base::flat_set seen_host_suffixes; @@ -126,7 +126,7 @@ bool ProcessHints(google::protobuf::RepeatedPtrField* hints, // data. // WARNING: Do not use |hint| after this call. Its contents will no // longer be valid. - hint_update_data->MoveHintIntoUpdateData(std::move(hint)); + update_data->MoveHintIntoUpdateData(std::move(hint)); did_process_hints = true; } } diff --git a/chromium/components/optimization_guide/hints_processing_util.h b/chromium/components/optimization_guide/hints_processing_util.h index f8aadc70362..cea8bb63642 100644 --- a/chromium/components/optimization_guide/hints_processing_util.h +++ b/chromium/components/optimization_guide/hints_processing_util.h @@ -13,7 +13,7 @@ class GURL; namespace optimization_guide { -class HintUpdateData; +class StoreUpdateData; // Returns the string representation of the optimization type. std::string GetStringNameForOptimizationType( @@ -42,11 +42,11 @@ const proto::PageHint* FindPageHintForURL(const GURL& gurl, std::string HashHostForDictionary(const std::string& host); // Verifies and processes |hints| and places the ones it supports into -// |hint_update_data|. +// |update_data|. // -// Returns true if there was at least one hint moved into |hint_update_data|. +// Returns true if there was at least one hint moved into |update_data|. bool ProcessHints(google::protobuf::RepeatedPtrField* hints, - HintUpdateData* hint_update_data); + StoreUpdateData* update_data); // Converts |proto_ect| into a net::EffectiveConnectionType. net::EffectiveConnectionType ConvertProtoEffectiveConnectionType( diff --git a/chromium/components/optimization_guide/hints_processing_util_unittest.cc b/chromium/components/optimization_guide/hints_processing_util_unittest.cc index a1a258a085b..f83aad8ce79 100644 --- a/chromium/components/optimization_guide/hints_processing_util_unittest.cc +++ b/chromium/components/optimization_guide/hints_processing_util_unittest.cc @@ -4,9 +4,9 @@ #include "components/optimization_guide/hints_processing_util.h" -#include "components/optimization_guide/hint_update_data.h" #include "components/optimization_guide/proto/hint_cache.pb.h" #include "components/optimization_guide/proto/hints.pb.h" +#include "components/optimization_guide/store_update_data.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" @@ -79,8 +79,8 @@ TEST(HintsProcessingUtilTest, ProcessHintsWithNoPageHintsAndUpdateData) { google::protobuf::RepeatedPtrField hints; *(hints.Add()) = hint; - std::unique_ptr update_data = - HintUpdateData::CreateComponentHintUpdateData(base::Version("1.0.0")); + std::unique_ptr update_data = + StoreUpdateData::CreateComponentStoreUpdateData(base::Version("1.0.0")); EXPECT_FALSE(ProcessHints(&hints, update_data.get())); // Verify there is 1 store entries: 1 for the metadata entry. EXPECT_EQ(1ul, update_data->TakeUpdateEntries()->size()); @@ -107,8 +107,8 @@ TEST(HintsProcessingUtilTest, ProcessHintsWithPageHintsAndUpdateData) { no_page_hints_hint.set_key_representation(proto::HOST_SUFFIX); *(hints.Add()) = no_page_hints_hint; - std::unique_ptr update_data = - HintUpdateData::CreateComponentHintUpdateData(base::Version("1.0.0")); + std::unique_ptr update_data = + StoreUpdateData::CreateComponentStoreUpdateData(base::Version("1.0.0")); EXPECT_TRUE(ProcessHints(&hints, update_data.get())); // Verify there are 2 store entries: 1 for the metadata entry plus // the 1 added hint entries. diff --git a/chromium/components/optimization_guide/optimization_guide_constants.cc b/chromium/components/optimization_guide/optimization_guide_constants.cc index 573d741f648..c38b6c211c0 100644 --- a/chromium/components/optimization_guide/optimization_guide_constants.cc +++ b/chromium/components/optimization_guide/optimization_guide_constants.cc @@ -11,10 +11,18 @@ const base::FilePath::CharType kUnindexedHintsFileName[] = const char kRulesetFormatVersionString[] = "1.0.0"; -const char kOptimizationGuideServiceDefaultURL[] = +const char kOptimizationGuideServiceGetHintsDefaultURL[] = "https://optimizationguide-pa.googleapis.com/v1:GetHints"; +const char kOptimizationGuideServiceGetModelsDefaultURL[] = + "https://optimizationguide-pa.googleapis.com/v1:GetModels"; + const char kLoadedHintLocalHistogramString[] = "OptimizationGuide.LoadedHint.Result"; +const char kOptimizationGuideHintStore[] = "previews_hint_cache_store"; + +const char kOptimizationGuidePredictionModelAndFeaturesStore[] = + "optimization_guide_model_and_features_store"; + } // namespace optimization_guide diff --git a/chromium/components/optimization_guide/optimization_guide_constants.h b/chromium/components/optimization_guide/optimization_guide_constants.h index cd81f88a87d..e80075ac773 100644 --- a/chromium/components/optimization_guide/optimization_guide_constants.h +++ b/chromium/components/optimization_guide/optimization_guide_constants.h @@ -15,12 +15,23 @@ extern const base::FilePath::CharType kUnindexedHintsFileName[]; extern const char kRulesetFormatVersionString[]; // The remote Optimization Guide Service production server to fetch hints from. -extern const char kOptimizationGuideServiceDefaultURL[]; +extern const char kOptimizationGuideServiceGetHintsDefaultURL[]; + +// The remote Optimization Guide Service production server to fetch models and +// hosts features from. +extern const char kOptimizationGuideServiceGetModelsDefaultURL[]; // The local histogram used to record that the component hints are stored in // the cache and are ready for use. extern const char kLoadedHintLocalHistogramString[]; +// The folder where the hint data will be stored on disk. +extern const char kOptimizationGuideHintStore[]; + +// The folder where the prediction model and host model features data will be +// stored on disk. +extern const char kOptimizationGuidePredictionModelAndFeaturesStore[]; + } // namespace optimization_guide #endif // COMPONENTS_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_CONSTANTS_H_ diff --git a/chromium/components/optimization_guide/optimization_guide_decider.h b/chromium/components/optimization_guide/optimization_guide_decider.h index a38abba6530..051a663c7eb 100644 --- a/chromium/components/optimization_guide/optimization_guide_decider.h +++ b/chromium/components/optimization_guide/optimization_guide_decider.h @@ -46,12 +46,15 @@ class OptimizationGuideDecider { const std::vector& optimization_types, const std::vector& optimization_targets) = 0; - // Returns whether the current conditions match |optimization_target| and - // |optimization_type| can be applied for the URL associated with - // |navigation_handle|. + // Returns whether the current conditions match |optimization_target|. + virtual OptimizationGuideDecision ShouldTargetNavigation( + content::NavigationHandle* navigation_handle, + proto::OptimizationTarget optimization_target) = 0; + + // Returns whether |optimization_type| can be applied for the URL associated + // with |navigation_handle|. virtual OptimizationGuideDecision CanApplyOptimization( content::NavigationHandle* navigation_handle, - proto::OptimizationTarget optimization_target, proto::OptimizationType optimization_type, OptimizationMetadata* optimization_metadata) = 0; diff --git a/chromium/components/optimization_guide/optimization_guide_enums.h b/chromium/components/optimization_guide/optimization_guide_enums.h index 9423c6c0b54..bc70ea43238 100644 --- a/chromium/components/optimization_guide/optimization_guide_enums.h +++ b/chromium/components/optimization_guide/optimization_guide_enums.h @@ -38,9 +38,15 @@ enum class OptimizationTypeDecision { kHadHintButNotLoadedInTime, // No hints were available in the cache that matched the page load. kNoHintAvailable, + // The OptimizationGuideDecider was not initialized yet. + kDeciderNotInitialized, + // A fetch to get the hint for the page load from the remote Optimization + // Guide Service was started, but was not available in time to make a + // decision. + kHintFetchStartedButNotAvailableInTime, // Add new values above this line. - kMaxValue = kNoHintAvailable, + kMaxValue = kHintFetchStartedButNotAvailableInTime, }; // The types of decisions that can be made for an optimization target. @@ -55,8 +61,36 @@ enum class OptimizationTargetDecision { // The model needed to make the target decision was not available on the // client. kModelNotAvailableOnClient, + // The page load is part of a model prediction holdback where all decisions + // will return |OptimizationGuideDecision::kFalse| in an attempt to not taint + // the data for understanding the production recall of the model. + kModelPredictionHoldback, + // The OptimizationGuideDecider was not initialized yet. + kDeciderNotInitialized, + + // Add new values above this line. + kMaxValue = kDeciderNotInitialized, +}; + +// The statuses for why the main frame of a navigation was covered by a hint or +// fetch from the remote Optimization Guide Service. +// +// Keep in sync with OptimizationGuideNavigationHostCoveredStatus in enums.xml. +enum class NavigationHostCoveredStatus { + kUnknown, + // The main frame host of the navigation was covered by a hint or was + // attempted to be fetched from the remote Optimization Guide Service in the + // last 7 days. + kCovered, + // A fetch for information from the remote Optimization Guide Service about + // the main frame host of the navigation was not attempted. + kFetchNotAttempted, + // A fetch for information from the remote Optimization Guide Service about + // the main frame host of the navigation was attempted but not successful. + kFetchNotSuccessful, + // Add new values above this line. - kMaxValue = kModelNotAvailableOnClient, + kMaxValue = kFetchNotSuccessful, }; } // namespace optimization_guide diff --git a/chromium/components/optimization_guide/optimization_guide_features.cc b/chromium/components/optimization_guide/optimization_guide_features.cc index 0f1374387d6..adc0434dc3f 100644 --- a/chromium/components/optimization_guide/optimization_guide_features.cc +++ b/chromium/components/optimization_guide/optimization_guide_features.cc @@ -41,13 +41,19 @@ const base::Feature kOptimizationHints { const base::Feature kOptimizationHintsExperiments{ "OptimizationHintsExperiments", base::FEATURE_DISABLED_BY_DEFAULT}; -// Enables fetching optimization hints from a remote Optimization Guide Service. -const base::Feature kOptimizationHintsFetching{ - "OptimizationHintsFetching", base::FEATURE_DISABLED_BY_DEFAULT}; +// Enables fetching from a remote Optimization Guide Service. +const base::Feature kRemoteOptimizationGuideFetching { + "OptimizationHintsFetching", +#if defined(OS_ANDROID) + base::FEATURE_ENABLED_BY_DEFAULT +#else // !defined(OS_ANDROID) + base::FEATURE_DISABLED_BY_DEFAULT +#endif // defined(OS_ANDROID) +}; -// Enables the initialization of the Optimization Guide Keyed Service. -const base::Feature kOptimizationGuideKeyedService{ - "OptimizationGuideKeyedService", base::FEATURE_DISABLED_BY_DEFAULT}; +const base::Feature kRemoteOptimizationGuideFetchingAnonymousDataConsent{ + "OptimizationHintsFetchingAnonymousDataConsent", + base::FEATURE_DISABLED_BY_DEFAULT}; // Enables the prediction of optimization targets. const base::Feature kOptimizationTargetPrediction{ @@ -59,21 +65,21 @@ size_t MaxHintsFetcherTopHostBlacklistSize() { // hosts on the blacklist are meant to cover the case that the engagement // scores on some of the top N host engagement scores decay and they fall out // of the top N. - return GetFieldTrialParamByFeatureAsInt(kOptimizationHintsFetching, + return GetFieldTrialParamByFeatureAsInt(kRemoteOptimizationGuideFetching, "top_host_blacklist_size_multiplier", - 2) * + 3) * MaxHostsForOptimizationGuideServiceHintsFetch(); } size_t MaxHostsForOptimizationGuideServiceHintsFetch() { return GetFieldTrialParamByFeatureAsInt( - kOptimizationHintsFetching, + kRemoteOptimizationGuideFetching, "max_hosts_for_optimization_guide_service_hints_fetch", 30); } size_t MaxHostsForRecordingSuccessfullyCovered() { return GetFieldTrialParamByFeatureAsInt( - kOptimizationHintsFetching, + kRemoteOptimizationGuideFetching, "max_hosts_for_recording_successfully_covered", 200); } @@ -82,19 +88,19 @@ double MinTopHostEngagementScoreThreshold() { // points for a navigation from the omnibox and 1.5 points for the first // navigation of the day. return GetFieldTrialParamByFeatureAsDouble( - kOptimizationHintsFetching, "min_top_host_engagement_score_threshold", - 3.0); + kRemoteOptimizationGuideFetching, + "min_top_host_engagement_score_threshold", 2.0); } base::TimeDelta StoredFetchedHintsFreshnessDuration() { return base::TimeDelta::FromDays(GetFieldTrialParamByFeatureAsInt( - kOptimizationHintsFetching, + kRemoteOptimizationGuideFetching, "max_store_duration_for_featured_hints_in_days", 7)); } base::TimeDelta DurationApplyLowEngagementScoreThreshold() { return base::TimeDelta::FromDays(GetFieldTrialParamByFeatureAsInt( - kOptimizationHintsFetching, + kRemoteOptimizationGuideFetching, "duration_apply_low_engagement_score_threshold_in_days", 30)); } @@ -109,38 +115,54 @@ std::string GetOptimizationGuideServiceAPIKey() { return google_apis::GetAPIKey(); } -GURL GetOptimizationGuideServiceURL() { +GURL GetOptimizationGuideServiceGetHintsURL() { // Command line override takes priority. base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); - if (command_line->HasSwitch(switches::kOptimizationGuideServiceURL)) { + if (command_line->HasSwitch(switches::kOptimizationGuideServiceGetHintsURL)) { // Assume the command line switch is correct and return it. return GURL(command_line->GetSwitchValueASCII( - switches::kOptimizationGuideServiceURL)); + switches::kOptimizationGuideServiceGetHintsURL)); } std::string url = base::GetFieldTrialParamValueByFeature( - kOptimizationHintsFetching, "optimization_guide_service_url"); + kRemoteOptimizationGuideFetching, "optimization_guide_service_url"); if (url.empty() || !GURL(url).SchemeIs(url::kHttpsScheme)) { if (!url.empty()) LOG(WARNING) << "Empty or invalid optimization_guide_service_url provided: " << url; - return GURL(kOptimizationGuideServiceDefaultURL); + return GURL(kOptimizationGuideServiceGetHintsDefaultURL); } return GURL(url); } +GURL GetOptimizationGuideServiceGetModelsURL() { + // Command line override takes priority. + base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); + if (command_line->HasSwitch( + switches::kOptimizationGuideServiceGetModelsURL)) { + // Assume the command line switch is correct and return it. + return GURL(command_line->GetSwitchValueASCII( + switches::kOptimizationGuideServiceGetModelsURL)); + } + + GURL get_models_url(kOptimizationGuideServiceGetModelsDefaultURL); + CHECK(get_models_url.SchemeIs(url::kHttpsScheme)); + return get_models_url; +} + bool IsOptimizationHintsEnabled() { return base::FeatureList::IsEnabled(kOptimizationHints); } -bool IsHintsFetchingEnabled() { - return base::FeatureList::IsEnabled(kOptimizationHintsFetching); +bool IsRemoteFetchingEnabled() { + return base::FeatureList::IsEnabled(kRemoteOptimizationGuideFetching); } -bool IsOptimizationGuideKeyedServiceEnabled() { - return base::FeatureList::IsEnabled(kOptimizationGuideKeyedService); +bool IsRemoteFetchingForAnonymousDataConsentEnabled() { + return base::FeatureList::IsEnabled( + kRemoteOptimizationGuideFetchingAnonymousDataConsent); } int MaxServerBloomFilterByteSize() { @@ -151,7 +173,7 @@ int MaxServerBloomFilterByteSize() { base::Optional GetMaxEffectiveConnectionTypeForNavigationHintsFetch() { std::string param_value = base::GetFieldTrialParamValueByFeature( - kOptimizationHintsFetching, + kRemoteOptimizationGuideFetching, "max_effective_connection_type_for_navigation_hints_fetch"); // Use a default value. @@ -165,9 +187,45 @@ base::TimeDelta GetHintsFetchRefreshDuration() { return base::TimeDelta::FromHours(72); } +base::TimeDelta StoredHostModelFeaturesFreshnessDuration() { + return base::TimeDelta::FromDays(GetFieldTrialParamByFeatureAsInt( + kOptimizationTargetPrediction, + "max_store_duration_for_host_model_features_in_days", 7)); +} + +size_t MaxHostsForOptimizationGuideServiceModelsFetch() { + return GetFieldTrialParamByFeatureAsInt( + kOptimizationTargetPrediction, + "max_hosts_for_optimization_guide_service_models_fetch", 30); +} + +size_t MaxHostModelFeaturesCacheSize() { + return GetFieldTrialParamByFeatureAsInt( + kOptimizationTargetPrediction, "max_host_model_features_cache_size", 100); +} + bool IsOptimizationTargetPredictionEnabled() { return base::FeatureList::IsEnabled(kOptimizationTargetPrediction); } +bool ShouldOverrideOptimizationTargetDecisionForMetricsPurposes( + proto::OptimizationTarget optimization_target) { + if (optimization_target != proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD) + return false; + + return base::GetFieldTrialParamByFeatureAsBool( + kOptimizationTargetPrediction, "painful_page_load_metrics_only", false); +} + +int PredictionModelFetchRandomMinDelaySecs() { + return GetFieldTrialParamByFeatureAsInt(kOptimizationTargetPrediction, + "fetch_random_min_delay_secs", 30); +} + +int PredictionModelFetchRandomMaxDelaySecs() { + return GetFieldTrialParamByFeatureAsInt(kOptimizationTargetPrediction, + "fetch_random_max_delay_secs", 180); +} + } // namespace features } // namespace optimization_guide diff --git a/chromium/components/optimization_guide/optimization_guide_features.h b/chromium/components/optimization_guide/optimization_guide_features.h index b367fd03e87..1ce8a9edc31 100644 --- a/chromium/components/optimization_guide/optimization_guide_features.h +++ b/chromium/components/optimization_guide/optimization_guide_features.h @@ -11,6 +11,7 @@ #include "base/feature_list.h" #include "base/optional.h" #include "base/time/time.h" +#include "components/optimization_guide/proto/models.pb.h" #include "net/nqe/effective_connection_type.h" #include "url/gurl.h" @@ -20,8 +21,8 @@ namespace features { extern const base::Feature kOptimizationHints; extern const base::Feature kOptimizationHintsExperiments; constexpr char kOptimizationHintsExperimentNameParam[] = "experiment_name"; -extern const base::Feature kOptimizationHintsFetching; -extern const base::Feature kOptimizationGuideKeyedService; +extern const base::Feature kRemoteOptimizationGuideFetching; +extern const base::Feature kRemoteOptimizationGuideFetchingAnonymousDataConsent; extern const base::Feature kOptimizationTargetPrediction; // The maximum number of hosts that can be stored in the @@ -44,7 +45,7 @@ size_t MaxHostsForRecordingSuccessfullyCovered(); double MinTopHostEngagementScoreThreshold(); // The amount of time a fetched hint will be considered fresh enough -// to be used and remain in the HintCacheStore. +// to be used and remain in the OptimizationGuideStore. base::TimeDelta StoredFetchedHintsFreshnessDuration(); // The duration of time after the blacklist initialization for which the low @@ -56,19 +57,23 @@ base::TimeDelta DurationApplyLowEngagementScoreThreshold(); // The API key for the One Platform Optimization Guide Service. std::string GetOptimizationGuideServiceAPIKey(); -// The host for the One Platform Optimization Guide Service. -GURL GetOptimizationGuideServiceURL(); +// The host for the One Platform Optimization Guide Service for hints. +GURL GetOptimizationGuideServiceGetHintsURL(); + +// The host for the One Platform Optimization Guide Service for Models and Host +// Model Features. +GURL GetOptimizationGuideServiceGetModelsURL(); // Whether server optimization hints are enabled. bool IsOptimizationHintsEnabled(); -// Returns true if the feature to fetch hints from the remote Optimization Guide +// Returns true if the feature to fetch from the remote Optimization Guide // Service is enabled. -bool IsHintsFetchingEnabled(); +bool IsRemoteFetchingEnabled(); -// Returns true if the initialization of the Optimization Guide Keyed Service is -// enabled. -bool IsOptimizationGuideKeyedServiceEnabled(); +// Returns true if the feature to fetch data for users that have consented to +// anonymous data collection is enabled but are not Data Saver users. +bool IsRemoteFetchingForAnonymousDataConsentEnabled(); // The maximum data byte size for a server-provided bloom filter. This is // a client-side safety limit for RAM use in case server sends too large of @@ -90,6 +95,32 @@ base::TimeDelta GetHintsFetchRefreshDuration(); // Returns true if optimization target prediction is enabled. bool IsOptimizationTargetPredictionEnabled(); +// The amount of time host model features will be considered fresh enough +// to be used and remain in the OptimizationGuideStore. +base::TimeDelta StoredHostModelFeaturesFreshnessDuration(); + +// The maximum number of hosts allowed to be requested by the client to the +// remote Optimzation Guide Service for use by prediction models. +size_t MaxHostsForOptimizationGuideServiceModelsFetch(); + +// The maximum number of hosts allowed to be maintained in a least-recently-used +// cache by the prediction manager. +size_t MaxHostModelFeaturesCacheSize(); + +// Returns true if the optimization target decision for |optimization_target| +// should not be propagated to the caller in an effort to fully understand the +// statistics for the served model and not taint the resulting data. +bool ShouldOverrideOptimizationTargetDecisionForMetricsPurposes( + proto::OptimizationTarget optimization_target); + +// Returns the minimum number of seconds to randomly delay before starting to +// fetch for prediction models and host model features. +int PredictionModelFetchRandomMinDelaySecs(); + +// Returns the maximum number of seconds to randomly delay before starting to +// fetch for prediction models and host model features. +int PredictionModelFetchRandomMaxDelaySecs(); + } // namespace features } // namespace optimization_guide diff --git a/chromium/components/optimization_guide/optimization_guide_features_unittest.cc b/chromium/components/optimization_guide/optimization_guide_features_unittest.cc index a11774e80f0..77c34ffe3c8 100644 --- a/chromium/components/optimization_guide/optimization_guide_features_unittest.cc +++ b/chromium/components/optimization_guide/optimization_guide_features_unittest.cc @@ -20,29 +20,29 @@ namespace optimization_guide { namespace { TEST(OptimizationGuideFeaturesTest, - TestGetOptimizationGuideServiceURLHTTPSOnly) { + TestGetOptimizationGuideServiceGetHintsURLHTTPSOnly) { base::test::ScopedFeatureList scoped_feature_list; scoped_feature_list.InitAndEnableFeatureWithParameters( - features::kOptimizationHintsFetching, + features::kRemoteOptimizationGuideFetching, {{"optimization_guide_service_url", "http://NotAnHTTPSServer.com"}}); - EXPECT_EQ(features::GetOptimizationGuideServiceURL().spec(), - kOptimizationGuideServiceDefaultURL); - EXPECT_TRUE( - features::GetOptimizationGuideServiceURL().SchemeIs(url::kHttpsScheme)); + EXPECT_EQ(features::GetOptimizationGuideServiceGetHintsURL().spec(), + kOptimizationGuideServiceGetHintsDefaultURL); + EXPECT_TRUE(features::GetOptimizationGuideServiceGetHintsURL().SchemeIs( + url::kHttpsScheme)); } TEST(OptimizationGuideFeaturesTest, - TestGetOptimizationGuideServiceURLViaFinch) { + TestGetOptimizationGuideServiceGetHintsURLViaFinch) { base::test::ScopedFeatureList scoped_feature_list; std::string optimization_guide_service_url = "https://finchserver.com/"; scoped_feature_list.InitAndEnableFeatureWithParameters( - features::kOptimizationHintsFetching, + features::kRemoteOptimizationGuideFetching, {{"optimization_guide_service_url", optimization_guide_service_url}}); - EXPECT_EQ(features::GetOptimizationGuideServiceURL().spec(), + EXPECT_EQ(features::GetOptimizationGuideServiceGetHintsURL().spec(), optimization_guide_service_url); } diff --git a/chromium/components/optimization_guide/optimization_guide_prefs.cc b/chromium/components/optimization_guide/optimization_guide_prefs.cc index 0650a0df65b..f6be0443cb2 100644 --- a/chromium/components/optimization_guide/optimization_guide_prefs.cc +++ b/chromium/components/optimization_guide/optimization_guide_prefs.cc @@ -16,27 +16,36 @@ namespace prefs { const char kHintsFetcherLastFetchAttempt[] = "optimization_guide.hintsfetcher.last_fetch_attempt"; +// A pref that stores the last time a prediction model and host model features +// fetch was attempted. This limits the frequency of fetching for updates and +// prevents a crash loop that continually fetches prediction models and host +// model features on startup. +const char kModelAndFeaturesLastFetchAttempt[] = + "optimization_guide.predictionmodelfetcher.last_fetch_attempt"; + // A dictionary pref that stores the set of hosts that cannot have hints fetched -// for until visited again after DataSaver was enabled. If The hash of the host -// is in the dictionary, then it is on the blacklist and should not be used, the -// |value| in the key-value pair is not used. -const char kHintsFetcherDataSaverTopHostBlacklist[] = +// for until visited again after fetching from the remote Optimization Guide +// Service was first allowed. If The hash of the host is in the dictionary, then +// it is on the blacklist and should not be used, the |value| in the key-value +// pair is not used. +const char kHintsFetcherTopHostBlacklist[] = "optimization_guide.hintsfetcher.top_host_blacklist"; // An integer pref that stores the state of the blacklist for the top host -// provider for blacklisting hosts after DataSaver is enabled. The state maps to -// the HintsFetcherTopHostBlacklistState enum. -const char kHintsFetcherDataSaverTopHostBlacklistState[] = +// provider for blacklisting hosts after fetching from the remote Optimization +// Guide Service was first allowed. The state maps to the +// HintsFetcherTopHostBlacklistState enum. +const char kHintsFetcherTopHostBlacklistState[] = "optimization_guide.hintsfetcher.top_host_blacklist_state"; -// Time when the blacklist was last initialized. Recorded as seconds since -// epoch. -const char kTimeBlacklistLastInitialized[] = +// Time when the top host blacklist was last initialized. Recorded as seconds +// since epoch. +const char kTimeHintsFetcherTopHostBlacklistLastInitialized[] = "optimization_guide.hintsfetcher.time_blacklist_last_initialized"; // If a host has site engagement score less than the value stored in this pref, // then hints fetcher may not fetch hints for that host. -const char kHintsFetcherDataSaverTopHostBlacklistMinimumEngagementScore[] = +const char kHintsFetcherTopHostBlacklistMinimumEngagementScore[] = "optimization_guide.hintsfetcher.top_host_blacklist_min_engagement_score"; // A dictionary pref that stores hosts that have had hints successfully fetched @@ -46,6 +55,14 @@ const char kHintsFetcherDataSaverTopHostBlacklistMinimumEngagementScore[] = const char kHintsFetcherHostsSuccessfullyFetched[] = "optimization_guide.hintsfetcher.hosts_successfully_fetched"; +// A double pref that stores the running mean FCP. +const char kSessionStatisticFCPMean[] = + "optimization_guide.session_statistic.fcp_mean"; + +// A double pref that stores the running FCP standard deviation. +const char kSessionStatisticFCPStdDev[] = + "optimization_guide.session_statistic.fcp_std_dev"; + // A string pref that stores the version of the Optimization Hints component // that is currently being processed. This pref is cleared once processing // completes. It is used for detecting a potential crash loop on processing a @@ -58,24 +75,32 @@ void RegisterProfilePrefs(PrefRegistrySimple* registry) { kHintsFetcherLastFetchAttempt, base::Time().ToDeltaSinceWindowsEpoch().InMicroseconds(), PrefRegistry::LOSSY_PREF); - registry->RegisterDictionaryPref(kHintsFetcherDataSaverTopHostBlacklist, + registry->RegisterInt64Pref( + kModelAndFeaturesLastFetchAttempt, + base::Time().ToDeltaSinceWindowsEpoch().InMicroseconds(), + PrefRegistry::LOSSY_PREF); + registry->RegisterDictionaryPref(kHintsFetcherTopHostBlacklist, PrefRegistry::LOSSY_PREF); registry->RegisterDictionaryPref(kHintsFetcherHostsSuccessfullyFetched, PrefRegistry::LOSSY_PREF); registry->RegisterIntegerPref( - kHintsFetcherDataSaverTopHostBlacklistState, + kHintsFetcherTopHostBlacklistState, static_cast(HintsFetcherTopHostBlacklistState::kNotInitialized), PrefRegistry::LOSSY_PREF); - registry->RegisterDoublePref(kTimeBlacklistLastInitialized, 0, - PrefRegistry::LOSSY_PREF); + registry->RegisterDoublePref(kTimeHintsFetcherTopHostBlacklistLastInitialized, + 0, PrefRegistry::LOSSY_PREF); + registry->RegisterDoublePref(kSessionStatisticFCPMean, 0, + PrefRegistry::LOSSY_PREF); + registry->RegisterDoublePref(kSessionStatisticFCPStdDev, 0, + PrefRegistry::LOSSY_PREF); // Use a default value of MinTopHostEngagementScoreThreshold() for the // threshold. This ensures that the users for which this pref can't be // computed (possibly because they had the blacklist initialized before this // pref was added to the code) use the default value for the site engagement // threshold. registry->RegisterDoublePref( - kHintsFetcherDataSaverTopHostBlacklistMinimumEngagementScore, + kHintsFetcherTopHostBlacklistMinimumEngagementScore, optimization_guide::features::MinTopHostEngagementScoreThreshold(), PrefRegistry::LOSSY_PREF); diff --git a/chromium/components/optimization_guide/optimization_guide_prefs.h b/chromium/components/optimization_guide/optimization_guide_prefs.h index 2a81f6fdf45..1c0c7048119 100644 --- a/chromium/components/optimization_guide/optimization_guide_prefs.h +++ b/chromium/components/optimization_guide/optimization_guide_prefs.h @@ -13,13 +13,15 @@ namespace optimization_guide { namespace prefs { extern const char kHintsFetcherLastFetchAttempt[]; -extern const char kHintsFetcherDataSaverTopHostBlacklist[]; -extern const char kHintsFetcherDataSaverTopHostBlacklistState[]; -extern const char kTimeBlacklistLastInitialized[]; -extern const char - kHintsFetcherDataSaverTopHostBlacklistMinimumEngagementScore[]; +extern const char kModelAndFeaturesLastFetchAttempt[]; +extern const char kHintsFetcherTopHostBlacklist[]; +extern const char kHintsFetcherTopHostBlacklistState[]; +extern const char kTimeHintsFetcherTopHostBlacklistLastInitialized[]; +extern const char kHintsFetcherTopHostBlacklistMinimumEngagementScore[]; extern const char kHintsFetcherHostsSuccessfullyFetched[]; extern const char kPendingHintsProcessingVersion[]; +extern const char kSessionStatisticFCPMean[]; +extern const char kSessionStatisticFCPStdDev[]; // State of |HintsFetcherTopHostsBlacklist|. The blacklist begins in // kNotInitialized and transitions to kInitialized after diff --git a/chromium/components/optimization_guide/optimization_guide_store.cc b/chromium/components/optimization_guide/optimization_guide_store.cc new file mode 100644 index 00000000000..7bfc4758df0 --- /dev/null +++ b/chromium/components/optimization_guide/optimization_guide_store.cc @@ -0,0 +1,1102 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/optimization_guide/optimization_guide_store.h" + +#include "base/bind.h" +#include "base/metrics/histogram_functions.h" +#include "base/metrics/histogram_macros.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" +#include "components/leveldb_proto/public/proto_database.h" +#include "components/leveldb_proto/public/proto_database_provider.h" +#include "components/leveldb_proto/public/shared_proto_database_client_list.h" +#include "components/optimization_guide/optimization_guide_prefs.h" +#include "components/optimization_guide/proto/hint_cache.pb.h" + +namespace optimization_guide { + +namespace { + +// Enforce that StoreEntryType enum is synced with the StoreEntryType proto +// (components/previews/content/proto/hint_cache.proto) +static_assert( + proto::StoreEntryType_MAX == + static_cast(OptimizationGuideStore::StoreEntryType::kMaxValue), + "mismatched StoreEntryType enums"); + +// The amount of data to build up in memory before converting to a sorted on- +// disk file. +constexpr size_t kDatabaseWriteBufferSizeBytes = 128 * 1024; + +// Delimiter that appears between the sections of a store entry key. +// Examples: +// "[StoreEntryType::kMetadata]_[MetadataType]" +// "[StoreEntryType::kComponentHint]_[component_version]_[host]" +constexpr char kKeySectionDelimiter = '_'; + +// Realistic minimum length of a host suffix. +const int kMinHostSuffix = 6; // eg., abc.tv + +// Enumerates the possible outcomes of loading metadata. Used in UMA histograms, +// so the order of enumerators should not be changed. +// +// Keep in sync with OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult +// in tools/metrics/histograms/enums.xml. +enum class OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult { + kSuccess = 0, + kLoadMetadataFailed = 1, + kSchemaMetadataMissing = 2, + kSchemaMetadataWrongVersion = 3, + kComponentMetadataMissing = 4, + kFetchedMetadataMissing = 5, + kComponentAndFetchedMetadataMissing = 6, + kMaxValue = kComponentAndFetchedMetadataMissing, +}; + +// Util class for recording the result of loading the metadata. The result is +// recorded when it goes out of scope and its destructor is called. +class ScopedLoadMetadataResultRecorder { + public: + ScopedLoadMetadataResultRecorder() + : result_(OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kSuccess) {} + ~ScopedLoadMetadataResultRecorder() { + UMA_HISTOGRAM_ENUMERATION( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", result_); + } + + void set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult result) { + result_ = result; + } + + private: + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult result_; +}; + +void RecordStatusChange(OptimizationGuideStore::Status status) { + UMA_HISTOGRAM_ENUMERATION("OptimizationGuide.HintCacheLevelDBStore.Status", + status); +} + +// Returns true if |key_prefix| is a prefix of |key|. +bool DatabasePrefixFilter(const std::string& key_prefix, + const std::string& key) { + return base::StartsWith(key, key_prefix, base::CompareCase::SENSITIVE); +} + +// Returns true if |key| is in |keys_to_remove|. +bool ExpiredKeyFilter(const base::flat_set& keys_to_remove, + const std::string& key) { + return keys_to_remove.find(key) != keys_to_remove.end(); +} + +} // namespace + +OptimizationGuideStore::OptimizationGuideStore( + leveldb_proto::ProtoDatabaseProvider* database_provider, + const base::FilePath& database_dir, + scoped_refptr store_task_runner) { + database_ = database_provider->GetDB( + leveldb_proto::ProtoDbType::HINT_CACHE_STORE, database_dir, + store_task_runner); + + RecordStatusChange(status_); +} + +OptimizationGuideStore::OptimizationGuideStore( + std::unique_ptr> database) + : database_(std::move(database)) { + RecordStatusChange(status_); +} + +OptimizationGuideStore::~OptimizationGuideStore() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); +} + +void OptimizationGuideStore::Initialize(bool purge_existing_data, + base::OnceClosure callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + UpdateStatus(Status::kInitializing); + + // Asynchronously initialize the store and run the callback once + // initialization completes. Initialization consists of the following steps: + // 1. Initialize the database. + // 2. If |purge_existing_data| is set to true, unconditionally purge + // database and skip to step 6. + // 3. Otherwise, retrieve the metadata entries (e.g. Schema and Component). + // 4. If schema is the wrong version, purge database and skip to step 6. + // 5. Otherwise, load all hint entry keys. + // 6. Run callback after purging database or retrieving hint entry keys. + + leveldb_env::Options options = leveldb_proto::CreateSimpleOptions(); + options.write_buffer_size = kDatabaseWriteBufferSizeBytes; + database_->Init(options, + base::BindOnce(&OptimizationGuideStore::OnDatabaseInitialized, + weak_ptr_factory_.GetWeakPtr(), + purge_existing_data, std::move(callback))); +} + +std::unique_ptr +OptimizationGuideStore::MaybeCreateUpdateDataForComponentHints( + const base::Version& version) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(version.IsValid()); + + if (!IsAvailable()) { + return nullptr; + } + + // Component updates are only permitted when the update version is newer than + // the store's current one. + if (component_version_.has_value() && version <= component_version_) { + return nullptr; + } + + // Create and return a StoreUpdateData object. This object has + // hints from the component moved into it and organizes them in a format + // usable by the store; the object will returned to the store in + // StoreComponentHints(). + return StoreUpdateData::CreateComponentStoreUpdateData(version); +} + +std::unique_ptr +OptimizationGuideStore::CreateUpdateDataForFetchedHints( + base::Time update_time, + base::Time expiry_time) const { + // Create and returns a StoreUpdateData object. This object has has hints + // from the GetHintsResponse moved into and organizes them in a format + // usable by the store. The object will be store with UpdateFetchedData(). + return StoreUpdateData::CreateFetchedStoreUpdateData(update_time, + expiry_time); +} + +void OptimizationGuideStore::UpdateComponentHints( + std::unique_ptr component_data, + base::OnceClosure callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(component_data); + DCHECK(!data_update_in_flight_); + DCHECK(component_data->component_version()); + + if (!IsAvailable()) { + std::move(callback).Run(); + return; + } + + // If this isn't a newer component version than the store's current one, then + // the simply return. There's nothing to update. + if (component_version_.has_value() && + component_data->component_version() <= component_version_) { + std::move(callback).Run(); + return; + } + + // Mark that there's now a component data update in-flight. While this is + // true, keys and hints will not be returned by the store. + data_update_in_flight_ = true; + + // Set the component version prior to requesting the update. This ensures that + // a second update request for the same component version won't be allowed. In + // the case where the update fails, the store will become unavailable, so it's + // safe to treat component version in the update as the new one. + SetComponentVersion(*component_data->component_version()); + + // The current keys are about to be removed, so clear out the keys available + // within the store. The keys will be populated after the component data + // update completes. + entry_keys_.reset(); + + // Purge any component hints that are missing the new component version + // prefix. + EntryKeyPrefix retain_prefix = + GetComponentHintEntryKeyPrefix(component_version_.value()); + EntryKeyPrefix filter_prefix = GetComponentHintEntryKeyPrefixWithoutVersion(); + + // Add the new component data and purge any old component hints from the db. + // After processing finishes, OnUpdateStore() is called, which loads + // the updated hint entry keys from the database. + database_->UpdateEntriesWithRemoveFilter( + component_data->TakeUpdateEntries(), + base::BindRepeating( + [](const EntryKeyPrefix& retain_prefix, + const EntryKeyPrefix& filter_prefix, const std::string& key) { + return key.compare(0, retain_prefix.length(), retain_prefix) != 0 && + key.compare(0, filter_prefix.length(), filter_prefix) == 0; + }, + retain_prefix, filter_prefix), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::UpdateFetchedHints( + std::unique_ptr fetched_hints_data, + base::OnceClosure callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(fetched_hints_data); + DCHECK(!data_update_in_flight_); + DCHECK(fetched_hints_data->update_time()); + + if (!IsAvailable()) { + std::move(callback).Run(); + return; + } + + fetched_update_time_ = *fetched_hints_data->update_time(); + + data_update_in_flight_ = true; + + entry_keys_.reset(); + + // This will remove the fetched metadata entry and insert all the entries + // currently in |leveldb_fetched_hints_data|. + database_->UpdateEntriesWithRemoveFilter( + fetched_hints_data->TakeUpdateEntries(), + base::BindRepeating(&DatabasePrefixFilter, + GetMetadataTypeEntryKey(MetadataType::kFetched)), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::PurgeExpiredFetchedHints() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!IsAvailable()) + return; + + // Load all the fetched hints to check their expiry times. + database_->LoadKeysAndEntriesWithFilter( + base::BindRepeating(&DatabasePrefixFilter, + GetFetchedHintEntryKeyPrefix()), + base::BindOnce(&OptimizationGuideStore::OnLoadEntriesToPurgeExpired, + weak_ptr_factory_.GetWeakPtr())); +} + +void OptimizationGuideStore::PurgeExpiredHostModelFeatures() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!IsAvailable()) + return; + + // Load all the host model features to check their expiry times. + database_->LoadKeysAndEntriesWithFilter( + base::BindRepeating(&DatabasePrefixFilter, + GetHostModelFeaturesEntryKeyPrefix()), + base::BindOnce(&OptimizationGuideStore::OnLoadEntriesToPurgeExpired, + weak_ptr_factory_.GetWeakPtr())); +} + +void OptimizationGuideStore::OnLoadEntriesToPurgeExpired( + bool success, + std::unique_ptr entries) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!success) + return; + + EntryKeySet expired_keys_to_remove; + int64_t now_since_epoch = + base::Time::Now().ToDeltaSinceWindowsEpoch().InSeconds(); + + for (const auto& entry : *entries) { + if (entry.second.has_expiry_time_secs() && + entry.second.expiry_time_secs() <= now_since_epoch) { + expired_keys_to_remove.insert(entry.first); + } + } + + data_update_in_flight_ = true; + entry_keys_.reset(); + + auto empty_entries = std::make_unique(); + + database_->UpdateEntriesWithRemoveFilter( + std::move(empty_entries), + base::BindRepeating(&ExpiredKeyFilter, std::move(expired_keys_to_remove)), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), base::DoNothing::Once())); +} + +bool OptimizationGuideStore::FindHintEntryKey( + const std::string& host, + EntryKey* out_hint_entry_key) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // Search for kFetched hint entry keys first, fetched hints should be + // fresher and preferred. + if (FindEntryKeyForHostWithPrefix(host, out_hint_entry_key, + GetFetchedHintEntryKeyPrefix())) { + return true; + } + + // Search for kComponent hint entry keys next. + DCHECK(!component_version_.has_value() || + component_hint_entry_key_prefix_ == + GetComponentHintEntryKeyPrefix(component_version_.value())); + if (FindEntryKeyForHostWithPrefix(host, out_hint_entry_key, + component_hint_entry_key_prefix_)) + return true; + + return false; +} + +bool OptimizationGuideStore::FindEntryKeyForHostWithPrefix( + const std::string& host, + EntryKey* out_entry_key, + const EntryKeyPrefix& entry_key_prefix) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(out_entry_key); + + // Look for longest host name suffix that has a hint. No need to continue + // lookups and substring work once get to a root domain like ".com" or + // ".co.in" (MinHostSuffix length check is a heuristic for that). + std::string host_suffix(host); + while (host_suffix.length() >= kMinHostSuffix) { + // Attempt to find an entry key associated with the current host suffix. + *out_entry_key = entry_key_prefix + host_suffix; + if (entry_keys_ && + entry_keys_->find(*out_entry_key) != entry_keys_->end()) { + return true; + } + + size_t pos = host_suffix.find_first_of('.'); + if (pos == std::string::npos) { + break; + } + host_suffix = host_suffix.substr(pos + 1); + } + return false; +} + +void OptimizationGuideStore::LoadHint(const EntryKey& hint_entry_key, + HintLoadedCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!IsAvailable()) { + std::move(callback).Run(hint_entry_key, nullptr); + return; + } + + database_->GetEntry(hint_entry_key, + base::BindOnce(&OptimizationGuideStore::OnLoadHint, + weak_ptr_factory_.GetWeakPtr(), + hint_entry_key, std::move(callback))); +} + +base::Time OptimizationGuideStore::GetFetchedHintsUpdateTime() const { + // If the store is not available, the metadata entries have not been loaded + // so there are no fetched hints. + if (!IsAvailable()) + return base::Time(); + return fetched_update_time_; +} + +base::Time OptimizationGuideStore::GetHostModelFeaturesUpdateTime() const { + // If the store is not available, the metadata entries have not been loaded + // so there are no host model features. + if (!IsAvailable()) + return base::Time(); + return host_model_features_update_time_; +} + +// static +const char OptimizationGuideStore::kStoreSchemaVersion[] = "1"; + +// static +OptimizationGuideStore::EntryKeyPrefix +OptimizationGuideStore::GetMetadataEntryKeyPrefix() { + return base::NumberToString(static_cast( + OptimizationGuideStore::StoreEntryType::kMetadata)) + + kKeySectionDelimiter; +} + +// static +OptimizationGuideStore::EntryKey +OptimizationGuideStore::GetMetadataTypeEntryKey(MetadataType metadata_type) { + return GetMetadataEntryKeyPrefix() + + base::NumberToString(static_cast(metadata_type)); +} + +// static +OptimizationGuideStore::EntryKeyPrefix +OptimizationGuideStore::GetComponentHintEntryKeyPrefixWithoutVersion() { + return base::NumberToString(static_cast( + OptimizationGuideStore::StoreEntryType::kComponentHint)) + + kKeySectionDelimiter; +} + +// static +OptimizationGuideStore::EntryKeyPrefix +OptimizationGuideStore::GetComponentHintEntryKeyPrefix( + const base::Version& component_version) { + return GetComponentHintEntryKeyPrefixWithoutVersion() + + component_version.GetString() + kKeySectionDelimiter; +} + +// static +OptimizationGuideStore::EntryKeyPrefix +OptimizationGuideStore::GetFetchedHintEntryKeyPrefix() { + return base::NumberToString(static_cast( + OptimizationGuideStore::StoreEntryType::kFetchedHint)) + + kKeySectionDelimiter; +} + +// static +OptimizationGuideStore::EntryKeyPrefix +OptimizationGuideStore::GetPredictionModelEntryKeyPrefix() { + return base::NumberToString(static_cast( + OptimizationGuideStore::StoreEntryType::kPredictionModel)) + + kKeySectionDelimiter; +} + +// static +OptimizationGuideStore::EntryKeyPrefix +OptimizationGuideStore::GetHostModelFeaturesEntryKeyPrefix() { + return base::NumberToString(static_cast( + OptimizationGuideStore::StoreEntryType::kHostModelFeatures)) + + kKeySectionDelimiter; +} + +void OptimizationGuideStore::UpdateStatus(Status new_status) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // Status::kUninitialized can only transition to Status::kInitializing. + DCHECK(status_ != Status::kUninitialized || + new_status == Status::kInitializing); + // Status::kInitializing can only transition to Status::kAvailable or + // Status::kFailed. + DCHECK(status_ != Status::kInitializing || new_status == Status::kAvailable || + new_status == Status::kFailed); + // Status::kAvailable can only transition to kStatus::Failed. + DCHECK(status_ != Status::kAvailable || new_status == Status::kFailed); + // The status can never transition from Status::kFailed. + DCHECK(status_ != Status::kFailed || new_status == Status::kFailed); + + // If the status is not changing, simply return; the remaining logic handles + // status changes. + if (status_ == new_status) { + return; + } + + status_ = new_status; + RecordStatusChange(status_); + + if (status_ == Status::kFailed) { + database_->Destroy( + base::BindOnce(&OptimizationGuideStore::OnDatabaseDestroyed, + weak_ptr_factory_.GetWeakPtr())); + ClearComponentVersion(); + entry_keys_.reset(); + } +} + +bool OptimizationGuideStore::IsAvailable() const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + return status_ == Status::kAvailable; +} + +void OptimizationGuideStore::PurgeDatabase(base::OnceClosure callback) { + // When purging the database, update the schema version to the current one. + EntryKey schema_entry_key = GetMetadataTypeEntryKey(MetadataType::kSchema); + proto::StoreEntry schema_entry; + schema_entry.set_version(kStoreSchemaVersion); + + auto entries_to_save = std::make_unique(); + entries_to_save->emplace_back(schema_entry_key, schema_entry); + + database_->UpdateEntriesWithRemoveFilter( + std::move(entries_to_save), + base::BindRepeating( + [](const std::string& schema_entry_key, const std::string& key) { + return key.compare(0, schema_entry_key.length(), + schema_entry_key) != 0; + }, + schema_entry_key), + base::BindOnce(&OptimizationGuideStore::OnPurgeDatabase, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::SetComponentVersion( + const base::Version& component_version) { + DCHECK(component_version.IsValid()); + component_version_ = component_version; + component_hint_entry_key_prefix_ = + GetComponentHintEntryKeyPrefix(component_version_.value()); +} + +void OptimizationGuideStore::ClearComponentVersion() { + component_version_.reset(); + component_hint_entry_key_prefix_.clear(); +} + +void OptimizationGuideStore::ClearFetchedHintsFromDatabase() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!data_update_in_flight_); + + base::UmaHistogramBoolean( + "OptimizationGuide.ClearFetchedHints.StoreAvailable", IsAvailable()); + if (!IsAvailable()) + return; + + data_update_in_flight_ = true; + auto entries_to_save = std::make_unique(); + + // TODO(mcrouse): Add histogram to record the number of hints being removed. + entry_keys_.reset(); + + // Removes all |kFetchedHint| store entries. OnUpdateStore will handle + // updating status and re-filling entry_keys with the entries still in the + // store. + database_->UpdateEntriesWithRemoveFilter( + std::move(entries_to_save), // this should be empty. + base::BindRepeating(&DatabasePrefixFilter, + GetFetchedHintEntryKeyPrefix()), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), base::DoNothing::Once())); +} + +void OptimizationGuideStore::MaybeLoadEntryKeys(base::OnceClosure callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // If the database is unavailable or if there's an in-flight component data + // update, then don't load the hint keys. Simply run the callback. + if (!IsAvailable() || data_update_in_flight_) { + std::move(callback).Run(); + return; + } + + // Create a new KeySet object. This is populated by the store's keys as the + // filter is run with each key on the DB's background thread. The filter + // itself always returns false, ensuring that no entries are ever actually + // loaded by the DB. Ownership of the KeySet is passed into the + // LoadKeysAndEntriesCallback callback, guaranteeing that the KeySet has a + // lifespan longer than the filter calls. + std::unique_ptr entry_keys(std::make_unique()); + EntryKeySet* raw_entry_keys_pointer = entry_keys.get(); + database_->LoadKeysAndEntriesWithFilter( + base::BindRepeating( + [](EntryKeySet* entry_keys, const std::string& filter_prefix, + const std::string& entry_key) { + if (entry_key.compare(0, filter_prefix.length(), filter_prefix) != + 0) { + entry_keys->insert(entry_key); + } + return false; + }, + raw_entry_keys_pointer, GetMetadataEntryKeyPrefix()), + base::BindOnce(&OptimizationGuideStore::OnLoadEntryKeys, + weak_ptr_factory_.GetWeakPtr(), std::move(entry_keys), + std::move(callback))); +} + +size_t OptimizationGuideStore::GetEntryKeyCount() const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + return entry_keys_ ? entry_keys_->size() : 0; +} + +void OptimizationGuideStore::OnDatabaseInitialized( + bool purge_existing_data, + base::OnceClosure callback, + leveldb_proto::Enums::InitStatus status) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (status != leveldb_proto::Enums::InitStatus::kOK) { + UpdateStatus(Status::kFailed); + std::move(callback).Run(); + return; + } + + // If initialization is set to purge all existing data, then simply trigger + // the purge and return. There's no need to load metadata entries that'll + // immediately be purged. + if (purge_existing_data) { + PurgeDatabase(std::move(callback)); + return; + } + + // Load all entries from the DB with the metadata key prefix. + database_->LoadKeysAndEntriesWithFilter( + leveldb_proto::KeyFilter(), leveldb::ReadOptions(), + GetMetadataEntryKeyPrefix(), + base::BindOnce(&OptimizationGuideStore::OnLoadMetadata, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::OnDatabaseDestroyed(bool /*success*/) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); +} + +void OptimizationGuideStore::OnLoadMetadata( + base::OnceClosure callback, + bool success, + std::unique_ptr metadata_entries) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(metadata_entries); + + // Create a scoped load metadata result recorder. It records the result when + // its destructor is called. + ScopedLoadMetadataResultRecorder result_recorder; + + if (!success) { + result_recorder.set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kLoadMetadataFailed); + + UpdateStatus(Status::kFailed); + std::move(callback).Run(); + return; + } + + // If the schema version in the DB is not the current version, then purge + // the database. + auto schema_entry = + metadata_entries->find(GetMetadataTypeEntryKey(MetadataType::kSchema)); + if (schema_entry == metadata_entries->end() || + !schema_entry->second.has_version() || + schema_entry->second.version() != kStoreSchemaVersion) { + if (schema_entry == metadata_entries->end()) { + result_recorder.set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kSchemaMetadataMissing); + } else { + result_recorder.set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kSchemaMetadataWrongVersion); + } + + PurgeDatabase(std::move(callback)); + return; + } + + // If the component metadata entry exists, then use it to set the component + // version. + bool component_metadata_missing = false; + auto component_entry = + metadata_entries->find(GetMetadataTypeEntryKey(MetadataType::kComponent)); + if (component_entry != metadata_entries->end()) { + DCHECK(component_entry->second.has_version()); + SetComponentVersion(base::Version(component_entry->second.version())); + } else { + result_recorder.set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kComponentMetadataMissing); + component_metadata_missing = true; + } + + auto fetched_entry = + metadata_entries->find(GetMetadataTypeEntryKey(MetadataType::kFetched)); + if (fetched_entry != metadata_entries->end()) { + DCHECK(fetched_entry->second.has_update_time_secs()); + fetched_update_time_ = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromSeconds(fetched_entry->second.update_time_secs())); + } else { + if (component_metadata_missing) { + result_recorder.set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kComponentAndFetchedMetadataMissing); + } else { + result_recorder.set_result( + OptimizationGuideHintCacheLevelDBStoreLoadMetadataResult:: + kFetchedMetadataMissing); + } + fetched_update_time_ = base::Time(); + } + + auto host_model_features_entry = metadata_entries->find( + GetMetadataTypeEntryKey(MetadataType::kHostModelFeatures)); + bool host_model_features_metadata_loaded = false; + host_model_features_update_time_ = base::Time(); + if (host_model_features_entry != metadata_entries->end()) { + DCHECK(host_model_features_entry->second.has_update_time_secs()); + host_model_features_update_time_ = + base::Time::FromDeltaSinceWindowsEpoch(base::TimeDelta::FromSeconds( + host_model_features_entry->second.update_time_secs())); + host_model_features_metadata_loaded = true; + } + // TODO(crbug/1001194): Metrics should be separated so that stores maintaining + // different information types only record metrics for the types of entries + // they store. + UMA_HISTOGRAM_BOOLEAN( + "OptimizationGuide.PredictionModelStore." + "HostModelFeaturesLoadMetadataResult", + host_model_features_metadata_loaded); + + UpdateStatus(Status::kAvailable); + MaybeLoadEntryKeys(std::move(callback)); +} + +void OptimizationGuideStore::OnPurgeDatabase(base::OnceClosure callback, + bool success) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + // The database can only be purged during initialization. + DCHECK_EQ(status_, Status::kInitializing); + + UpdateStatus(success ? Status::kAvailable : Status::kFailed); + std::move(callback).Run(); +} + +void OptimizationGuideStore::OnUpdateStore(base::OnceClosure callback, + bool success) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(data_update_in_flight_); + + data_update_in_flight_ = false; + if (!success) { + UpdateStatus(Status::kFailed); + std::move(callback).Run(); + return; + } + MaybeLoadEntryKeys(std::move(callback)); +} + +void OptimizationGuideStore::OnLoadEntryKeys( + std::unique_ptr hint_entry_keys, + base::OnceClosure callback, + bool success, + std::unique_ptr /*unused*/) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!entry_keys_); + + if (!success) { + UpdateStatus(Status::kFailed); + std::move(callback).Run(); + return; + } + + // If the store was set to unavailable after the request was started, or if + // there's an in-flight component data update, which means the keys are + // about to be invalidated, then the loaded keys should not be considered + // valid. Reset the keys so that they are cleared. + if (!IsAvailable() || data_update_in_flight_) { + hint_entry_keys.reset(); + } + + entry_keys_ = std::move(hint_entry_keys); + std::move(callback).Run(); +} + +void OptimizationGuideStore::OnLoadHint( + const std::string& entry_key, + HintLoadedCallback callback, + bool success, + std::unique_ptr entry) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // If either the request failed, the store was set to unavailable after the + // request was started, or there's an in-flight component data update, which + // means the entry is about to be invalidated, then the loaded hint should + // not be considered valid. Reset the entry so that no hint is returned to + // the requester. + if (!success || !IsAvailable() || data_update_in_flight_) { + entry.reset(); + } + + if (!entry || !entry->has_hint()) { + std::unique_ptr loaded_hint(nullptr); + std::move(callback).Run(entry_key, std::move(loaded_hint)); + return; + } + + if (entry->has_expiry_time_secs() && + entry->expiry_time_secs() <= + base::Time::Now().ToDeltaSinceWindowsEpoch().InSeconds()) { + // An expired hint should be loaded rarely if the user is regularly fetching + // and storing fresh hints. Expired fetched hints are removed each time + // fresh hints are fetched and placed into the store. In the future, the + // expired hints could be asynchronously removed if necessary. + // An empty hint is returned instead of the expired one. + UMA_HISTOGRAM_BOOLEAN( + "OptimizationGuide.HintCacheStore.OnLoadHint.FetchedHintExpired", true); + std::unique_ptr loaded_hint(nullptr); + std::move(callback).Run(entry_key, std::move(loaded_hint)); + return; + } + + // Release the Hint into a Hint unique_ptr. This eliminates the need for any + // copies of the entry's hint. + std::unique_ptr loaded_hint(entry->release_hint()); + + StoreEntryType store_entry_type = + static_cast(entry->entry_type()); + UMA_HISTOGRAM_ENUMERATION("OptimizationGuide.HintCache.HintType.Loaded", + store_entry_type); + + if (store_entry_type == StoreEntryType::kFetchedHint) { + UMA_HISTOGRAM_CUSTOM_TIMES( + "OptimizationGuide.HintCache.FetchedHint.TimeToExpiration", + base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromSeconds(entry->expiry_time_secs())) - + base::Time::Now(), + base::TimeDelta::FromHours(1), base::TimeDelta::FromDays(15), 50); + } + std::move(callback).Run(entry_key, std::move(loaded_hint)); +} + +std::unique_ptr +OptimizationGuideStore::CreateUpdateDataForPredictionModels() const { + // Create and returns a StoreUpdateData object. This object has prediction + // models from the GetModelsResponse moved into and organizes them in a format + // usable by the store. The object will be stored with + // UpdatePredictionModels(). + return StoreUpdateData::CreatePredictionModelStoreUpdateData(); +} + +void OptimizationGuideStore::UpdatePredictionModels( + std::unique_ptr prediction_models_update_data, + base::OnceClosure callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(prediction_models_update_data); + DCHECK(!data_update_in_flight_); + + if (!IsAvailable()) { + std::move(callback).Run(); + return; + } + + data_update_in_flight_ = true; + + entry_keys_.reset(); + + std::unique_ptr entry_vectors = + prediction_models_update_data->TakeUpdateEntries(); + + database_->UpdateEntries( + std::move(entry_vectors), std::make_unique(), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +bool OptimizationGuideStore::FindPredictionModelEntryKey( + proto::OptimizationTarget optimization_target, + OptimizationGuideStore::EntryKey* out_prediction_model_entry_key) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!entry_keys_) + return false; + *out_prediction_model_entry_key = + GetPredictionModelEntryKeyPrefix() + + base::NumberToString(static_cast(optimization_target)); + if (entry_keys_->find(*out_prediction_model_entry_key) != entry_keys_->end()) + return true; + return false; +} + +void OptimizationGuideStore::LoadPredictionModel( + const EntryKey& prediction_model_entry_key, + PredictionModelLoadedCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!IsAvailable()) { + std::move(callback).Run(nullptr); + return; + } + + database_->GetEntry( + prediction_model_entry_key, + base::BindOnce(&OptimizationGuideStore::OnLoadPredictionModel, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::OnLoadPredictionModel( + PredictionModelLoadedCallback callback, + bool success, + std::unique_ptr entry) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // If either the request failed, the store was set to unavailable after the + // request was started, or there's an in-flight update, which + // means the entry is about to be invalidated, then the loaded model should + // not be considered valid. Reset the entry so that nothing is returned to + // the requester. + UMA_HISTOGRAM_BOOLEAN("OptimizationGuide.PredictionModelStore.OnLoadCollided", + data_update_in_flight_); + if (!success || !IsAvailable() || data_update_in_flight_) { + entry.reset(); + } + + if (!entry || !entry->has_prediction_model()) { + std::unique_ptr loaded_prediction_model(nullptr); + std::move(callback).Run(std::move(loaded_prediction_model)); + return; + } + + std::unique_ptr loaded_prediction_model( + entry->release_prediction_model()); + std::move(callback).Run(std::move(loaded_prediction_model)); +} + +std::unique_ptr +OptimizationGuideStore::CreateUpdateDataForHostModelFeatures( + base::Time host_model_features_update_time, + base::Time expiry_time) const { + // Create and returns a StoreUpdateData object. This object has host model + // features from the GetModelsResponse moved into and organizes them in a + // format usable by the store. The object will be stored with + // UpdateHostModelFeatures(). + return StoreUpdateData::CreateHostModelFeaturesStoreUpdateData( + host_model_features_update_time, expiry_time); +} + +void OptimizationGuideStore::UpdateHostModelFeatures( + std::unique_ptr host_model_features_update_data, + base::OnceClosure callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(host_model_features_update_data->update_time()); + + if (!IsAvailable()) { + std::move(callback).Run(); + return; + } + + host_model_features_update_time_ = + *host_model_features_update_data->update_time(); + + // TODO(crbug/1001194): Add protection/lock around setting + // |data_update_in_flight_|. + data_update_in_flight_ = true; + + entry_keys_.reset(); + + // This will remove the host model features metadata entry and insert all the + // entries currently in |host_model_features_update_data|. + database_->UpdateEntriesWithRemoveFilter( + host_model_features_update_data->TakeUpdateEntries(), + base::BindRepeating( + &DatabasePrefixFilter, + GetMetadataTypeEntryKey(MetadataType::kHostModelFeatures)), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +bool OptimizationGuideStore::FindHostModelFeaturesEntryKey( + const std::string& host, + OptimizationGuideStore::EntryKey* out_host_model_features_entry_key) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!entry_keys_) + return false; + + return FindEntryKeyForHostWithPrefix(host, out_host_model_features_entry_key, + GetHostModelFeaturesEntryKeyPrefix()); +} + +void OptimizationGuideStore::LoadAllHostModelFeatures( + AllHostModelFeaturesLoadedCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!IsAvailable()) { + std::move(callback).Run({}); + return; + } + + // Load all the host model features within the store. + database_->LoadEntriesWithFilter( + base::BindRepeating(&DatabasePrefixFilter, + GetHostModelFeaturesEntryKeyPrefix()), + base::BindOnce(&OptimizationGuideStore::OnLoadAllHostModelFeatures, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::LoadHostModelFeatures( + const EntryKey& host_model_features_entry_key, + HostModelFeaturesLoadedCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!IsAvailable()) { + std::move(callback).Run({}); + return; + } + + // Load all the host model features within the store. + database_->GetEntry( + host_model_features_entry_key, + base::BindOnce(&OptimizationGuideStore::OnLoadHostModelFeatures, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void OptimizationGuideStore::OnLoadHostModelFeatures( + HostModelFeaturesLoadedCallback callback, + bool success, + std::unique_ptr entry) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // If either the request failed, the store was set to unavailable after the + // request was started, or there's an in-flight update, which means the entry + // is about to be invalidated, then the loaded host model features should not + // be considered valid. Reset the entry so that nothing is returned to the + // requester. + if (!success || !IsAvailable() || data_update_in_flight_) { + entry.reset(); + } + if (!entry || !entry->has_host_model_features()) { + std::move(callback).Run(nullptr); + return; + } + + std::unique_ptr loaded_host_model_features( + entry->release_host_model_features()); + std::move(callback).Run(std::move(loaded_host_model_features)); +} + +void OptimizationGuideStore::OnLoadAllHostModelFeatures( + AllHostModelFeaturesLoadedCallback callback, + bool success, + std::unique_ptr> entries) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // If either the request failed, the store was set to unavailable after the + // request was started, or there's an in-flight update, which means the entry + // is about to be invalidated, then the loaded host model features should not + // be considered valid. Reset the entry so that nothing is returned to the + // requester. + if (!success || !IsAvailable() || data_update_in_flight_) { + entries.reset(); + } + + if (!entries || entries->size() == 0) { + std::unique_ptr> + loaded_host_model_features(nullptr); + std::move(callback).Run(std::move(loaded_host_model_features)); + return; + } + + std::unique_ptr> + loaded_host_model_features = + std::make_unique>(); + for (auto& entry : *entries.get()) { + if (!entry.has_host_model_features()) + continue; + loaded_host_model_features->emplace_back(entry.host_model_features()); + } + std::move(callback).Run(std::move(loaded_host_model_features)); +} + +void OptimizationGuideStore::ClearHostModelFeaturesFromDatabase() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!data_update_in_flight_); + + base::UmaHistogramBoolean( + "OptimizationGuide.ClearHostModelFeatures.StoreAvailable", IsAvailable()); + if (!IsAvailable()) + return; + + data_update_in_flight_ = true; + auto entries_to_save = std::make_unique(); + + entry_keys_.reset(); + + // Removes all |kHostModelFeatures| store entries. OnUpdateStore will handle + // updating status and re-filling entry_keys with the entries still in the + // store. + database_->UpdateEntriesWithRemoveFilter( + std::move(entries_to_save), // this should be empty. + base::BindRepeating(&DatabasePrefixFilter, + GetHostModelFeaturesEntryKeyPrefix()), + base::BindOnce(&OptimizationGuideStore::OnUpdateStore, + weak_ptr_factory_.GetWeakPtr(), base::DoNothing::Once())); +} + +} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/optimization_guide_store.h b/chromium/components/optimization_guide/optimization_guide_store.h new file mode 100644 index 00000000000..7a82002d42d --- /dev/null +++ b/chromium/components/optimization_guide/optimization_guide_store.h @@ -0,0 +1,481 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_STORE_H_ +#define COMPONENTS_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_STORE_H_ + +#include +#include + +#include "base/callback.h" +#include "base/containers/flat_set.h" +#include "base/macros.h" +#include "base/memory/scoped_refptr.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/sequence_checker.h" +#include "base/version.h" +#include "components/leveldb_proto/public/proto_database.h" +#include "components/leveldb_proto/public/proto_database_provider.h" +#include "components/optimization_guide/proto/models.pb.h" +#include "components/optimization_guide/store_update_data.h" + +namespace base { +class SequencedTaskRunner; +} // namespace base + +namespace optimization_guide { +namespace proto { +class Hint; +class StoreEntry; +} // namespace proto + +// The HintCache backing store, which is responsible for storing all hints that +// are locally available. While the HintCache itself may retain some hints in a +// memory cache, all of its hints are initially loaded asynchronously by the +// store. All calls to this store must be made from the same thread. +class OptimizationGuideStore { + public: + using HintLoadedCallback = + base::OnceCallback)>; + using PredictionModelLoadedCallback = + base::OnceCallback)>; + using HostModelFeaturesLoadedCallback = + base::OnceCallback)>; + using AllHostModelFeaturesLoadedCallback = base::OnceCallback>)>; + using EntryKey = std::string; + using StoreEntryProtoDatabase = + leveldb_proto::ProtoDatabase; + + // Status of the store. The store begins in kUninitialized, transitions to + // kInitializing after Initialize() is called, and transitions to kAvailable + // if initialization successfully completes. In the case where anything fails, + // the store transitions to kFailed, at which point it is fully purged and + // becomes unusable. + // + // Keep in sync with OptimizationGuideHintCacheLevelDBStoreStatus in + // tools/metrics/histograms/enums.xml. + enum class Status { + kUninitialized = 0, + kInitializing = 1, + kAvailable = 2, + kFailed = 3, + kMaxValue = kFailed, + }; + + // Store entry types within the store appear at the start of the keys of + // entries. These values are converted into strings within the key: a key + // starting with "1_" signifies a metadata entry and one starting with "2_" + // signifies a component hint entry. Adding this to the start of the key + // allows the store to quickly perform operations on all entries of a specific + // key type. Given that store entry type comparisons may be performed many + // times, the entry type string is kept as small as possible. + // Example metadata entry type key: + // "[StoreEntryType::kMetadata]_[MetadataType::kSchema]" ==> "1_1" + // Example component hint store entry type key: + // "[StoreEntryType::kComponentHint]_[component_version]_[host]" + // ==> "2_55_foo.com" + // NOTE: The order and value of the existing store entry types within the enum + // cannot be changed, but new types can be added to the end. + // StoreEntryType should remain synchronized with the + // HintCacheStoreEntryType in enums.xml. + // Also ensure to add to the OptimizationGuide.StoreEntryTypes histogram + // suffixes if adding a new one. + enum class StoreEntryType { + kEmpty = 0, + kMetadata = 1, + kComponentHint = 2, + kFetchedHint = 3, + kPredictionModel = 4, + kHostModelFeatures = 5, + kMaxValue = kHostModelFeatures, + }; + + OptimizationGuideStore( + leveldb_proto::ProtoDatabaseProvider* database_provider, + const base::FilePath& database_dir, + scoped_refptr store_task_runner); + // For tests only. + explicit OptimizationGuideStore( + std::unique_ptr database); + virtual ~OptimizationGuideStore(); + + // Initializes the store. If |purge_existing_data| is set to true, + // then the cache is purged during initialization and starts in a fresh state. + // When initialization completes, the provided callback is run asynchronously. + // Virtualized for testing. + virtual void Initialize(bool purge_existing_data, base::OnceClosure callback); + + // Creates and returns a StoreUpdateData object for component hints. This + // object is used to collect hints within a component in a format usable on a + // background thread and is later returned to the store in + // UpdateComponentHints(). The StoreUpdateData object is only created when the + // provided component version is newer than the store's version, indicating + // fresh hints. If the component's version is not newer than the store's + // version, then no StoreUpdateData is created and nullptr is returned. This + // prevents unnecessary processing of the component's hints by the caller. + std::unique_ptr MaybeCreateUpdateDataForComponentHints( + const base::Version& version) const; + + // Creates and returns a HintsUpdateData object for Fetched Hints. + // This object is used to collect a batch of hints in a format that is usable + // to update the store on a background thread. This is always created when + // hints have been successfully fetched from the remote Optimization Guide + // Service so the store can expire old hints, remove hints specified by the + // server, and store the fresh hints. + std::unique_ptr CreateUpdateDataForFetchedHints( + base::Time update_time, + base::Time expiry_time) const; + + // Updates the component hints and version contained within the store. When + // this is called, all pre-existing component hints within the store is purged + // and only the new hints are retained. After the store is fully updated with + // the new component hints, the callback is run asynchronously. + void UpdateComponentHints(std::unique_ptr component_data, + base::OnceClosure callback); + + // Updates the fetched hints contained in the store, including the + // metadata entry. The callback is run asynchronously after the database + // stores the hints. + void UpdateFetchedHints(std::unique_ptr fetched_hints_data, + base::OnceClosure callback); + + // Removes fetched hint store entries from |this|. |entry_keys_| is + // updated after the fetched hint entries are removed. + void ClearFetchedHintsFromDatabase(); + + // Finds the most specific hint entry key for the specified host. Returns + // true if a hint entry key is found, in which case |out_hint_entry_key| is + // populated with the key. All keys for kFetched hints are considered before + // kComponent hints as they are updated more frequently. + bool FindHintEntryKey(const std::string& host, + EntryKey* out_hint_entry_key) const; + + // Loads the hint specified by |hint_entry_key|. + // After the load finishes, the hint data is passed to |callback|. In the case + // where the hint cannot be loaded, the callback is run with a nullptr. + // Depending on the load result, the callback may be synchronous or + // asynchronous. + void LoadHint(const EntryKey& hint_entry_key, HintLoadedCallback callback); + + // Returns the time that the fetched hints in the store can be updated. If + // |this| is not available, base::Time() is returned. + base::Time GetFetchedHintsUpdateTime() const; + + // Removes all fetched hints that have expired from the store. + // |entry_keys_| is updated after the expired fetched hints are + // removed. + void PurgeExpiredFetchedHints(); + + // Removes all host model features that have expired from the store. + // |entry_keys_| is updated after the expired host model features are + // removed. + void PurgeExpiredHostModelFeatures(); + + // Creates and returns a StoreUpdateData object for Prediction Models. This + // object is used to collect a batch of prediction models in a format that is + // usable to update the store on a background thread. This is always created + // when prediction models have been successfully fetched from the remote + // Optimization Guide Service so the store can update old prediction models. + std::unique_ptr CreateUpdateDataForPredictionModels() const; + + // Updates the prediction models contained in the store. The callback is run + // asynchronously after the database stores the prediction models. Virtualized + // for testing. + virtual void UpdatePredictionModels( + std::unique_ptr prediction_models_update_data, + base::OnceClosure callback); + + // Finds the entry key for the prediction model if it is known to the store. + // Returns true if an entry key is found and |out_prediction_model_entry_key| + // is populated with the matching key. + // Virtualized for testing. + virtual bool FindPredictionModelEntryKey( + proto::OptimizationTarget optimization_target, + OptimizationGuideStore::EntryKey* out_prediction_model_entry_key); + + // Loads the prediction model specified by |prediction_model_entry_key|. After + // the load finishes, the prediction model data is passed to |callback|. In + // the case where the prediction model cannot be loaded, the callback is run + // with a nullptr. Depending on the load result, the callback may be + // synchronous or asynchronous. + // Virtualized for testing. + virtual void LoadPredictionModel(const EntryKey& prediction_model_entry_key, + PredictionModelLoadedCallback callback); + + // Creates and returns a StoreUpdateData object for host model features. This + // object is used to collect a batch of host model features in a format that + // is usable to update the store on a background thread. This is always + // created when host model features have been successfully fetched from the + // remote Optimization Guide Service so the store can update old host model + // features. + std::unique_ptr CreateUpdateDataForHostModelFeatures( + base::Time host_model_features_update_time, + base::Time expiry_time) const; + + // Updates the host model features contained in the store. The callback is run + // asynchronously after the database stores the host model features. + // Virtualized for testing. + virtual void UpdateHostModelFeatures( + std::unique_ptr host_model_features_update_data, + base::OnceClosure callback); + + // Finds the entry key for the host model features for |host| if it is known + // to the store. Returns true if an entry key is found and + // |out_host_model_features_entry_key| is populated with the matching key. + bool FindHostModelFeaturesEntryKey( + const std::string& host, + OptimizationGuideStore::EntryKey* out_host_model_features_entry_key) + const; + + // Loads the host model features specified by |host_model_features_entry_key|. + // After the load finishes, the host model features data is passed to + // |callback|. In the case where the host model features cannot be loaded, the + // callback is run with a nullptr. Depending on the load result, the callback + // may be synchronous or asynchronous. + void LoadHostModelFeatures(const EntryKey& host_model_features_entry_key, + HostModelFeaturesLoadedCallback callback); + + // Loads all the host model features known to the store. After the load + // finishes, the host model features data is passed back to |callback|. In the + // case where the host model features cannot be loaded, the callback is run + // with a nullptr. Depending on the load result, the callback may be + // synchronous or asynchronous. + // Virtualized for testing. + virtual void LoadAllHostModelFeatures( + AllHostModelFeaturesLoadedCallback callback); + + // Returns the time that the host model features in the store can be updated. + // If |this| is not available, base::Time() is returned. + base::Time GetHostModelFeaturesUpdateTime() const; + + // Clears all host model features from the database and resets the entry keys. + void ClearHostModelFeaturesFromDatabase(); + + private: + friend class OptimizationGuideStoreTest; + friend class StoreUpdateData; + friend class TestOptimizationGuideStore; + + using EntryKeyPrefix = std::string; + using EntryKeySet = base::flat_set; + + using EntryVector = + leveldb_proto::ProtoDatabase::KeyEntryVector; + using EntryMap = std::map; + + // Metadata types within the store. The metadata type appears at the end of + // metadata entry keys. These values are converted into strings within the + // key. + // Example metadata type keys: + // "[StoreEntryType::kMetadata]_[MetadataType::kSchema]" ==> "1_1" + // "[StoreEntryType::kMetadata]_[MetadataType::kComponent]" ==> "1_2" + // NOTE: The order and value of the existing metadata types within the enum + // cannot be changed, but new types can be added to the end. + enum class MetadataType { + kSchema = 1, + kComponent = 2, + kFetched = 3, + kHostModelFeatures = 4, + }; + + // Current schema version of the hint cache store. When this is changed, + // pre-existing store data from an earlier version is purged. + static const char kStoreSchemaVersion[]; + + // Returns prefix in the key of every metadata entry type entry: "1_" + static EntryKeyPrefix GetMetadataEntryKeyPrefix(); + + // Returns entry key for the specified metadata type entry: "1_[MetadataType]" + static EntryKey GetMetadataTypeEntryKey(MetadataType metadata_type); + + // Returns prefix in the key of every component hint entry: "2_" + static EntryKeyPrefix GetComponentHintEntryKeyPrefixWithoutVersion(); + + // Returns prefix in the key of component hint entries for the specified + // component version: "2_[component_version]_" + static EntryKeyPrefix GetComponentHintEntryKeyPrefix( + const base::Version& component_version); + + // Returns prefix of the key of every fetched hint entry: "3_". + static EntryKeyPrefix GetFetchedHintEntryKeyPrefix(); + + // Returns prefix of the key of every prediction model entry: "4_". + static EntryKeyPrefix GetPredictionModelEntryKeyPrefix(); + + // Returns prefix of the key of every host model features entry: "5_". + static EntryKeyPrefix GetHostModelFeaturesEntryKeyPrefix(); + + // Updates the status of the store to the specified value, validates the + // transition, and destroys the database in the case where the status + // transitions to Status::kFailed. + void UpdateStatus(Status new_status); + + // Returns true if the current status is Status::kAvailable. + bool IsAvailable() const; + + // Asynchronously purges all existing entries from the database and runs the + // callback after it completes. This should only be run during initialization. + void PurgeDatabase(base::OnceClosure callback); + + // Updates |component_version_| and |component_hint_entry_key_prefix_| for + // the new component version. This must be called rather than directly + // modifying |component_version_|, as it ensures that both member variables + // are updated in sync. + void SetComponentVersion(const base::Version& component_version); + + // Resets |component_version_| and |component_hint_entry_key_prefix_| back to + // their default state. Called after the database is destroyed. + void ClearComponentVersion(); + + // Asynchronously loads the entry keys from the store, populates |entry_keys_| + // with them, and runs the provided callback after they finish loading. In the + // case where there is currently an in-flight update, this does nothing, as + // the entry keys will be loaded after the update completes. + void MaybeLoadEntryKeys(base::OnceClosure callback); + + // Returns the total entry keys contained within the store. + size_t GetEntryKeyCount() const; + + // Finds the most specific host suffix of the host name that the store has an + // entry with the provided prefix, |entry_key_prefix|. |out_entry_key| is + // populated with the entry key for the corresponding hint. Returns true if + // an entry was successfully found. + bool FindEntryKeyForHostWithPrefix( + const std::string& host, + EntryKey* out_entry_key, + const EntryKeyPrefix& entry_key_prefix) const; + + // Callback that identifies any expired |entries| and + // asynchronously removes them from the store. + void OnLoadEntriesToPurgeExpired(bool success, + std::unique_ptr entries); + + // Callback that runs after the database finishes being initialized. If + // |purge_existing_data| is true, then unconditionally purges the database; + // otherwise, triggers loading of the metadata. + void OnDatabaseInitialized(bool purge_existing_data, + base::OnceClosure callback, + leveldb_proto::Enums::InitStatus status); + + // Callback that is run after the database finishes being destroyed. + void OnDatabaseDestroyed(bool success); + + // Callback that runs after the metadata finishes being loaded. This + // validates the schema version, sets the component version, and either purges + // the store (on a schema version mismatch) or loads all hint entry keys (on + // a schema version match). + void OnLoadMetadata(base::OnceClosure callback, + bool success, + std::unique_ptr metadata_entries); + + // Callback that runs after the database is purged during initialization. + void OnPurgeDatabase(base::OnceClosure callback, bool success); + + // Callback that runs after the data within the store is fully + // updated. If the update was successful, it attempts to load all of the + // entry keys contained within the database. + void OnUpdateStore(base::OnceClosure callback, bool success); + + // Callback that runs after the hint entry keys are fully loaded. If there's + // currently an in-flight component update, then the hint entry keys will be + // loaded again after the component update completes, so the results are + // tossed; otherwise, |entry_keys| is moved into |entry_keys_|. + // Regardless of the outcome of loading the keys, the callback always runs. + void OnLoadEntryKeys(std::unique_ptr entry_keys, + base::OnceClosure callback, + bool success, + std::unique_ptr unused); + + // Callback that runs after a hint entry is loaded from the database. If + // there's currently an in-flight component update, then the hint is about to + // be invalidated, so results are tossed; otherwise, the hint is released into + // the callback, allowing the caller to own the hint without copying it. + // Regardless of the success or failure of retrieving the key, the callback + // always runs (it simply runs with a nullptr on failure). + void OnLoadHint(const EntryKey& entry_key, + HintLoadedCallback callback, + bool success, + std::unique_ptr entry); + + // Callback that runs after a prediction model entry is loaded from the + // database. If there's currently an in-flight update, then the data could be + // invalidated, so loaded model is discarded. Otherwise, the prediction model + // is released into the callback, allowing the caller to own the prediction + // model without copying it. Regardless of the success or failure of + // retrieving the key, the callback always runs (it simply runs with a nullptr + // on failure). + void OnLoadPredictionModel(PredictionModelLoadedCallback callback, + bool success, + std::unique_ptr entry); + + // Callback that runs after a host model features entry is loaded from the + // database. If there's currently an in-flight update, then the data could be + // invalidated, so loaded host model features data is discarded. Otherwise, + // the host model features are released into the callback, allowing the caller + // to own the host model features without copying it. Regardless of the + // success or failure of retrieving the key, the callback always runs (it + // simply runs with a nullptr on failure). + void OnLoadHostModelFeatures(HostModelFeaturesLoadedCallback callback, + bool success, + std::unique_ptr entry); + + // Callback that runs after all the host model features entries are loaded + // from the database. If there's currently an in-flight update, then the data + // could be invalidated, so loaded host model features data is discarded. + // Otherwise, the host model features are released into the callback, allowing + // the caller to own the host model features without copying it. Regardless of + // the success or failure of retrieving the key, the callback always runs (it + // simply runs with a nullptr on failure). + void OnLoadAllHostModelFeatures( + AllHostModelFeaturesLoadedCallback callback, + bool success, + std::unique_ptr> entry); + + // Proto database used by the store. + std::unique_ptr database_; + + // The current status of the store. It should only be updated through + // UpdateStatus(), which validates status transitions and triggers + // accompanying logic. + Status status_ = Status::kUninitialized; + + // The current component version of the store. This should only be updated + // via SetComponentVersion(), which ensures that both |component_version_| + // and |component_hint_key_prefix_| are updated at the same time. + base::Optional component_version_; + + // The current entry key prefix shared by all component hints containd within + // the store. While this could be generated on the fly using + // |component_version_|, it is retaind separately as an optimization, as it + // is needed often. + EntryKeyPrefix component_hint_entry_key_prefix_; + + // If a component data update is in the middle of being processed; when this + // is true, keys and hints will not be returned by the store. + bool data_update_in_flight_ = false; + + // The next update time for the fetched hints that are currently in the + // store. + base::Time fetched_update_time_; + + // The next update time for the host model features that are currently in the + // store. + base::Time host_model_features_update_time_; + + // The keys of the entries available within the store. + std::unique_ptr entry_keys_; + + SEQUENCE_CHECKER(sequence_checker_); + + base::WeakPtrFactory weak_ptr_factory_{this}; + + DISALLOW_COPY_AND_ASSIGN(OptimizationGuideStore); +}; + +} // namespace optimization_guide + +#endif // COMPONENTS_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_STORE_H_ diff --git a/chromium/components/optimization_guide/optimization_guide_store_unittest.cc b/chromium/components/optimization_guide/optimization_guide_store_unittest.cc new file mode 100644 index 00000000000..c468415d5ce --- /dev/null +++ b/chromium/components/optimization_guide/optimization_guide_store_unittest.cc @@ -0,0 +1,2070 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/optimization_guide/optimization_guide_store.h" + +#include +#include + +#include "base/bind.h" +#include "base/macros.h" +#include "base/optional.h" +#include "base/strings/string_number_conversions.h" +#include "base/test/metrics/histogram_tester.h" +#include "components/leveldb_proto/testing/fake_db.h" +#include "components/optimization_guide/optimization_guide_features.h" +#include "components/optimization_guide/proto/hint_cache.pb.h" +#include "components/optimization_guide/proto/models.pb.h" +#include "components/optimization_guide/store_update_data.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "url/gurl.h" + +using leveldb_proto::test::FakeDB; +using testing::Mock; + +namespace optimization_guide { + +namespace { + +constexpr char kDefaultComponentVersion[] = "1.0.0"; +constexpr char kUpdateComponentVersion[] = "2.0.0"; + +std::string GetHostSuffix(size_t id) { + // Host suffix alternates between two different domain types depending on + // whether the id is odd or even. + if (id % 2 == 0) { + return "domain" + base::NumberToString(id) + ".org"; + } else { + return "different.domain" + base::NumberToString(id) + ".co.in"; + } +} + +enum class MetadataSchemaState { + kMissing, + kInvalid, + kValid, +}; + +std::unique_ptr CreatePredictionModel() { + std::unique_ptr prediction_model = + std::make_unique(); + + optimization_guide::proto::ModelInfo* model_info = + prediction_model->mutable_model_info(); + model_info->set_version(1); + model_info->add_supported_model_features( + proto::CLIENT_MODEL_FEATURE_EFFECTIVE_CONNECTION_TYPE); + model_info->set_optimization_target( + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD); + model_info->add_supported_model_types( + proto::ModelType::MODEL_TYPE_DECISION_TREE); + return prediction_model; +} + +} // namespace + +class OptimizationGuideStoreTest : public testing::Test { + public: + using StoreEntry = proto::StoreEntry; + using StoreEntryMap = std::map; + + OptimizationGuideStoreTest() : db_(nullptr) {} + + void TearDown() override { last_loaded_hint_.reset(); } + + // Initializes the entries contained within the database on startup. + void SeedInitialData( + MetadataSchemaState state, + base::Optional component_hint_count = base::Optional(), + base::Optional fetched_hints_update = + base::Optional(), + base::Optional host_model_features_update = + base::Optional()) { + db_store_.clear(); + + // Add a metadata schema entry if its state isn't kMissing. The version + // entry version is set to the store's current version if the state is + // kValid; otherwise, it's set to the invalid version of "0". + if (state == MetadataSchemaState::kValid) { + db_store_[OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kSchema)] + .set_version(OptimizationGuideStore::kStoreSchemaVersion); + } else if (state == MetadataSchemaState::kInvalid) { + db_store_[OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kSchema)] + .set_version("0"); + } + + // If the database is being seeded with component hints, it is indicated + // with a provided count. Add the component metadata with the default + // component version and then add the indicated number of component hints. + // if (component_hint_count && component_hint_count > + // static_cast(0)) { + if (component_hint_count && component_hint_count > 0u) { + db_store_[OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kComponent)] + .set_version(kDefaultComponentVersion); + OptimizationGuideStore::EntryKeyPrefix component_hint_key_prefix = + OptimizationGuideStore::GetComponentHintEntryKeyPrefix( + base::Version(kDefaultComponentVersion)); + for (size_t i = 0; i < component_hint_count.value(); ++i) { + std::string host_suffix = GetHostSuffix(i); + StoreEntry& entry = db_store_[component_hint_key_prefix + host_suffix]; + entry.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kComponentHint)); + proto::Hint* hint = entry.mutable_hint(); + hint->set_key(host_suffix); + hint->set_key_representation(proto::HOST_SUFFIX); + proto::PageHint* page_hint = hint->add_page_hints(); + page_hint->set_page_pattern("page pattern " + base::NumberToString(i)); + } + } + if (fetched_hints_update) { + db_store_[OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kFetched)] + .set_update_time_secs( + fetched_hints_update->ToDeltaSinceWindowsEpoch().InSeconds()); + } + if (host_model_features_update) { + db_store_[OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kHostModelFeatures)] + .set_update_time_secs( + host_model_features_update->ToDeltaSinceWindowsEpoch() + .InSeconds()); + } + } + + // Moves the specified number of component hints into the update data. + void SeedComponentUpdateData(StoreUpdateData* update_data, + size_t component_hint_count) { + for (size_t i = 0; i < component_hint_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + proto::Hint hint; + hint.set_key(host_suffix); + hint.set_key_representation(proto::HOST_SUFFIX); + proto::PageHint* page_hint = hint.add_page_hints(); + page_hint->set_page_pattern("page pattern " + base::NumberToString(i)); + update_data->MoveHintIntoUpdateData(std::move(hint)); + } + } + // Moves the specified number of fetched hints into the update data. + void SeedFetchedUpdateData(StoreUpdateData* update_data, + size_t fetched_hint_count) { + for (size_t i = 0; i < fetched_hint_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + proto::Hint hint; + hint.set_key(host_suffix); + hint.set_key_representation(proto::HOST_SUFFIX); + proto::PageHint* page_hint = hint.add_page_hints(); + page_hint->set_page_pattern("page pattern " + base::NumberToString(i)); + update_data->MoveHintIntoUpdateData(std::move(hint)); + } + } + + // Moves a prediction model with |optimization_target| into the update data. + void SeedPredictionModelUpdateData( + StoreUpdateData* update_data, + optimization_guide::proto::OptimizationTarget optimization_target) { + std::unique_ptr + prediction_model = CreatePredictionModel(); + prediction_model->mutable_model_info()->set_optimization_target( + optimization_target); + update_data->CopyPredictionModelIntoUpdateData(*prediction_model); + } + + // Moves |host_model_features_count| into |update_data|. + void SeedHostModelFeaturesUpdateData(StoreUpdateData* update_data, + size_t host_model_features_count) { + for (size_t i = 0; i < host_model_features_count; i++) { + std::string host_suffix = GetHostSuffix(i); + proto::HostModelFeatures host_model_features; + proto::ModelFeature* model_feature = + host_model_features.add_model_features(); + model_feature->set_feature_name("host_feat1"); + model_feature->set_double_value(2.0); + host_model_features.set_host(host_suffix); + update_data->CopyHostModelFeaturesIntoUpdateData(host_model_features); + } + } + + void CreateDatabase() { + // Reset everything. + db_ = nullptr; + guide_store_.reset(); + + // Setup the fake db and the class under test. + auto db = std::make_unique>(&db_store_); + db_ = db.get(); + + guide_store_ = std::make_unique(std::move(db)); + } + + void InitializeDatabase(bool success, bool purge_existing_data = false) { + EXPECT_CALL(*this, OnInitialized()); + guide_store()->Initialize( + purge_existing_data, + base::BindOnce(&OptimizationGuideStoreTest::OnInitialized, + base::Unretained(this))); + // OnDatabaseInitialized callback + db()->InitStatusCallback(success ? leveldb_proto::Enums::kOK + : leveldb_proto::Enums::kError); + } + + void InitializeStore(MetadataSchemaState state, + bool purge_existing_data = false) { + InitializeDatabase(true /*=success*/, purge_existing_data); + + if (purge_existing_data) { + // OnPurgeDatabase callback + db()->UpdateCallback(true); + return; + } + + // OnLoadMetadata callback + db()->LoadCallback(true); + if (state == MetadataSchemaState::kValid) { + // OnLoadEntryKeys callback + db()->LoadCallback(true); + } else { + // OnPurgeDatabase callback + db()->UpdateCallback(true); + } + } + + void UpdateComponentHints(std::unique_ptr component_data, + bool update_success = true, + bool load_hint_entry_keys_success = true) { + EXPECT_CALL(*this, OnUpdateStore()); + guide_store()->UpdateComponentHints( + std::move(component_data), + base::BindOnce(&OptimizationGuideStoreTest::OnUpdateStore, + base::Unretained(this))); + // OnUpdateStore callback + db()->UpdateCallback(update_success); + if (update_success) { + // OnLoadEntryKeys callback + db()->LoadCallback(load_hint_entry_keys_success); + } + } + + void UpdateFetchedHints(std::unique_ptr fetched_data, + bool update_success = true, + bool load_hint_entry_keys_success = true) { + EXPECT_CALL(*this, OnUpdateStore()); + guide_store()->UpdateFetchedHints( + std::move(fetched_data), + base::BindOnce(&OptimizationGuideStoreTest::OnUpdateStore, + base::Unretained(this))); + // OnUpdateStore callback + db()->UpdateCallback(update_success); + if (update_success) { + // OnLoadEntryKeys callback + db()->LoadCallback(load_hint_entry_keys_success); + } + } + + void UpdatePredictionModels( + std::unique_ptr prediction_models_data, + bool update_success = true, + bool load_prediction_models_entry_keys_success = true) { + EXPECT_CALL(*this, OnUpdateStore()); + guide_store()->UpdatePredictionModels( + std::move(prediction_models_data), + base::BindOnce(&OptimizationGuideStoreTest::OnUpdateStore, + base::Unretained(this))); + // OnUpdateStore callback + db()->UpdateCallback(update_success); + if (update_success) { + // OnLoadEntryKeys callback + db()->LoadCallback(load_prediction_models_entry_keys_success); + } + } + + void UpdateHostModelFeatures( + std::unique_ptr host_model_features_data, + bool update_success = true, + bool load_host_model_features_entry_keys_success = true) { + EXPECT_CALL(*this, OnUpdateStore()); + guide_store()->UpdateHostModelFeatures( + std::move(host_model_features_data), + base::BindOnce(&OptimizationGuideStoreTest::OnUpdateStore, + base::Unretained(this))); + // OnUpdateStore callback + db()->UpdateCallback(update_success); + if (update_success) { + // OnLoadEntryKeys callback + db()->LoadCallback(load_host_model_features_entry_keys_success); + } + } + + void ClearFetchedHintsFromDatabase() { + guide_store()->ClearFetchedHintsFromDatabase(); + db()->UpdateCallback(true); + db()->LoadCallback(true); + } + + void ClearHostModelFeaturesFromDatabase() { + guide_store()->ClearHostModelFeaturesFromDatabase(); + db()->UpdateCallback(true); + db()->LoadCallback(true); + } + + void PurgeExpiredFetchedHints() { + guide_store()->PurgeExpiredFetchedHints(); + + // OnLoadExpiredEntriesToPurge + db()->LoadCallback(true); + // OnUpdateStore + db()->UpdateCallback(true); + // OnLoadEntryKeys callback + db()->LoadCallback(true); + } + + void PurgeExpiredHostModelFeatures() { + guide_store()->PurgeExpiredHostModelFeatures(); + + // OnLoadExpiredEntriesToPurge + db()->LoadCallback(true); + // OnUpdateStore + db()->UpdateCallback(true); + // OnLoadEntryKeys callback + db()->LoadCallback(true); + } + + bool IsMetadataSchemaEntryKeyPresent() const { + return IsKeyPresent(OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kSchema)); + } + + // Verifies that the fetched metadata has the expected next update time. + void ExpectFetchedMetadata(base::Time update_time) const { + const auto& metadata_entry = + db_store_.find(OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kFetched)); + if (metadata_entry != db_store_.end()) { + // The next update time should have same time up to the second as the + // metadata entry is stored in seconds. + EXPECT_TRUE( + base::Time::FromDeltaSinceWindowsEpoch(base::TimeDelta::FromSeconds( + metadata_entry->second.update_time_secs())) - + update_time < + base::TimeDelta::FromSeconds(1)); + } else { + FAIL() << "No fetched metadata found"; + } + } + + // Verifies that the host model features metadata has the expected next update + // time. + void ExpectHostModelFeaturesMetadata(base::Time update_time) const { + const auto& metadata_entry = + db_store_.find(OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kHostModelFeatures)); + if (metadata_entry != db_store_.end()) { + // The next update time should have same time up to the second as the + // metadata entry is stored in seconds. + EXPECT_TRUE( + base::Time::FromDeltaSinceWindowsEpoch(base::TimeDelta::FromSeconds( + metadata_entry->second.update_time_secs())) - + update_time < + base::TimeDelta::FromSeconds(1)); + } else { + FAIL() << "No host model features metadata found"; + } + } + // Verifies that the component metadata has the expected version and all + // expected component hints are present. + void ExpectComponentHintsPresent(const std::string& version, + int count) const { + const auto& metadata_entry = + db_store_.find(OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kComponent)); + if (metadata_entry != db_store_.end()) { + EXPECT_EQ(metadata_entry->second.version(), version); + } else { + FAIL() << "No component metadata found"; + } + + OptimizationGuideStore::EntryKeyPrefix component_hint_entry_key_prefix = + OptimizationGuideStore::GetComponentHintEntryKeyPrefix( + base::Version(version)); + for (int i = 0; i < count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey hint_entry_key = + component_hint_entry_key_prefix + host_suffix; + const auto& hint_entry = db_store_.find(hint_entry_key); + if (hint_entry == db_store_.end()) { + FAIL() << "No entry found for component hint: " << hint_entry_key; + continue; + } + + if (!hint_entry->second.has_hint()) { + FAIL() << "Component hint entry does not have hint: " << hint_entry_key; + continue; + } + + EXPECT_EQ(hint_entry->second.hint().key(), host_suffix); + } + } + + // Returns true if the data is present for the given key. + bool IsKeyPresent(const OptimizationGuideStore::EntryKey& entry_key) const { + return db_store_.find(entry_key) != db_store_.end(); + } + + size_t GetDBStoreEntryCount() const { return db_store_.size(); } + size_t GetStoreEntryKeyCount() const { + return guide_store_->GetEntryKeyCount(); + } + + OptimizationGuideStore* guide_store() { return guide_store_.get(); } + FakeDB* db() { return db_; } + + const OptimizationGuideStore::EntryKey& last_loaded_entry_key() const { + return last_loaded_entry_key_; + } + + proto::Hint* last_loaded_hint() { return last_loaded_hint_.get(); } + + proto::HostModelFeatures* last_loaded_host_model_features() { + return last_loaded_host_model_features_.get(); + } + + std::vector* last_loaded_all_host_model_features() { + return last_loaded_all_host_model_features_.get(); + } + + proto::PredictionModel* last_loaded_prediction_model() { + return last_loaded_prediction_model_.get(); + } + + void OnHintLoaded(const OptimizationGuideStore::EntryKey& hint_entry_key, + std::unique_ptr loaded_hint) { + last_loaded_entry_key_ = hint_entry_key; + last_loaded_hint_ = std::move(loaded_hint); + } + + void OnHostModelFeaturesLoaded( + std::unique_ptr loaded_host_model_features) { + last_loaded_host_model_features_ = std::move(loaded_host_model_features); + } + + void OnAllHostModelFeaturesLoaded( + std::unique_ptr> + loaded_all_host_model_features) { + last_loaded_all_host_model_features_ = + std::move(loaded_all_host_model_features); + } + + void OnPredictionModelLoaded( + std::unique_ptr loaded_prediction_model) { + last_loaded_prediction_model_ = std::move(loaded_prediction_model); + } + + MOCK_METHOD0(OnInitialized, void()); + MOCK_METHOD0(OnUpdateStore, void()); + + private: + FakeDB* db_; + StoreEntryMap db_store_; + std::unique_ptr guide_store_; + + OptimizationGuideStore::EntryKey last_loaded_entry_key_; + std::unique_ptr last_loaded_hint_; + std::unique_ptr last_loaded_host_model_features_; + std::unique_ptr> + last_loaded_all_host_model_features_; + std::unique_ptr last_loaded_prediction_model_; + + DISALLOW_COPY_AND_ASSIGN(OptimizationGuideStoreTest); +}; + +TEST_F(OptimizationGuideStoreTest, NoInitialization) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kMissing); + CreateDatabase(); + + histogram_tester.ExpectTotalCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnInitializeWithNoInitialData) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kMissing); + CreateDatabase(); + InitializeDatabase(false /*=success*/); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectTotalCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnLoadMetadataWithNoInitialData) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kMissing); + CreateDatabase(); + InitializeDatabase(true /*=success*/); + + // OnLoadMetadata callback + db()->LoadCallback(false); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 1 /* kLoadMetadataFailed */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnUpdateMetadataNoInitialData) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kMissing); + CreateDatabase(); + + InitializeDatabase(true /*=success*/); + + // OnLoadMetadata callback + db()->LoadCallback(true); + // OnPurgeDatabase callback + db()->UpdateCallback(false); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 2 /* kSchemaMetadataMissing */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnInitializeWithInitialData) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kValid, 10); + CreateDatabase(); + InitializeDatabase(false /*=success*/); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectTotalCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnLoadMetadataWithInitialData) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kValid, 10); + CreateDatabase(); + InitializeDatabase(true /*=success*/); + + // OnLoadMetadata callback + db()->LoadCallback(false); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 1 /* kLoadMetadataFailed */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnUpdateMetadataWithInvalidSchemaEntry) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kInvalid, 10); + CreateDatabase(); + InitializeDatabase(true /*=success*/); + + // OnLoadMetadata callback + db()->LoadCallback(true); + // OnPurgeDatabase callback + db()->UpdateCallback(false); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 3 /* kSchemaMetadataWrongVersion */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeFailedOnLoadHintEntryKeysWithInitialData) { + base::HistogramTester histogram_tester; + + SeedInitialData(MetadataSchemaState::kValid, 10, base::Time().Now()); + CreateDatabase(); + InitializeDatabase(true /*=success*/); + + // OnLoadMetadata callback + db()->LoadCallback(true); + // OnLoadEntryKeys callback + db()->LoadCallback(false); + + // In the case where initialization fails, the store should be fully purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 0 /* kSuccess */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore." + "HostModelFeaturesLoadMetadataResult", + false, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 1); +} + +TEST_F(OptimizationGuideStoreTest, InitializeSucceededWithoutSchemaEntry) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kMissing; + SeedInitialData(schema_state); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry and nothing else. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 2 /* kSchemaMetadataMissing */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, InitializeSucceededWithInvalidSchemaEntry) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kInvalid; + SeedInitialData(schema_state); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry and nothing else. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 3 /* kSchemaMetadataWrongVersion */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, InitializeSucceededWithValidSchemaEntry) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry and nothing else. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 4 /* kComponentMetadataMissing*/, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 5 /* kFetchedMetadataMissing*/, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 6 /* kComponentAndFetchedMetadataMissing*/, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore." + "HostModelFeaturesLoadMetadataResult", + false, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeSucceededWithInvalidSchemaEntryAndInitialData) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kInvalid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry and nothing else, as + // the initial component hints are all purged. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 3 /* kSchemaMetadataWrongVersion */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, InitializeSucceededWithPurgeExistingData) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state, true /*=purge_existing_data*/); + + // The store should contain the schema metadata entry and nothing else. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(1)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + + histogram_tester.ExpectTotalCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", 0); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeSucceededWithValidSchemaEntryAndInitialData) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t component_hint_count = 10; + SeedInitialData(schema_state, component_hint_count, + base::Time().Now(), /* fetch_update_time */ + base::Time().Now() /* host_model_features_update_time */); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry, the component metadata + // entry, and all of the initial component hints. + EXPECT_EQ(GetDBStoreEntryCount(), + static_cast(component_hint_count + 4)); + EXPECT_EQ(GetStoreEntryKeyCount(), component_hint_count); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + ExpectComponentHintsPresent(kDefaultComponentVersion, component_hint_count); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 0 /* kSuccess */, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore." + "HostModelFeaturesLoadMetadataResult", + true, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeSucceededWithValidSchemaEntryAndComponentDataOnly) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t component_hint_count = 10; + SeedInitialData(schema_state, component_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry, the component metadata + // entry, and all of the initial component hints. + EXPECT_EQ(GetDBStoreEntryCount(), + static_cast(component_hint_count + 2)); + EXPECT_EQ(GetStoreEntryKeyCount(), component_hint_count); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + ExpectComponentHintsPresent(kDefaultComponentVersion, component_hint_count); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 4 /* kComponentMetadataMissing*/, 0); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 5 /* kFetchedMetadataMissing*/, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 6 /* kComponentAndFetchedMetadataMissing*/, 0); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore." + "HostModelFeaturesLoadMetadataResult", + false, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, + InitializeSucceededWithValidSchemaEntryAndFetchedMetaData) { + base::HistogramTester histogram_tester; + + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t component_hint_count = 0; + SeedInitialData(schema_state, component_hint_count, base::Time().Now()); + CreateDatabase(); + InitializeStore(schema_state); + + // The store should contain the schema metadata entry, the component metadata + // entry, and all of the initial component hints. + EXPECT_EQ(GetDBStoreEntryCount(), + static_cast(component_hint_count + 2)); + EXPECT_EQ(GetStoreEntryKeyCount(), component_hint_count); + + EXPECT_TRUE(IsMetadataSchemaEntryKeyPresent()); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.LoadMetadataResult", + 4 /* kComponentMetadataMissing*/, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore." + "HostModelFeaturesLoadMetadataResult", + false, 1); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 0 /* kUninitialized */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 1 /* kInitializing */, + 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 2 /* kAvailable */, 1); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheLevelDBStore.Status", 3 /* kFailed */, 0); +} + +TEST_F(OptimizationGuideStoreTest, + CreateComponentUpdateDataFailsForUninitializedStore) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + + // StoreUpdateData for a component update should only be created if the store + // is initialized. + EXPECT_FALSE(guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion))); +} + +TEST_F(OptimizationGuideStoreTest, + CreateComponentUpdateDataFailsForEarlierVersion) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state); + + // No StoreUpdateData for a component update should be created when the + // component version of the update is older than the store's component + // version. + EXPECT_FALSE(guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version("0.0.0"))); +} + +TEST_F(OptimizationGuideStoreTest, + CreateComponentUpdateDataFailsForCurrentVersion) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state); + + // No StoreUpdateData should be created when the component version of the + // update is the same as the store's component version. + EXPECT_FALSE(guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kDefaultComponentVersion))); +} + +TEST_F(OptimizationGuideStoreTest, + CreateComponentUpdateDataSucceedsWithNoPreexistingVersion) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state); + CreateDatabase(); + InitializeStore(schema_state); + + // StoreUpdateData for a component update should be created when there is no + // pre-existing component in the store. + EXPECT_TRUE(guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kDefaultComponentVersion))); +} + +TEST_F(OptimizationGuideStoreTest, + CreateComponentUpdateDataSucceedsForNewerVersion) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state); + + // StoreUpdateData for a component update should be created when the component + // version of the update is newer than the store's component version. + EXPECT_TRUE(guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion))); +} + +TEST_F(OptimizationGuideStoreTest, UpdateComponentHintsUpdateEntriesFails) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), 5); + + UpdateComponentHints(std::move(update_data), false /*update_success*/); + + // The store should be purged if the component data update fails. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); +} + +TEST_F(OptimizationGuideStoreTest, UpdateComponentHintsGetKeysFails) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 10); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), 5); + + UpdateComponentHints(std::move(update_data), true /*update_success*/, + false /*load_hints_keys_success*/); + + // The store should be purged if loading the keys after the component update + // fails. + EXPECT_EQ(GetDBStoreEntryCount(), static_cast(0)); + EXPECT_EQ(GetStoreEntryKeyCount(), static_cast(0)); +} + +TEST_F(OptimizationGuideStoreTest, UpdateComponentHints) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + size_t update_hint_count = 5; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), update_hint_count); + UpdateComponentHints(std::move(update_data)); + + // When the component update succeeds, the store should contain the schema + // metadata entry, the component metadata entry, and all of the update's + // component hints. + EXPECT_EQ(GetDBStoreEntryCount(), update_hint_count + 2); + EXPECT_EQ(GetStoreEntryKeyCount(), update_hint_count); + ExpectComponentHintsPresent(kUpdateComponentVersion, update_hint_count); +} + +TEST_F(OptimizationGuideStoreTest, + UpdateComponentHintsAfterInitializationDataPurge) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + size_t update_hint_count = 5; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state, true /*=purge_existing_data*/); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), update_hint_count); + UpdateComponentHints(std::move(update_data)); + + // When the component update succeeds, the store should contain the schema + // metadata entry, the component metadata entry, and all of the update's + // component hints. + EXPECT_EQ(GetDBStoreEntryCount(), update_hint_count + 2); + EXPECT_EQ(GetStoreEntryKeyCount(), update_hint_count); + ExpectComponentHintsPresent(kUpdateComponentVersion, update_hint_count); +} + +TEST_F(OptimizationGuideStoreTest, + CreateComponentDataWithAlreadyUpdatedVersionFails) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + size_t update_hint_count = 5; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), update_hint_count); + UpdateComponentHints(std::move(update_data)); + + // StoreUpdateData for the component update should not be created for a second + // component update with the same version as the first component update. + EXPECT_FALSE(guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion))); +} + +TEST_F(OptimizationGuideStoreTest, + UpdateComponentHintsWithUpdatedVersionFails) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + size_t update_hint_count_1 = 5; + size_t update_hint_count_2 = 15; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + // Create two updates for the same component version with different counts. + std::unique_ptr update_data_1 = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + std::unique_ptr update_data_2 = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data_1); + SeedComponentUpdateData(update_data_1.get(), update_hint_count_1); + ASSERT_TRUE(update_data_2); + SeedComponentUpdateData(update_data_2.get(), update_hint_count_2); + + // Update the component data with the same component version twice: + // first with |update_data_1| and then with |update_data_2|. + UpdateComponentHints(std::move(update_data_1)); + + EXPECT_CALL(*this, OnUpdateStore()); + guide_store()->UpdateComponentHints( + std::move(update_data_2), + base::BindOnce(&OptimizationGuideStoreTest::OnUpdateStore, + base::Unretained(this))); + + // Verify that the store is populated with the component data from + // |update_data_1| and not |update_data_2|. + EXPECT_EQ(GetDBStoreEntryCount(), update_hint_count_1 + 2); + EXPECT_EQ(GetStoreEntryKeyCount(), update_hint_count_1); + ExpectComponentHintsPresent(kUpdateComponentVersion, update_hint_count_1); +} + +TEST_F(OptimizationGuideStoreTest, LoadHintOnUnavailableStore) { + size_t initial_hint_count = 10; + SeedInitialData(MetadataSchemaState::kValid, initial_hint_count); + CreateDatabase(); + + const OptimizationGuideStore::EntryKey kInvalidEntryKey = "invalid"; + guide_store()->LoadHint( + kInvalidEntryKey, + base::BindOnce(&OptimizationGuideStoreTest::OnHintLoaded, + base::Unretained(this))); + + // Verify that the OnHintLoaded callback runs when the store is unavailable + // and that both the key and the hint were correctly set in it. + EXPECT_EQ(last_loaded_entry_key(), kInvalidEntryKey); + EXPECT_FALSE(last_loaded_hint()); +} + +TEST_F(OptimizationGuideStoreTest, LoadHintFailure) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t hint_count = 10; + SeedInitialData(schema_state, hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + const OptimizationGuideStore::EntryKey kInvalidEntryKey = "invalid"; + guide_store()->LoadHint( + kInvalidEntryKey, + base::BindOnce(&OptimizationGuideStoreTest::OnHintLoaded, + base::Unretained(this))); + + // OnLoadHint callback + db()->GetCallback(false); + + // Verify that the OnHintLoaded callback runs when the store is unavailable + // and that both the key and the hint were correctly set in it. + EXPECT_EQ(last_loaded_entry_key(), kInvalidEntryKey); + EXPECT_FALSE(last_loaded_hint()); +} + +TEST_F(OptimizationGuideStoreTest, LoadHintSuccessInitialData) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t hint_count = 10; + SeedInitialData(schema_state, hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + // Verify that all component hints in the initial data can successfully be + // loaded from the store. + for (size_t i = 0; i < hint_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey hint_entry_key; + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + continue; + } + + guide_store()->LoadHint( + hint_entry_key, + base::BindOnce(&OptimizationGuideStoreTest::OnHintLoaded, + base::Unretained(this))); + + // OnLoadHint callback + db()->GetCallback(true); + + EXPECT_EQ(last_loaded_entry_key(), hint_entry_key); + if (!last_loaded_hint()) { + FAIL() << "Loaded hint was null for entry key: " << hint_entry_key; + continue; + } + + EXPECT_EQ(last_loaded_hint()->key(), host_suffix); + } +} + +TEST_F(OptimizationGuideStoreTest, LoadHintSuccessUpdateData) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + size_t update_hint_count = 5; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), update_hint_count); + UpdateComponentHints(std::move(update_data)); + + // Verify that all component hints within a successful component update can + // be loaded from the store. + for (size_t i = 0; i < update_hint_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey hint_entry_key; + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + continue; + } + + guide_store()->LoadHint( + hint_entry_key, + base::BindOnce(&OptimizationGuideStoreTest::OnHintLoaded, + base::Unretained(this))); + + // OnLoadHint callback + db()->GetCallback(true); + + EXPECT_EQ(last_loaded_entry_key(), hint_entry_key); + if (!last_loaded_hint()) { + FAIL() << "Loaded hint was null for entry key: " << hint_entry_key; + continue; + } + + EXPECT_EQ(last_loaded_hint()->key(), host_suffix); + } +} + +TEST_F(OptimizationGuideStoreTest, FindHintEntryKeyOnUnavailableStore) { + size_t initial_hint_count = 10; + SeedInitialData(MetadataSchemaState::kValid, initial_hint_count); + CreateDatabase(); + + std::string host_suffix = GetHostSuffix(0); + OptimizationGuideStore::EntryKey hint_entry_key; + + // Verify that hint entry keys can't be found when the store is unavailable. + EXPECT_FALSE(guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); +} + +TEST_F(OptimizationGuideStoreTest, FindHintEntryKeyInitialData) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t hint_count = 10; + SeedInitialData(schema_state, hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + // Verify that all hints contained within the initial store data are reported + // as being found and hints that are not containd within the initial data are + // properly reported as not being found. + for (size_t i = 0; i < hint_count * 2; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey hint_entry_key; + bool success = + guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key); + EXPECT_EQ(success, i < hint_count); + } +} + +TEST_F(OptimizationGuideStoreTest, FindHintEntryKeyUpdateData) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + size_t update_hint_count = 5; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + SeedComponentUpdateData(update_data.get(), update_hint_count); + UpdateComponentHints(std::move(update_data)); + + // Verify that all hints contained within the component update are reported + // by the store as being found and hints that are not containd within the + // component update are properly reported as not being found. + for (size_t i = 0; i < update_hint_count * 2; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey hint_entry_key; + bool success = + guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key); + EXPECT_EQ(success, i < update_hint_count); + } +} + +TEST_F(OptimizationGuideStoreTest, FetchedHintsMetadataStored) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 10, update_time); + CreateDatabase(); + InitializeStore(schema_state); + + ExpectFetchedMetadata(update_time); +} + +TEST_F(OptimizationGuideStoreTest, FindHintEntryKeyForFetchedHints) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t update_hint_count = 5; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 0); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForFetchedHints( + update_time, update_time + optimization_guide::features:: + StoredFetchedHintsFreshnessDuration()); + ASSERT_TRUE(update_data); + SeedFetchedUpdateData(update_data.get(), update_hint_count); + UpdateFetchedHints(std::move(update_data)); + + for (size_t i = 0; i < update_hint_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey hint_entry_key; + bool success = + guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key); + EXPECT_EQ(success, i < update_hint_count); + } +} + +TEST_F(OptimizationGuideStoreTest, + FindHintEntryKeyCheckFetchedBeforeComponentHints) { + base::HistogramTester histogram_tester; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + base::Version version("2.0.0"); + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + + proto::Hint hint1; + hint1.set_key("domain1.org"); + hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint1)); + proto::Hint hint2; + hint2.set_key("host.domain2.org"); + hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint2)); + + UpdateComponentHints(std::move(update_data)); + + // Add fetched hints to the store that overlap with the same hosts as the + // initial set. + update_data = guide_store()->CreateUpdateDataForFetchedHints( + update_time, + update_time + + optimization_guide::features::StoredFetchedHintsFreshnessDuration()); + + proto::Hint hint; + hint.set_key("domain2.org"); + hint.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint)); + + UpdateFetchedHints(std::move(update_data)); + + // Hint for host.domain2.org should be a fetched hint ("3_" prefix) + // as fetched hints take priority. + std::string host_suffix = "host.domain2.org"; + OptimizationGuideStore::EntryKey hint_entry_key; + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + } + + EXPECT_EQ(hint_entry_key, "3_domain2.org"); + + host_suffix = "subdomain.domain1.org"; + + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + } + + EXPECT_EQ(hint_entry_key, "2_2.0.0_domain1.org"); +} + +TEST_F(OptimizationGuideStoreTest, ClearFetchedHints) { + base::HistogramTester histogram_tester; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + base::Version version("2.0.0"); + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + + proto::Hint hint1; + hint1.set_key("domain1.org"); + hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint1)); + proto::Hint hint2; + hint2.set_key("host.domain2.org"); + hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint2)); + + UpdateComponentHints(std::move(update_data)); + + // Add fetched hints to the store that overlap with the same hosts as the + // initial set. + update_data = guide_store()->CreateUpdateDataForFetchedHints( + update_time, update_time + base::TimeDelta().FromDays(7)); + + proto::Hint fetched_hint1; + fetched_hint1.set_key("domain2.org"); + fetched_hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); + proto::Hint fetched_hint2; + fetched_hint2.set_key("domain3.org"); + fetched_hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); + + UpdateFetchedHints(std::move(update_data)); + + // Hint for host.domain2.org should be a fetched hint ("3_" prefix) + // as fetched hints take priority. + std::string host_suffix = "host.domain2.org"; + OptimizationGuideStore::EntryKey hint_entry_key; + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + } + + EXPECT_EQ(hint_entry_key, "3_domain2.org"); + + host_suffix = "subdomain.domain1.org"; + + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + } + + EXPECT_EQ(hint_entry_key, "2_2.0.0_domain1.org"); + + // Remove the fetched hints from the OptimizationGuideStore. + ClearFetchedHintsFromDatabase(); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.ClearFetchedHints.StoreAvailable", true, 1); + + host_suffix = "domain1.org"; + // Component hint should still exist. + EXPECT_TRUE(guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); + + host_suffix = "domain3.org"; + // Fetched hint should not still exist. + EXPECT_FALSE(guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); + + // Add Components back - newer version. + base::Version version3("3.0.0"); + std::unique_ptr update_data2 = + guide_store()->MaybeCreateUpdateDataForComponentHints(version3); + + ASSERT_TRUE(update_data2); + + proto::Hint new_hint2; + new_hint2.set_key("domain2.org"); + new_hint2.set_key_representation(proto::HOST_SUFFIX); + update_data2->MoveHintIntoUpdateData(std::move(new_hint2)); + + UpdateComponentHints(std::move(update_data2)); + + host_suffix = "host.domain2.org"; + EXPECT_TRUE(guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)); + + update_data = guide_store()->CreateUpdateDataForFetchedHints( + update_time, + update_time + + optimization_guide::features::StoredFetchedHintsFreshnessDuration()); + proto::Hint new_hint; + new_hint.set_key("domain1.org"); + new_hint.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(new_hint)); + + UpdateFetchedHints(std::move(update_data)); + + // Add fetched hints to the store that overlap with the same hosts as the + // initial set. + host_suffix = "subdomain.domain1.org"; + + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + } + + EXPECT_EQ(hint_entry_key, "3_domain1.org"); +} + +TEST_F(OptimizationGuideStoreTest, FetchHintsPurgeExpiredFetchedHints) { + base::HistogramTester histogram_tester; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + base::Version version("2.0.0"); + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + + proto::Hint hint1; + hint1.set_key("domain1.org"); + hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint1)); + proto::Hint hint2; + hint2.set_key("host.domain2.org"); + hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint2)); + + UpdateComponentHints(std::move(update_data)); + + // Add fetched hints to the store that overlap with the same hosts as the + // initial set. + update_data = guide_store()->CreateUpdateDataForFetchedHints( + update_time, update_time + base::TimeDelta().FromDays(7)); + + proto::Hint fetched_hint1; + fetched_hint1.set_key("domain2.org"); + fetched_hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); + proto::Hint fetched_hint2; + fetched_hint2.set_key("domain3.org"); + fetched_hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); + + UpdateFetchedHints(std::move(update_data)); + + // Add expired fetched hints to the store. + update_data = guide_store()->CreateUpdateDataForFetchedHints( + update_time, update_time - base::TimeDelta().FromDays(7)); + + proto::Hint fetched_hint3; + fetched_hint1.set_key("domain4.org"); + fetched_hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); + proto::Hint fetched_hint4; + fetched_hint2.set_key("domain5.org"); + fetched_hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); + + UpdateFetchedHints(std::move(update_data)); + + PurgeExpiredFetchedHints(); + + OptimizationGuideStore::EntryKey hint_entry_key; + EXPECT_FALSE(guide_store()->FindHintEntryKey("domain4.org", &hint_entry_key)); + EXPECT_FALSE(guide_store()->FindHintEntryKey("domain5.org", &hint_entry_key)); + EXPECT_TRUE(guide_store()->FindHintEntryKey("domain2.org", &hint_entry_key)); + EXPECT_TRUE(guide_store()->FindHintEntryKey("domain3.org", &hint_entry_key)); +} + +TEST_F(OptimizationGuideStoreTest, FetchedHintsLoadExpiredHint) { + base::HistogramTester histogram_tester; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t initial_hint_count = 10; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + base::Version version("2.0.0"); + std::unique_ptr update_data = + guide_store()->MaybeCreateUpdateDataForComponentHints( + base::Version(kUpdateComponentVersion)); + ASSERT_TRUE(update_data); + + proto::Hint hint1; + hint1.set_key("domain1.org"); + hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint1)); + proto::Hint hint2; + hint2.set_key("host.domain2.org"); + hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(hint2)); + + UpdateComponentHints(std::move(update_data)); + + // Add fetched hints to the store that expired. + update_data = guide_store()->CreateUpdateDataForFetchedHints( + update_time, update_time - base::TimeDelta().FromDays(10)); + + proto::Hint fetched_hint1; + fetched_hint1.set_key("domain2.org"); + fetched_hint1.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint1)); + proto::Hint fetched_hint2; + fetched_hint2.set_key("domain3.org"); + fetched_hint2.set_key_representation(proto::HOST_SUFFIX); + update_data->MoveHintIntoUpdateData(std::move(fetched_hint2)); + + UpdateFetchedHints(std::move(update_data)); + + // Hint for host.domain2.org should be a fetched hint ("3_" prefix) + // as fetched hints take priority. + std::string host_suffix = "host.domain2.org"; + OptimizationGuideStore::EntryKey hint_entry_key; + if (!guide_store()->FindHintEntryKey(host_suffix, &hint_entry_key)) { + FAIL() << "Hint entry not found for host suffix: " << host_suffix; + } + EXPECT_EQ(hint_entry_key, "3_domain2.org"); + guide_store()->LoadHint( + hint_entry_key, base::BindOnce(&OptimizationGuideStoreTest::OnHintLoaded, + base::Unretained(this))); + + // OnLoadHint callback + db()->GetCallback(true); + + // |hint_entry_key| will be a fetched hint but the entry will be empty. + EXPECT_EQ(last_loaded_entry_key(), hint_entry_key); + EXPECT_FALSE(last_loaded_hint()); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.HintCacheStore.OnLoadHint.FetchedHintExpired", true, + 1); +} + +TEST_F(OptimizationGuideStoreTest, FindPredictionModelEntryKey) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 0); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForPredictionModels(); + ASSERT_TRUE(update_data); + SeedPredictionModelUpdateData(update_data.get(), + proto::OPTIMIZATION_TARGET_UNKNOWN); + SeedPredictionModelUpdateData(update_data.get(), + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD); + UpdatePredictionModels(std::move(update_data)); + + OptimizationGuideStore::EntryKey entry_key; + bool success = guide_store()->FindPredictionModelEntryKey( + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD, &entry_key); + EXPECT_TRUE(success); + EXPECT_EQ(entry_key, "4_1"); +} + +TEST_F(OptimizationGuideStoreTest, + FindEntryKeyMissingForMissingPredictionModel) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 0); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForPredictionModels(); + ASSERT_TRUE(update_data); + SeedPredictionModelUpdateData(update_data.get(), + proto::OPTIMIZATION_TARGET_UNKNOWN); + UpdatePredictionModels(std::move(update_data)); + + OptimizationGuideStore::EntryKey entry_key; + bool success = guide_store()->FindPredictionModelEntryKey( + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD, &entry_key); + EXPECT_FALSE(success); + EXPECT_EQ(entry_key, "4_1"); +} + +TEST_F(OptimizationGuideStoreTest, LoadPredictionModel) { + base::HistogramTester histogram_tester; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 0); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForPredictionModels(); + ASSERT_TRUE(update_data); + SeedPredictionModelUpdateData(update_data.get(), + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD); + UpdatePredictionModels(std::move(update_data)); + + OptimizationGuideStore::EntryKey entry_key; + bool success = guide_store()->FindPredictionModelEntryKey( + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD, &entry_key); + EXPECT_TRUE(success); + + guide_store()->LoadPredictionModel( + entry_key, + base::BindOnce(&OptimizationGuideStoreTest::OnPredictionModelLoaded, + base::Unretained(this))); + // OnPredictionModelLoaded callback + db()->GetCallback(true); + + EXPECT_TRUE(last_loaded_prediction_model()); + + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore.OnLoadCollided", false, 1); +} + +TEST_F(OptimizationGuideStoreTest, LoadPredictionModelOnUnavailableStore) { + base::HistogramTester histogram_tester; + size_t initial_hint_count = 10; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, initial_hint_count); + CreateDatabase(); + InitializeStore(schema_state); + + const OptimizationGuideStore::EntryKey kInvalidEntryKey = "4_2"; + guide_store()->LoadPredictionModel( + kInvalidEntryKey, + base::BindOnce(&OptimizationGuideStoreTest::OnPredictionModelLoaded, + base::Unretained(this))); + // OnPredictionModelLoaded callback + db()->GetCallback(true); + + // Verify that the OnPredictionModelLoaded callback runs when the store is + // unavailable and that the prediction model was correctly set. + EXPECT_FALSE(last_loaded_prediction_model()); + // The load failed because of an unavailable store, not because of a + // collision. + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore.OnLoadCollided", false, 1); +} + +TEST_F(OptimizationGuideStoreTest, LoadPredictionModelWithUpdateInFlight) { + base::HistogramTester histogram_tester; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + SeedInitialData(schema_state, 0); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForPredictionModels(); + ASSERT_TRUE(update_data); + SeedPredictionModelUpdateData(update_data.get(), + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD); + guide_store()->UpdatePredictionModels( + std::move(update_data), + base::BindOnce(&OptimizationGuideStoreTest::OnUpdateStore, + base::Unretained(this))); + + const OptimizationGuideStore::EntryKey kEntryKey = "4_1"; + guide_store()->LoadPredictionModel( + kEntryKey, + base::BindOnce(&OptimizationGuideStoreTest::OnPredictionModelLoaded, + base::Unretained(this))); + + db()->GetCallback(true); + + // Verify that the OnPredictionModelLoaded callback runs when the store is + // unavailable and that the prediction model was correctly set. + EXPECT_FALSE(last_loaded_prediction_model()); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.PredictionModelStore.OnLoadCollided", true, 1); +} + +TEST_F(OptimizationGuideStoreTest, HostModelFeaturesMetadataStored) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 10, update_time, + base::Time().Now() /* host_model_features_update_time */); + CreateDatabase(); + InitializeStore(schema_state); + + ExpectHostModelFeaturesMetadata(update_time); +} + +TEST_F(OptimizationGuideStoreTest, FindEntryKeyForHostModelFeatures) { + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + size_t update_host_model_features_count = 5; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 0, + base::Time().Now() /* host_model_features_update_time */); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForHostModelFeatures( + update_time, update_time + + optimization_guide::features:: + StoredHostModelFeaturesFreshnessDuration()); + ASSERT_TRUE(update_data); + SeedHostModelFeaturesUpdateData(update_data.get(), + update_host_model_features_count); + UpdateHostModelFeatures(std::move(update_data)); + + for (size_t i = 0; i < update_host_model_features_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey entry_key; + bool success = + guide_store()->FindHostModelFeaturesEntryKey(host_suffix, &entry_key); + EXPECT_EQ(success, i < update_host_model_features_count); + } +} + +TEST_F(OptimizationGuideStoreTest, LoadHostModelFeaturesForHost) { + base::HistogramTester histogram_tester; + size_t update_host_model_features_count = 5; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 0, base::Time().Now()); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForHostModelFeatures( + update_time, update_time + + optimization_guide::features:: + StoredHostModelFeaturesFreshnessDuration()); + ASSERT_TRUE(update_data); + SeedHostModelFeaturesUpdateData(update_data.get(), + update_host_model_features_count); + UpdateHostModelFeatures(std::move(update_data)); + + for (size_t i = 0; i < update_host_model_features_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey entry_key; + bool success = + guide_store()->FindHostModelFeaturesEntryKey(host_suffix, &entry_key); + EXPECT_TRUE(success); + + guide_store()->LoadHostModelFeatures( + entry_key, + base::BindOnce(&OptimizationGuideStoreTest::OnHostModelFeaturesLoaded, + base::Unretained(this))); + + // OnPredictionModelLoaded callback + db()->GetCallback(true); + + if (!last_loaded_host_model_features()) { + FAIL() << "Loaded host model features was null for entry key: " + << entry_key; + continue; + } + + EXPECT_EQ(last_loaded_host_model_features()->host(), host_suffix); + } +} + +TEST_F(OptimizationGuideStoreTest, LoadAllHostModelFeatures) { + base::HistogramTester histogram_tester; + size_t update_host_model_features_count = 5; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 0, base::Time().Now()); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForHostModelFeatures( + update_time, update_time + + optimization_guide::features:: + StoredHostModelFeaturesFreshnessDuration()); + ASSERT_TRUE(update_data); + SeedHostModelFeaturesUpdateData(update_data.get(), + update_host_model_features_count); + UpdateHostModelFeatures(std::move(update_data)); + guide_store()->LoadAllHostModelFeatures( + base::BindOnce(&OptimizationGuideStoreTest::OnAllHostModelFeaturesLoaded, + base::Unretained(this))); + + // OnAllHostModelFeaturesLoaded callback + db()->LoadCallback(true); + + std::vector* all_host_model_features = + last_loaded_all_host_model_features(); + EXPECT_TRUE(all_host_model_features); + EXPECT_EQ(update_host_model_features_count, all_host_model_features->size()); + + // Build a list of the hosts that are stored in the store. + base::flat_set hosts = {}; + for (size_t i = 0; i < update_host_model_features_count; i++) + hosts.insert(GetHostSuffix(i)); + + // Make sure all of the hosts of the host model features are returned. + for (const auto& host_model_features : *all_host_model_features) + EXPECT_NE(hosts.find(host_model_features.host()), hosts.end()); +} + +TEST_F(OptimizationGuideStoreTest, ClearHostModelFeatures) { + base::HistogramTester histogram_tester; + size_t update_host_model_features_count = 5; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 0, base::Time().Now()); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForHostModelFeatures( + update_time, update_time + + optimization_guide::features:: + StoredHostModelFeaturesFreshnessDuration()); + ASSERT_TRUE(update_data); + SeedHostModelFeaturesUpdateData(update_data.get(), + update_host_model_features_count); + UpdateHostModelFeatures(std::move(update_data)); + + for (size_t i = 0; i < update_host_model_features_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey entry_key; + EXPECT_TRUE( + guide_store()->FindHostModelFeaturesEntryKey(host_suffix, &entry_key)); + } + + // Remove the host model features from the OptimizationGuideStore. + ClearHostModelFeaturesFromDatabase(); + histogram_tester.ExpectBucketCount( + "OptimizationGuide.ClearHostModelFeatures.StoreAvailable", true, 1); + + for (size_t i = 0; i < update_host_model_features_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey entry_key; + EXPECT_FALSE( + guide_store()->FindHostModelFeaturesEntryKey(host_suffix, &entry_key)); + } +} + +TEST_F(OptimizationGuideStoreTest, PurgeExpiredHostModelFeatures) { + base::HistogramTester histogram_tester; + size_t update_host_model_features_count = 5; + MetadataSchemaState schema_state = MetadataSchemaState::kValid; + base::Time update_time = base::Time().Now(); + SeedInitialData(schema_state, 0, base::Time().Now()); + CreateDatabase(); + InitializeStore(schema_state); + + std::unique_ptr update_data = + guide_store()->CreateUpdateDataForHostModelFeatures( + update_time, update_time - + optimization_guide::features:: + StoredHostModelFeaturesFreshnessDuration()); + ASSERT_TRUE(update_data); + SeedHostModelFeaturesUpdateData(update_data.get(), + update_host_model_features_count); + UpdateHostModelFeatures(std::move(update_data)); + + for (size_t i = 0; i < update_host_model_features_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey entry_key; + EXPECT_TRUE( + guide_store()->FindHostModelFeaturesEntryKey(host_suffix, &entry_key)); + } + + // Remove expired host model features from the opt. guide store. + PurgeExpiredHostModelFeatures(); + + for (size_t i = 0; i < update_host_model_features_count; ++i) { + std::string host_suffix = GetHostSuffix(i); + OptimizationGuideStore::EntryKey entry_key; + EXPECT_FALSE( + guide_store()->FindHostModelFeaturesEntryKey(host_suffix, &entry_key)); + } +} + +} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/optimization_guide_switches.cc b/chromium/components/optimization_guide/optimization_guide_switches.cc index f9fb814e302..11cb8e844bf 100644 --- a/chromium/components/optimization_guide/optimization_guide_switches.cc +++ b/chromium/components/optimization_guide/optimization_guide_switches.cc @@ -25,6 +25,12 @@ const char kHintsProtoOverride[] = "optimization_guide_hints_override"; // hosts. const char kFetchHintsOverride[] = "optimization-guide-fetch-hints-override"; +// Overrides scheduling and time delays for fetching prediction models and host +// model features. This causes a prediction model and host model features fetch +// immediately on start up. +const char kFetchModelsAndHostModelFeaturesOverrideTimer[] = + "optimization-guide-fetch-models-and-features-override"; + // Overrides the hints fetch scheduling and delay, causing a hints fetch // immediately on start up using the TopHostProvider. This is meant for testing. const char kFetchHintsOverrideTimer[] = @@ -32,28 +38,46 @@ const char kFetchHintsOverrideTimer[] = // Overrides the Optimization Guide Service URL that the HintsFetcher will // request remote hints from. -const char kOptimizationGuideServiceURL[] = "optimization-guide-service-url"; +const char kOptimizationGuideServiceGetHintsURL[] = + "optimization-guide-service-get-hosts-url"; + +// Overrides the Optimization Guide Service URL that the PredictionModelFetcher +// will request remote models and host features from. +const char kOptimizationGuideServiceGetModelsURL[] = + "optimization-guide-service-get-models-url"; // Overrides the Optimization Guide Service API Key for remote requests to be // made. const char kOptimizationGuideServiceAPIKey[] = "optimization-guide-service-api-key"; -// Purges the hint cache store on startup, so that it's guaranteed to be using -// fresh data. -const char kPurgeHintCacheStore[] = "purge_hint_cache_store"; +// Purges the store containing fetched and component hints on startup, so that +// it's guaranteed to be using fresh data. +const char kPurgeHintsStore[] = "purge-optimization-guide-store"; + +// Purges the store containing prediction medels and host model features on +// startup, so that it's guaranteed to be using fresh data. +const char kPurgeModelAndFeaturesStore[] = "purge-model-and-features-store"; const char kDisableFetchingHintsAtNavigationStartForTesting[] = "disable-fetching-hints-at-navigation-start"; +const char kDisableCheckingUserPermissionsForTesting[] = + "disable-checking-optimization-guide-user-permissions"; + bool IsHintComponentProcessingDisabled() { return base::CommandLine::ForCurrentProcess()->HasSwitch(kHintsProtoOverride); } -bool ShouldPurgeHintCacheStoreOnStartup() { +bool ShouldPurgeOptimizationGuideStoreOnStartup() { base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess(); return cmd_line->HasSwitch(kHintsProtoOverride) || - cmd_line->HasSwitch(kPurgeHintCacheStore); + cmd_line->HasSwitch(kPurgeHintsStore); +} + +bool ShouldPurgeModelAndFeaturesStoreOnStartup() { + base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess(); + return cmd_line->HasSwitch(kPurgeModelAndFeaturesStore); } // Parses a list of hosts to have hints fetched for. This overrides scheduling @@ -83,6 +107,11 @@ bool ShouldOverrideFetchHintsTimer() { kFetchHintsOverrideTimer); } +bool ShouldOverrideFetchModelsAndFeaturesTimer() { + return base::CommandLine::ForCurrentProcess()->HasSwitch( + kFetchModelsAndHostModelFeaturesOverrideTimer); +} + std::unique_ptr ParseComponentConfigFromCommandLine() { base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess(); @@ -111,7 +140,12 @@ ParseComponentConfigFromCommandLine() { bool DisableFetchingHintsAtNavigationStartForTesting() { base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); return command_line->HasSwitch( - switches::kDisableFetchingHintsAtNavigationStartForTesting); + kDisableFetchingHintsAtNavigationStartForTesting); +} + +bool ShouldOverrideCheckingUserPermissionsToFetchHintsForTesting() { + base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); + return command_line->HasSwitch(kDisableCheckingUserPermissionsForTesting); } } // namespace switches diff --git a/chromium/components/optimization_guide/optimization_guide_switches.h b/chromium/components/optimization_guide/optimization_guide_switches.h index ce18d374222..6460a3470a3 100644 --- a/chromium/components/optimization_guide/optimization_guide_switches.h +++ b/chromium/components/optimization_guide/optimization_guide_switches.h @@ -21,20 +21,28 @@ namespace switches { extern const char kHintsProtoOverride[]; extern const char kFetchHintsOverride[]; extern const char kFetchHintsOverrideTimer[]; -extern const char kOptimizationGuideServiceURL[]; +extern const char kFetchModelsAndHostModelFeaturesOverrideTimer[]; +extern const char kOptimizationGuideServiceGetHintsURL[]; +extern const char kOptimizationGuideServiceGetModelsURL[]; extern const char kOptimizationGuideServiceAPIKey[]; -extern const char kPurgeHintCacheStore[]; +extern const char kPurgeHintsStore[]; +extern const char kPurgeModelAndFeaturesStore[]; extern const char kDisableFetchingHintsAtNavigationStartForTesting[]; +extern const char kDisableCheckingUserPermissionsForTesting[]; // Returns whether the hint component should be processed. // Available hint components are only processed if a proto override isn't being // used; otherwise, the hints from the proto override are used instead. bool IsHintComponentProcessingDisabled(); -// Returns whether hints should be purged during startup if the explicit purge -// switch exists or if a proto override is being used--in which case the hints -// need to come from the override instead. -bool ShouldPurgeHintCacheStoreOnStartup(); +// Returns whether all entries within the store should be purged during startup +// if the explicit purge switch exists or if a proto override is being used, in +// which case the hints need to come from the override instead. +bool ShouldPurgeOptimizationGuideStoreOnStartup(); + +// Returns whether all entries within the store should be purged during startup +// if the explicit purge switch exists. +bool ShouldPurgeModelAndFeaturesStoreOnStartup(); // Parses a list of hosts to have hints fetched for. This overrides scheduling // of the first hints fetch and forces it to occur immediately. If no hosts are @@ -45,6 +53,10 @@ ParseHintsFetchOverrideFromCommandLine(); // Whether the hints fetcher timer should be overridden. bool ShouldOverrideFetchHintsTimer(); +// Whether the prediction model and host model features fetcher timer should be +// overridden. +bool ShouldOverrideFetchModelsAndFeaturesTimer(); + // Attempts to parse a base64 encoded Optimization Guide Configuration proto // from the command line. If no proto is given or if it is encoded incorrectly, // nullptr is returned. @@ -55,6 +67,11 @@ ParseComponentConfigFromCommandLine(); // start should be disabled. Returns true only in tests. bool DisableFetchingHintsAtNavigationStartForTesting(); +// Returns true if checking of the user's permissions to fetch hints from the +// remote Optimization Guide Service should be ignored. Returns true only in +// tests. +bool ShouldOverrideCheckingUserPermissionsToFetchHintsForTesting(); + } // namespace switches } // namespace optimization_guide diff --git a/chromium/components/optimization_guide/proto/hint_cache.proto b/chromium/components/optimization_guide/proto/hint_cache.proto index 6cfc52246e9..255a74d42c8 100644 --- a/chromium/components/optimization_guide/proto/hint_cache.proto +++ b/chromium/components/optimization_guide/proto/hint_cache.proto @@ -9,6 +9,7 @@ option optimize_for = LITE_RUNTIME; package optimization_guide.proto; import "hints.proto"; +import "models.proto"; // The types of StoreEntries. // @@ -28,6 +29,13 @@ enum StoreEntryType { // fetched from the remote Optimization Guide Service. |update_time_secs| // should be provided. FETCHED_HINT = 3; + // StoreEntryType when storing a prediction model entry, holds a prediction + // model provided by the remote Optimization Guide Service. + PREDICTION_MODEL = 4; + // StoreEntryType when storing a host model features entry, holds the model + // features keyed provided by host from the remote Optimization Guide + // Service. |update_time_secs| should be provided. + HOST_MODEL_FEATURES = 5; } message StoreEntry { @@ -37,12 +45,18 @@ message StoreEntry { optional string version = 1; // The actual hint data. optional Hint hint = 2; - // Time when top host fetched hints are still usable but update should - // be requested. This is set on the fetched metadata entry. + // Time when top host fetched hints are still usable but update should be + // requested. This is set on the fetched metadata entry and host model feature + // metadata entries. optional int64 update_time_secs = 3; // The type of entry stored. optional StoreEntryType entry_type = 4; - // Expiry time for this Hint entry (i.e., when hint is no longer usable). - // This is set on OnePlatform Hint entries. + // Expiry time for this Hint entry (i.e., when hint is no longer usable). This + // is set on OnePlatform Hint entries, as well as PredictionModel and + // HostModelFeatures entries. optional int64 expiry_time_secs = 5; + // The actual PredictionModel data. + optional PredictionModel prediction_model = 6; + // The actual HostModelFeature data. + optional HostModelFeatures host_model_features = 7; } diff --git a/chromium/components/optimization_guide/proto/models.proto b/chromium/components/optimization_guide/proto/models.proto index 98d66fd84e9..ed062c4da38 100644 --- a/chromium/components/optimization_guide/proto/models.proto +++ b/chromium/components/optimization_guide/proto/models.proto @@ -199,6 +199,10 @@ message ModelInfo { repeated ClientModelFeature supported_model_features = 3; // The set of model types the requesting client can use to make predictions. repeated ModelType supported_model_types = 4; + // The set of host model features that are referenced by the model. + // + // Note that this should only be populated if part of the response. + repeated string supported_host_model_features = 5; } // The scenarios for which the optimization guide has models for. diff --git a/chromium/components/optimization_guide/store_update_data.cc b/chromium/components/optimization_guide/store_update_data.cc new file mode 100644 index 00000000000..6bdd5f9ed7b --- /dev/null +++ b/chromium/components/optimization_guide/store_update_data.cc @@ -0,0 +1,211 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/optimization_guide/store_update_data.h" + +#include "base/strings/string_number_conversions.h" +#include "components/optimization_guide/optimization_guide_store.h" +#include "components/optimization_guide/proto/hint_cache.pb.h" +#include "components/optimization_guide/proto/hints.pb.h" +#include "components/optimization_guide/proto/models.pb.h" + +namespace optimization_guide { + +// static +std::unique_ptr +StoreUpdateData::CreateComponentStoreUpdateData( + const base::Version& component_version) { + std::unique_ptr update_data(new StoreUpdateData( + base::Optional(component_version), + base::Optional(), base::Optional())); + return update_data; +} + +// static +std::unique_ptr StoreUpdateData::CreateFetchedStoreUpdateData( + base::Time fetch_update_time, + base::Time expiry_time) { + std::unique_ptr update_data( + new StoreUpdateData(base::Optional(), + base::Optional(fetch_update_time), + base::Optional(expiry_time))); + return update_data; +} + +// static +std::unique_ptr +StoreUpdateData::CreatePredictionModelStoreUpdateData() { + std::unique_ptr prediction_model_update_data( + new StoreUpdateData()); + return prediction_model_update_data; +} + +// static +std::unique_ptr +StoreUpdateData::CreateHostModelFeaturesStoreUpdateData( + base::Time host_model_features_update_time, + base::Time expiry_time) { + std::unique_ptr host_model_features_update_data( + new StoreUpdateData(host_model_features_update_time, expiry_time)); + return host_model_features_update_data; +} + +StoreUpdateData::StoreUpdateData(base::Time host_model_features_update_time, + base::Time expiry_time) + : update_time_(host_model_features_update_time), + expiry_time_(expiry_time), + entries_to_save_(std::make_unique()) { + entry_key_prefix_ = + OptimizationGuideStore::GetHostModelFeaturesEntryKeyPrefix(); + proto::StoreEntry metadata_host_model_features_entry; + metadata_host_model_features_entry.set_entry_type( + static_cast( + OptimizationGuideStore::StoreEntryType::kMetadata)); + metadata_host_model_features_entry.set_update_time_secs( + host_model_features_update_time.ToDeltaSinceWindowsEpoch().InSeconds()); + entries_to_save_->emplace_back( + OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kHostModelFeatures), + std::move(metadata_host_model_features_entry)); + + // |this| may be modified on another thread after construction but all + // future modifications, from that call forward, must be made on the same + // thread. + DETACH_FROM_SEQUENCE(sequence_checker_); +} + +StoreUpdateData::StoreUpdateData() + : entries_to_save_(std::make_unique()) { + entry_key_prefix_ = + OptimizationGuideStore::GetPredictionModelEntryKeyPrefix(); + + // |this| may be modified on another thread after construction but all + // future modifications, from that call forward, must be made on the same + // thread. + DETACH_FROM_SEQUENCE(sequence_checker_); +} + +StoreUpdateData::StoreUpdateData( + base::Optional component_version, + base::Optional fetch_update_time, + base::Optional expiry_time) + : component_version_(component_version), + update_time_(fetch_update_time), + expiry_time_(expiry_time), + entries_to_save_(std::make_unique()) { + DCHECK_NE(!component_version_, !update_time_); + + if (component_version_.has_value()) { + entry_key_prefix_ = OptimizationGuideStore::GetComponentHintEntryKeyPrefix( + *component_version_); + + // Add a component metadata entry for the component's version. + proto::StoreEntry metadata_component_entry; + + metadata_component_entry.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kMetadata)); + metadata_component_entry.set_version(component_version_->GetString()); + entries_to_save_->emplace_back( + OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kComponent), + std::move(metadata_component_entry)); + } else if (update_time_.has_value()) { + entry_key_prefix_ = OptimizationGuideStore::GetFetchedHintEntryKeyPrefix(); + proto::StoreEntry metadata_fetched_entry; + metadata_fetched_entry.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kMetadata)); + metadata_fetched_entry.set_update_time_secs( + update_time_->ToDeltaSinceWindowsEpoch().InSeconds()); + entries_to_save_->emplace_back( + OptimizationGuideStore::GetMetadataTypeEntryKey( + OptimizationGuideStore::MetadataType::kFetched), + std::move(metadata_fetched_entry)); + } else { + NOTREACHED(); + } + // |this| may be modified on another thread after construction but all + // future modifications, from that call forward, must be made on the same + // thread. + DETACH_FROM_SEQUENCE(sequence_checker_); +} + +StoreUpdateData::~StoreUpdateData() {} + +void StoreUpdateData::MoveHintIntoUpdateData(proto::Hint&& hint) { + // All future modifications must be made by the same thread. Note, |this| may + // have been constructed on another thread. + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!entry_key_prefix_.empty()); + + // To avoid any unnecessary copying, the hint is moved into proto::StoreEntry. + OptimizationGuideStore::EntryKey hint_entry_key = + entry_key_prefix_ + hint.key(); + proto::StoreEntry entry_proto; + if (component_version()) { + entry_proto.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kComponentHint)); + } else if (update_time()) { + DCHECK(expiry_time()); + entry_proto.set_expiry_time_secs( + expiry_time_->ToDeltaSinceWindowsEpoch().InSeconds()); + entry_proto.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kFetchedHint)); + } + entry_proto.set_allocated_hint(new proto::Hint(std::move(hint))); + entries_to_save_->emplace_back(std::move(hint_entry_key), + std::move(entry_proto)); +} + +void StoreUpdateData::CopyHostModelFeaturesIntoUpdateData( + const proto::HostModelFeatures& host_model_features) { + // All future modifications must be made by the same thread. Note, |this| may + // have been constructed on another thread. + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!entry_key_prefix_.empty()); + DCHECK(expiry_time()); + + // To avoid any unnecessary copying, the host model feature data is moved into + // proto::StoreEntry. + OptimizationGuideStore::EntryKey host_model_features_entry_key = + entry_key_prefix_ + host_model_features.host(); + proto::StoreEntry entry_proto; + entry_proto.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kHostModelFeatures)); + entry_proto.set_expiry_time_secs( + expiry_time_->ToDeltaSinceWindowsEpoch().InSeconds()); + entry_proto.mutable_host_model_features()->CopyFrom(host_model_features); + entries_to_save_->emplace_back(std::move(host_model_features_entry_key), + std::move(entry_proto)); +} + +void StoreUpdateData::CopyPredictionModelIntoUpdateData( + const proto::PredictionModel& prediction_model) { + // All future modifications must be made by the same thread. Note, |this| may + // have been constructed on another thread. + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!entry_key_prefix_.empty()); + + // To avoid any unnecessary copying, the prediction model is moved into + // proto::StoreEntry. + OptimizationGuideStore::EntryKey prediction_model_entry_key = + entry_key_prefix_ + + base::NumberToString(static_cast( + prediction_model.model_info().optimization_target())); + proto::StoreEntry entry_proto; + entry_proto.set_entry_type(static_cast( + OptimizationGuideStore::StoreEntryType::kPredictionModel)); + entry_proto.mutable_prediction_model()->CopyFrom(prediction_model); + entries_to_save_->emplace_back(std::move(prediction_model_entry_key), + std::move(entry_proto)); +} + +std::unique_ptr StoreUpdateData::TakeUpdateEntries() { + // TakeUpdateEntries is not be sequence checked as it only gives up ownership + // of the entries_to_save_ and does not modify any state. + DCHECK(entries_to_save_); + + return std::move(entries_to_save_); +} + +} // namespace optimization_guide diff --git a/chromium/components/optimization_guide/store_update_data.h b/chromium/components/optimization_guide/store_update_data.h new file mode 100644 index 00000000000..c175fbbc96d --- /dev/null +++ b/chromium/components/optimization_guide/store_update_data.h @@ -0,0 +1,110 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OPTIMIZATION_GUIDE_STORE_UPDATE_DATA_H_ +#define COMPONENTS_OPTIMIZATION_GUIDE_STORE_UPDATE_DATA_H_ + +#include + +#include "base/macros.h" +#include "base/optional.h" +#include "base/sequence_checker.h" +#include "base/time/time.h" +#include "base/version.h" +#include "components/leveldb_proto/public/proto_database.h" + +namespace optimization_guide { +namespace proto { +class Hint; +class HostModelFeatures; +class PredictionModel; +class StoreEntry; +} // namespace proto + +using EntryVector = + leveldb_proto::ProtoDatabase::KeyEntryVector; + +// Holds hint, prediction model, or host model features data for updating the +// OptimizationGuideStore. +class StoreUpdateData { + public: + ~StoreUpdateData(); + + // Creates an update data object for a component hint update. + static std::unique_ptr CreateComponentStoreUpdateData( + const base::Version& component_version); + + // Creates an update data object for a fetched hint update. + static std::unique_ptr CreateFetchedStoreUpdateData( + base::Time fetch_update_time, + base::Time expiry_time); + + // Creates an update data object for a prediction model update. + static std::unique_ptr + CreatePredictionModelStoreUpdateData(); + + // Creates an update data object for a host model features update. + static std::unique_ptr + CreateHostModelFeaturesStoreUpdateData( + base::Time host_model_features_update_time, + base::Time expiry_time); + + // Returns the component version of a component hint update. + const base::Optional component_version() const { + return component_version_; + } + + // Returns the next update time for the entries in the store update. + const base::Optional update_time() const { return update_time_; } + + // Returns the expiry time for the hints in a fetched hint update. + const base::Optional expiry_time() const { return expiry_time_; } + + // Moves |hint| into this update data. After MoveHintIntoUpdateData() is + // called, |hint| is no longer valid. + void MoveHintIntoUpdateData(proto::Hint&& hint); + + // Copies |host_model_features| into this update data. + void CopyHostModelFeaturesIntoUpdateData( + const proto::HostModelFeatures& host_model_features); + + // Copies |prediction_model| into this update data. + void CopyPredictionModelIntoUpdateData( + const proto::PredictionModel& prediction_model); + + // Returns the store entry updates along with ownership to them. + std::unique_ptr TakeUpdateEntries(); + + private: + StoreUpdateData(base::Optional component_version, + base::Optional fetch_update_time, + base::Optional expiry_time); + StoreUpdateData(base::Time host_model_features_update_time, + base::Time expiry_time); + StoreUpdateData(); + + // The component version of the update data for a component update. + base::Optional component_version_; + + // The time when the entries in this update need to be updated. + base::Optional update_time_; + + // The time when entries in this update expire. + base::Optional expiry_time_; + + // The prefix to add to the key of every store entry. It is set + // during construction for appropriate type of update. + std::string entry_key_prefix_; + + // The vector of entries to save. + std::unique_ptr entries_to_save_; + + SEQUENCE_CHECKER(sequence_checker_); + + DISALLOW_COPY_AND_ASSIGN(StoreUpdateData); +}; + +} // namespace optimization_guide + +#endif // COMPONENTS_OPTIMIZATION_GUIDE_STORE_UPDATE_DATA_H_ diff --git a/chromium/components/optimization_guide/store_update_data_unittest.cc b/chromium/components/optimization_guide/store_update_data_unittest.cc new file mode 100644 index 00000000000..2d2b91f4c3b --- /dev/null +++ b/chromium/components/optimization_guide/store_update_data_unittest.cc @@ -0,0 +1,121 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/optimization_guide/store_update_data.h" + +#include +#include + +#include "base/macros.h" +#include "base/time/time.h" +#include "base/version.h" +#include "components/optimization_guide/optimization_guide_features.h" +#include "components/optimization_guide/proto/hint_cache.pb.h" +#include "components/optimization_guide/proto/hints.pb.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace optimization_guide { + +namespace { + +TEST(StoreUpdateDataTest, BuildComponentStoreUpdateData) { + // Verify creating a Component Hint update package. + base::Version v1("1.2.3.4"); + proto::Hint hint1; + hint1.set_key("foo.org"); + hint1.set_key_representation(proto::HOST_SUFFIX); + proto::PageHint* page_hint1 = hint1.add_page_hints(); + page_hint1->set_page_pattern("slowpage"); + proto::Hint hint2; + hint2.set_key("bar.com"); + hint2.set_key_representation(proto::HOST_SUFFIX); + proto::PageHint* page_hint2 = hint2.add_page_hints(); + page_hint2->set_page_pattern("slowpagealso"); + + std::unique_ptr component_update = + StoreUpdateData::CreateComponentStoreUpdateData(v1); + component_update->MoveHintIntoUpdateData(std::move(hint1)); + component_update->MoveHintIntoUpdateData(std::move(hint2)); + EXPECT_TRUE(component_update->component_version().has_value()); + EXPECT_FALSE(component_update->update_time().has_value()); + EXPECT_EQ(v1, *component_update->component_version()); + // Verify there are 3 store entries: 1 for the metadata entry plus + // the 2 added hint entries. + EXPECT_EQ(3ul, component_update->TakeUpdateEntries()->size()); +} + +TEST(StoreUpdateDataTest, BuildFetchUpdateData) { + // Verify creating a Fetched Hint update package. + base::Time update_time = base::Time::Now(); + proto::Hint hint1; + hint1.set_key("foo.org"); + hint1.set_key_representation(proto::HOST_SUFFIX); + proto::PageHint* page_hint1 = hint1.add_page_hints(); + page_hint1->set_page_pattern("slowpage"); + + std::unique_ptr fetch_update = + StoreUpdateData::CreateFetchedStoreUpdateData( + update_time, update_time + optimization_guide::features:: + StoredFetchedHintsFreshnessDuration()); + fetch_update->MoveHintIntoUpdateData(std::move(hint1)); + EXPECT_FALSE(fetch_update->component_version().has_value()); + EXPECT_TRUE(fetch_update->update_time().has_value()); + EXPECT_EQ(update_time, *fetch_update->update_time()); + // Verify there are 2 store entries: 1 for the metadata entry plus + // the 1 added hint entries. + EXPECT_EQ(2ul, fetch_update->TakeUpdateEntries()->size()); +} + +TEST(StoreUpdateDataTest, BuildPredictionModelUpdateData) { + // Verify creating a Prediction Model update data. + proto::PredictionModel prediction_model; + + proto::ModelInfo* model_info = prediction_model.mutable_model_info(); + model_info->set_version(1); + model_info->add_supported_model_features( + proto::CLIENT_MODEL_FEATURE_EFFECTIVE_CONNECTION_TYPE); + model_info->set_optimization_target( + proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD); + model_info->add_supported_model_types( + proto::ModelType::MODEL_TYPE_DECISION_TREE); + + std::unique_ptr prediction_model_update = + StoreUpdateData::CreatePredictionModelStoreUpdateData(); + prediction_model_update->CopyPredictionModelIntoUpdateData(prediction_model); + EXPECT_FALSE(prediction_model_update->component_version().has_value()); + EXPECT_FALSE(prediction_model_update->update_time().has_value()); + // Verify there is 1 store entry. + EXPECT_EQ(1ul, prediction_model_update->TakeUpdateEntries()->size()); +} + +TEST(StoreUpdateDataTest, BuildHostModelFeaturesUpdateData) { + // Verify creating a Prediction Model update data. + base::Time host_model_features_update_time = base::Time::Now(); + + proto::HostModelFeatures host_model_features; + host_model_features.set_host("foo.com"); + proto::ModelFeature* model_feature = host_model_features.add_model_features(); + model_feature->set_feature_name("host_feat1"); + model_feature->set_double_value(2.0); + + std::unique_ptr host_model_features_update = + StoreUpdateData::CreateHostModelFeaturesStoreUpdateData( + host_model_features_update_time, + host_model_features_update_time + + optimization_guide::features:: + StoredHostModelFeaturesFreshnessDuration()); + host_model_features_update->CopyHostModelFeaturesIntoUpdateData( + std::move(host_model_features)); + EXPECT_FALSE(host_model_features_update->component_version().has_value()); + EXPECT_TRUE(host_model_features_update->update_time().has_value()); + EXPECT_EQ(host_model_features_update_time, + *host_model_features_update->update_time()); + // Verify there are 2 store entries, 1 for the metadata entry and 1 for the + // added host model features entry. + EXPECT_EQ(2ul, host_model_features_update->TakeUpdateEntries()->size()); +} + +} // namespace + +} // namespace optimization_guide -- cgit v1.2.1