diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-10-24 11:30:15 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-10-30 12:56:19 +0000 |
commit | 6036726eb981b6c4b42047513b9d3f4ac865daac (patch) | |
tree | 673593e70678e7789766d1f732eb51f613a2703b /chromium/components/feed | |
parent | 466052c4e7c052268fd931888cd58961da94c586 (diff) | |
download | qtwebengine-chromium-6036726eb981b6c4b42047513b9d3f4ac865daac.tar.gz |
BASELINE: Update Chromium to 70.0.3538.78
Change-Id: Ie634710bf039e26c1957f4ae45e101bd4c434ae7
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/feed')
48 files changed, 2533 insertions, 1321 deletions
diff --git a/chromium/components/feed/DEPS b/chromium/components/feed/DEPS index 8bbbd796747..d58e934df74 100644 --- a/chromium/components/feed/DEPS +++ b/chromium/components/feed/DEPS @@ -3,6 +3,7 @@ include_rules = [ "+components/image_fetcher", "+components/keyed_service/core", "+components/leveldb_proto", + "+components/offline_pages", "+components/prefs", "+components/variations", "+components/version_info", diff --git a/chromium/components/feed/OWNERS b/chromium/components/feed/OWNERS index 7f1733c213f..90e498d6f46 100644 --- a/chromium/components/feed/OWNERS +++ b/chromium/components/feed/OWNERS @@ -1,6 +1,7 @@ fgorski@chromium.org pavely@chromium.org pnoland@chromium.org +skym@chromium.org zea@chromium.org per-file *Test.java=aluo@chromium.org diff --git a/chromium/components/feed/content/BUILD.gn b/chromium/components/feed/content/BUILD.gn new file mode 100644 index 00000000000..3a49db2d9f5 --- /dev/null +++ b/chromium/components/feed/content/BUILD.gn @@ -0,0 +1,27 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +if (is_android) { + import("//build/config/android/rules.gni") +} + +source_set("feed_content") { + sources = [ + "feed_host_service.cc", + "feed_host_service.h", + "feed_offline_host.cc", + "feed_offline_host.h", + ] + + public_deps = [ + "//base", + "//components/feed/core:feed_core", + "//components/keyed_service/core", + ] + + deps = [ + "//components/offline_pages/core", + "//components/offline_pages/core/prefetch", + ] +} diff --git a/chromium/components/feed/core/feed_host_service.cc b/chromium/components/feed/content/feed_host_service.cc index fe202d6ff4c..d8beba3539f 100644 --- a/chromium/components/feed/core/feed_host_service.cc +++ b/chromium/components/feed/content/feed_host_service.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/feed_host_service.h" +#include "components/feed/content/feed_host_service.h" #include <utility> @@ -12,11 +12,15 @@ FeedHostService::FeedHostService( std::unique_ptr<FeedImageManager> image_manager, std::unique_ptr<FeedNetworkingHost> networking_host, std::unique_ptr<FeedSchedulerHost> scheduler_host, - std::unique_ptr<FeedStorageDatabase> storage_database) + std::unique_ptr<FeedContentDatabase> content_database, + std::unique_ptr<FeedJournalDatabase> journal_database, + std::unique_ptr<FeedOfflineHost> offline_host) : image_manager_(std::move(image_manager)), networking_host_(std::move(networking_host)), scheduler_host_(std::move(scheduler_host)), - storage_database_(std::move(storage_database)) {} + content_database_(std::move(content_database)), + journal_database_(std::move(journal_database)), + offline_host_(std::move(offline_host)) {} FeedHostService::~FeedHostService() = default; @@ -32,8 +36,16 @@ FeedSchedulerHost* FeedHostService::GetSchedulerHost() { return scheduler_host_.get(); } -FeedStorageDatabase* FeedHostService::GetStorageDatabase() { - return storage_database_.get(); +FeedContentDatabase* FeedHostService::GetContentDatabase() { + return content_database_.get(); +} + +FeedJournalDatabase* FeedHostService::GetJournalDatabase() { + return journal_database_.get(); +} + +FeedOfflineHost* FeedHostService::GetOfflineHost() { + return offline_host_.get(); } } // namespace feed diff --git a/chromium/components/feed/core/feed_host_service.h b/chromium/components/feed/content/feed_host_service.h index 223f668750a..53ec04293f6 100644 --- a/chromium/components/feed/core/feed_host_service.h +++ b/chromium/components/feed/content/feed_host_service.h @@ -2,16 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_FEED_CORE_FEED_HOST_SERVICE_H_ -#define COMPONENTS_FEED_CORE_FEED_HOST_SERVICE_H_ +#ifndef COMPONENTS_FEED_CONTENT_FEED_HOST_SERVICE_H_ +#define COMPONENTS_FEED_CONTENT_FEED_HOST_SERVICE_H_ #include <memory> #include "base/macros.h" +#include "components/feed/content/feed_offline_host.h" +#include "components/feed/core/feed_content_database.h" #include "components/feed/core/feed_image_manager.h" +#include "components/feed/core/feed_journal_database.h" #include "components/feed/core/feed_networking_host.h" #include "components/feed/core/feed_scheduler_host.h" -#include "components/feed/core/feed_storage_database.h" #include "components/keyed_service/core/keyed_service.h" namespace feed { @@ -26,23 +28,29 @@ class FeedHostService : public KeyedService { FeedHostService(std::unique_ptr<FeedImageManager> image_manager, std::unique_ptr<FeedNetworkingHost> networking_host, std::unique_ptr<FeedSchedulerHost> scheduler_host, - std::unique_ptr<FeedStorageDatabase> storage_database); + std::unique_ptr<FeedContentDatabase> content_database, + std::unique_ptr<FeedJournalDatabase> journal_database, + std::unique_ptr<FeedOfflineHost> offline_host); ~FeedHostService() override; FeedImageManager* GetImageManager(); FeedNetworkingHost* GetNetworkingHost(); FeedSchedulerHost* GetSchedulerHost(); - FeedStorageDatabase* GetStorageDatabase(); + FeedContentDatabase* GetContentDatabase(); + FeedJournalDatabase* GetJournalDatabase(); + FeedOfflineHost* GetOfflineHost(); private: std::unique_ptr<FeedImageManager> image_manager_; std::unique_ptr<FeedNetworkingHost> networking_host_; std::unique_ptr<FeedSchedulerHost> scheduler_host_; - std::unique_ptr<FeedStorageDatabase> storage_database_; + std::unique_ptr<FeedContentDatabase> content_database_; + std::unique_ptr<FeedJournalDatabase> journal_database_; + std::unique_ptr<FeedOfflineHost> offline_host_; DISALLOW_COPY_AND_ASSIGN(FeedHostService); }; } // namespace feed -#endif // COMPONENTS_FEED_CORE_FEED_HOST_SERVICE_H_ +#endif // COMPONENTS_FEED_CONTENT_FEED_HOST_SERVICE_H_ diff --git a/chromium/components/feed/content/feed_offline_host.cc b/chromium/components/feed/content/feed_offline_host.cc new file mode 100644 index 00000000000..72c53dba8e8 --- /dev/null +++ b/chromium/components/feed/content/feed_offline_host.cc @@ -0,0 +1,54 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/content/feed_offline_host.h" + +namespace feed { + +FeedOfflineHost::FeedOfflineHost( + offline_pages::OfflinePageModel* offline_page_model, + offline_pages::PrefetchService* prefetch_service, + FeedSchedulerHost* feed_scheduler_host) + : offline_page_model_(offline_page_model), + prefetch_service_(prefetch_service), + feed_scheduler_host_(feed_scheduler_host), + weak_ptr_factory_(this) { + DCHECK(offline_page_model_); + DCHECK(prefetch_service_); + DCHECK(feed_scheduler_host_); +} + +FeedOfflineHost::~FeedOfflineHost() = default; + +void FeedOfflineHost::GetCurrentArticleSuggestions( + offline_pages::SuggestionsProvider::SuggestionCallback + suggestions_callback) { + // TODO(skym): Call into bridge callback. +} + +void FeedOfflineHost::ReportArticleListViewed() { + // TODO(skym): Call FeedSchedulerHost::OnSuggestionsShown(). +} + +void FeedOfflineHost::ReportArticleViewed(GURL article_url) { + // TODO(skym): Call FeedSchedulerHost::OnSuggestionConsumed(). +} + +void FeedOfflineHost::OfflinePageModelLoaded( + offline_pages::OfflinePageModel* model) { + // Ignored. +} + +void FeedOfflineHost::OfflinePageAdded( + offline_pages::OfflinePageModel* model, + const offline_pages::OfflinePageItem& added_page) { + // TODO(skym): Call into bridge callback. +} + +void FeedOfflineHost::OfflinePageDeleted( + const offline_pages::OfflinePageModel::DeletedPageInfo& page_info) { + // TODO(skym): Call into bridge callback. +} + +} // namespace feed diff --git a/chromium/components/feed/content/feed_offline_host.h b/chromium/components/feed/content/feed_offline_host.h new file mode 100644 index 00000000000..2af25b69180 --- /dev/null +++ b/chromium/components/feed/content/feed_offline_host.h @@ -0,0 +1,67 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CONTENT_FEED_OFFLINE_HOST_H_ +#define COMPONENTS_FEED_CONTENT_FEED_OFFLINE_HOST_H_ + +#include "base/callback.h" +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "components/offline_pages/core/offline_page_model.h" +#include "components/offline_pages/core/prefetch/suggestions_provider.h" + +class GURL; + +namespace offline_pages { +class PrefetchService; +} // namespace offline_pages + +namespace feed { + +class FeedSchedulerHost; + +// Responsible for wiring up connections for Feed operations pertaining to +// articles that can be loaded from Offline Pages component. Most significantly +// this class connects Prefetch and the Feed, and tracks offlined articles the +// Feed may have badged for this user. This knowledge is later used when Feed +// articles are opened to populate load params. +class FeedOfflineHost : public offline_pages::SuggestionsProvider, + public offline_pages::OfflinePageModel::Observer { + public: + FeedOfflineHost(offline_pages::OfflinePageModel* offline_page_model, + offline_pages::PrefetchService* prefetch_service, + FeedSchedulerHost* feed_scheduler_host); + ~FeedOfflineHost() override; + + // offline_pages::SuggestionsProvider: + void GetCurrentArticleSuggestions( + offline_pages::SuggestionsProvider::SuggestionCallback + suggestions_callback) override; + void ReportArticleListViewed() override; + void ReportArticleViewed(GURL article_url) override; + + // offline_pages::OfflinePageModel::Observer: + void OfflinePageModelLoaded(offline_pages::OfflinePageModel* model) override; + void OfflinePageAdded( + offline_pages::OfflinePageModel* model, + const offline_pages::OfflinePageItem& added_page) override; + void OfflinePageDeleted( + const offline_pages::OfflinePageModel::DeletedPageInfo& page_info) + override; + + private: + // The following objects all outlive us, so it is safe to hold raw pointers to + // them. This is guaranteed by the FeedHostServiceFactory. + offline_pages::OfflinePageModel* offline_page_model_; + offline_pages::PrefetchService* prefetch_service_; + FeedSchedulerHost* feed_scheduler_host_; + + base::WeakPtrFactory<FeedOfflineHost> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(FeedOfflineHost); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CONTENT_FEED_OFFLINE_HOST_H_ diff --git a/chromium/components/feed/core/BUILD.gn b/chromium/components/feed/core/BUILD.gn index f7b47dd0b13..3954cdcdaf9 100644 --- a/chromium/components/feed/core/BUILD.gn +++ b/chromium/components/feed/core/BUILD.gn @@ -8,20 +8,30 @@ if (is_android) { source_set("feed_core") { sources = [ - "feed_host_service.cc", - "feed_host_service.h", + "feed_content_database.cc", + "feed_content_database.h", + "feed_content_mutation.cc", + "feed_content_mutation.h", + "feed_content_operation.cc", + "feed_content_operation.h", "feed_image_database.cc", "feed_image_database.h", "feed_image_manager.cc", "feed_image_manager.h", + "feed_journal_database.cc", + "feed_journal_database.h", + "feed_journal_mutation.cc", + "feed_journal_mutation.h", + "feed_journal_operation.cc", + "feed_journal_operation.h", "feed_networking_host.cc", "feed_networking_host.h", "feed_scheduler_host.cc", "feed_scheduler_host.h", - "feed_storage_database.cc", - "feed_storage_database.h", "pref_names.cc", "pref_names.h", + "refresh_throttler.cc", + "refresh_throttler.h", "time_serialization.cc", "time_serialization.h", "user_classifier.cc", @@ -33,7 +43,6 @@ source_set("feed_core") { "//components/feed:feature_list", "//components/feed/core/proto", "//components/image_fetcher/core:core", - "//components/keyed_service/core", "//components/leveldb_proto", "//net", ] @@ -64,11 +73,15 @@ if (is_android) { source_set("core_unit_tests") { testonly = true sources = [ + "feed_content_database_unittest.cc", + "feed_content_mutation_unittest.cc", "feed_image_database_unittest.cc", "feed_image_manager_unittest.cc", + "feed_journal_database_unittest.cc", + "feed_journal_mutation_unittest.cc", "feed_networking_host_unittest.cc", "feed_scheduler_host_unittest.cc", - "feed_storage_database_unittest.cc", + "refresh_throttler_unittest.cc", "user_classifier_unittest.cc", ] diff --git a/chromium/components/feed/core/feed_content_database.cc b/chromium/components/feed/core/feed_content_database.cc new file mode 100644 index 00000000000..6126a56a266 --- /dev/null +++ b/chromium/components/feed/core/feed_content_database.cc @@ -0,0 +1,294 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_content_database.h" + +#include <unordered_set> + +#include "base/bind.h" +#include "base/files/file_path.h" +#include "base/strings/string_util.h" +#include "base/sys_info.h" +#include "base/task/post_task.h" +#include "base/threading/thread_task_runner_handle.h" +#include "components/feed/core/proto/content_storage.pb.h" +#include "components/leveldb_proto/proto_database_impl.h" + +namespace feed { + +namespace { +using StorageEntryVector = + leveldb_proto::ProtoDatabase<ContentStorageProto>::KeyEntryVector; + +// Statistics are logged to UMA with this string as part of histogram name. They +// can all be found under LevelDB.*.FeedContentDatabase. Changing this needs to +// synchronize with histograms.xml, AND will also become incompatible with older +// browsers still reporting the previous values. +const char kContentDatabaseUMAClientName[] = "FeedContentDatabase"; + +const char kContentDatabaseFolder[] = "content"; + +const size_t kDatabaseWriteBufferSizeBytes = 512 * 1024; +const size_t kDatabaseWriteBufferSizeBytesForLowEndDevice = 128 * 1024; + +bool DatabaseKeyFilter(const std::unordered_set<std::string>& key_set, + const std::string& key) { + return key_set.find(key) != key_set.end(); +} + +bool DatabasePrefixFilter(const std::string& key_prefix, + const std::string& key) { + return base::StartsWith(key, key_prefix, base::CompareCase::SENSITIVE); +} + +} // namespace + +FeedContentDatabase::FeedContentDatabase(const base::FilePath& database_folder) + : FeedContentDatabase( + database_folder, + std::make_unique< + leveldb_proto::ProtoDatabaseImpl<ContentStorageProto>>( + base::CreateSequencedTaskRunnerWithTraits( + {base::MayBlock(), base::TaskPriority::BEST_EFFORT, + base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN}))) {} + +FeedContentDatabase::FeedContentDatabase( + const base::FilePath& database_folder, + std::unique_ptr<leveldb_proto::ProtoDatabase<ContentStorageProto>> + storage_database) + : database_status_(UNINITIALIZED), + storage_database_(std::move(storage_database)), + weak_ptr_factory_(this) { + leveldb_env::Options options = leveldb_proto::CreateSimpleOptions(); + if (base::SysInfo::IsLowEndDevice()) { + options.write_buffer_size = kDatabaseWriteBufferSizeBytesForLowEndDevice; + } else { + options.write_buffer_size = kDatabaseWriteBufferSizeBytes; + } + + base::FilePath storage_folder = + database_folder.AppendASCII(kContentDatabaseFolder); + storage_database_->Init( + kContentDatabaseUMAClientName, storage_folder, options, + base::BindOnce(&FeedContentDatabase::OnDatabaseInitialized, + weak_ptr_factory_.GetWeakPtr())); +} + +FeedContentDatabase::~FeedContentDatabase() = default; + +bool FeedContentDatabase::IsInitialized() const { + return INITIALIZED == database_status_; +} + +void FeedContentDatabase::LoadContent(const std::vector<std::string>& keys, + ContentLoadCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + std::unordered_set<std::string> key_set(keys.begin(), keys.end()); + + storage_database_->LoadEntriesWithFilter( + base::BindRepeating(&DatabaseKeyFilter, std::move(key_set)), + base::BindOnce(&FeedContentDatabase::OnLoadEntriesForLoadContent, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void FeedContentDatabase::LoadContentByPrefix(const std::string& prefix, + ContentLoadCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + storage_database_->LoadEntriesWithFilter( + base::BindRepeating(&DatabasePrefixFilter, std::move(prefix)), + base::BindOnce(&FeedContentDatabase::OnLoadEntriesForLoadContent, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void FeedContentDatabase::LoadAllContentKeys(ContentKeyCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + storage_database_->LoadKeys( + base::BindOnce(&FeedContentDatabase::OnLoadKeysForLoadAllContentKeys, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void FeedContentDatabase::CommitContentMutation( + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(content_mutation); + + PerformNextOperation(std::move(content_mutation), std::move(callback)); +} + +void FeedContentDatabase::PerformNextOperation( + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback) { + DCHECK(!content_mutation->Empty()); + + ContentOperation operation = content_mutation->TakeFristOperation(); + + switch (operation.type()) { + case ContentOperation::CONTENT_DELETE: + // TODO(gangwu): If deletes are continuous, we should combine them into + // one commit. + DeleteContent(std::move(operation), std::move(content_mutation), + std::move(callback)); + break; + case ContentOperation::CONTENT_DELETE_BY_PREFIX: + DeleteContentByPrefix(std::move(operation), std::move(content_mutation), + std::move(callback)); + break; + case ContentOperation::CONTENT_UPSERT: + // TODO(gangwu): If upserts are continuous, we should combine them into + // one commit. + UpsertContent(std::move(operation), std::move(content_mutation), + std::move(callback)); + break; + case ContentOperation::CONTENT_DELETE_ALL: + DeleteAllContent(std::move(operation), std::move(content_mutation), + std::move(callback)); + break; + default: + // Operation type is not supported, therefore failing immediately. + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), false)); + } +} + +void FeedContentDatabase::UpsertContent( + ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback) { + DCHECK_EQ(operation.type(), ContentOperation::CONTENT_UPSERT); + + auto contents_to_save = std::make_unique<StorageEntryVector>(); + ContentStorageProto proto; + proto.set_key(operation.key()); + proto.set_content_data(operation.value()); + contents_to_save->emplace_back(proto.key(), std::move(proto)); + + storage_database_->UpdateEntries( + std::move(contents_to_save), std::make_unique<std::vector<std::string>>(), + base::BindOnce(&FeedContentDatabase::OnOperationCommitted, + weak_ptr_factory_.GetWeakPtr(), + std::move(content_mutation), std::move(callback))); +} + +void FeedContentDatabase::DeleteContent( + ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback) { + DCHECK_EQ(operation.type(), ContentOperation::CONTENT_DELETE); + + auto content_to_delete = std::make_unique<std::vector<std::string>>( + std::initializer_list<std::string>({operation.key()})); + + storage_database_->UpdateEntries( + std::make_unique<StorageEntryVector>(), std::move(content_to_delete), + base::BindOnce(&FeedContentDatabase::OnOperationCommitted, + weak_ptr_factory_.GetWeakPtr(), + std::move(content_mutation), std::move(callback))); +} + +void FeedContentDatabase::DeleteContentByPrefix( + ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback) { + DCHECK_EQ(operation.type(), ContentOperation::CONTENT_DELETE_BY_PREFIX); + + storage_database_->UpdateEntriesWithRemoveFilter( + std::make_unique<StorageEntryVector>(), + base::BindRepeating(&DatabasePrefixFilter, operation.prefix()), + base::BindOnce(&FeedContentDatabase::OnOperationCommitted, + weak_ptr_factory_.GetWeakPtr(), + std::move(content_mutation), std::move(callback))); +} + +void FeedContentDatabase::DeleteAllContent( + ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback) { + DCHECK_EQ(operation.type(), ContentOperation::CONTENT_DELETE_ALL); + + std::string key_prefix = ""; + storage_database_->UpdateEntriesWithRemoveFilter( + std::make_unique<StorageEntryVector>(), + base::BindRepeating(&DatabasePrefixFilter, std::move(key_prefix)), + base::BindOnce(&FeedContentDatabase::OnOperationCommitted, + weak_ptr_factory_.GetWeakPtr(), + std::move(content_mutation), std::move(callback))); +} + +void FeedContentDatabase::OnDatabaseInitialized(bool success) { + DCHECK_EQ(database_status_, UNINITIALIZED); + + if (success) { + database_status_ = INITIALIZED; + } else { + database_status_ = INIT_FAILURE; + DVLOG(1) << "FeedContentDatabase init failed."; + } +} + +void FeedContentDatabase::OnLoadEntriesForLoadContent( + ContentLoadCallback callback, + bool success, + std::unique_ptr<std::vector<ContentStorageProto>> content) { + std::vector<KeyAndData> results; + + if (!success || !content) { + DVLOG_IF(1, !success) << "FeedContentDatabase load content failed."; + std::move(callback).Run(std::move(results)); + return; + } + + for (const auto& proto : *content) { + DCHECK(proto.has_key()); + DCHECK(proto.has_content_data()); + + results.emplace_back(proto.key(), proto.content_data()); + } + + std::move(callback).Run(std::move(results)); +} + +void FeedContentDatabase::OnLoadKeysForLoadAllContentKeys( + ContentKeyCallback callback, + bool success, + std::unique_ptr<std::vector<std::string>> keys) { + if (!success || !keys) { + DVLOG_IF(1, !success) << "FeedContentDatabase load content keys failed."; + std::vector<std::string> results; + std::move(callback).Run(std::move(results)); + return; + } + + // We std::move the |*keys|'s entries to |callback|, after that, |keys| become + // a pointer holding an empty vector, then we can safely delete unique_ptr + // |keys| when it out of scope. + std::move(callback).Run(std::move(*keys)); +} + +void FeedContentDatabase::OnOperationCommitted( + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback, + bool success) { + // Commit is unsuccessful, skip processing the other operations since + // ContentStorage.java requires "In the event of a failure, processing is + // halted immediately". + if (!success) { + DVLOG(1) << "FeedContentDatabase committed failed."; + std::move(callback).Run(success); + return; + } + + // All operations were committed successfully, call |callback|. + if (content_mutation->Empty()) { + std::move(callback).Run(success); + return; + } + + PerformNextOperation(std::move(content_mutation), std::move(callback)); +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_content_database.h b/chromium/components/feed/core/feed_content_database.h new file mode 100644 index 00000000000..ceb5d2b27e1 --- /dev/null +++ b/chromium/components/feed/core/feed_content_database.h @@ -0,0 +1,131 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_FEED_CONTENT_DATABASE_H_ +#define COMPONENTS_FEED_CORE_FEED_CONTENT_DATABASE_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "base/memory/weak_ptr.h" +#include "base/sequenced_task_runner.h" +#include "components/feed/core/feed_content_mutation.h" +#include "components/feed/core/feed_content_operation.h" +#include "components/leveldb_proto/proto_database.h" + +namespace feed { + +class ContentStorageProto; + +// FeedContentDatabase is leveldb backend store for Feed's content storage data. +// Feed's content data are key-value pairs. +class FeedContentDatabase { + public: + enum State { + UNINITIALIZED, + INITIALIZED, + INIT_FAILURE, + }; + + using KeyAndData = std::pair<std::string, std::string>; + + // Returns the storage data as a vector of key-value pairs when calling + // loading data. + using ContentLoadCallback = base::OnceCallback<void(std::vector<KeyAndData>)>; + + // Returns the content keys as a vector when calling loading all content keys. + using ContentKeyCallback = base::OnceCallback<void(std::vector<std::string>)>; + + // Returns whether the commit operation succeeded when calling for database + // operations, or return whether the entry exists when calling for checking + // the entry's existence. + using ConfirmationCallback = base::OnceCallback<void(bool)>; + + // Initializes the database with |database_folder|. + explicit FeedContentDatabase(const base::FilePath& database_folder); + + // Initializes the database with |database_folder|. Creates storage using the + // given |storage_database| for local storage. Useful for testing. + FeedContentDatabase( + const base::FilePath& database_folder, + std::unique_ptr<leveldb_proto::ProtoDatabase<ContentStorageProto>> + storage_database); + + ~FeedContentDatabase(); + + // Returns true if initialization has finished successfully, else false. + // While this is false, initialization may already started, or initialization + // failed. + bool IsInitialized() const; + + // Loads the content data for the |keys| and passes them to |callback|. + void LoadContent(const std::vector<std::string>& keys, + ContentLoadCallback callback); + + // Loads the content data whose key matches |prefix|, and passes them to + // |callback|. + void LoadContentByPrefix(const std::string& prefix, + ContentLoadCallback callback); + + // Loads all content keys in the storage, and passes them to |callback|. + void LoadAllContentKeys(ContentKeyCallback callback); + + // Commits the operations in the |content_mutation|. |callback| will be called + // when all the operations are committed. Or if any operation failed, database + // will stop process any operations and passed error to |callback|. + void CommitContentMutation(std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback); + + private: + // These methods work with |CommitContentMutation|. They process + // |ContentOperation| in |ContentMutation| which is passed to + // |PerformNextOperation| by |CommitContentMutation|. + void PerformNextOperation(std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback); + void UpsertContent(ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback); + void DeleteContent(ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback); + void DeleteContentByPrefix(ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback); + void DeleteAllContent(ContentOperation operation, + std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback); + + // Callback methods given to |storage_database_| for async responses. + void OnDatabaseInitialized(bool success); + void OnLoadEntriesForLoadContent( + ContentLoadCallback callback, + bool success, + std::unique_ptr<std::vector<ContentStorageProto>> content); + void OnLoadKeysForLoadAllContentKeys( + ContentKeyCallback callback, + bool success, + std::unique_ptr<std::vector<std::string>> keys); + void OnOperationCommitted(std::unique_ptr<ContentMutation> content_mutation, + ConfirmationCallback callback, + bool success); + + // Status of the database initialization. + State database_status_; + + // The database for storing content storage information. + std::unique_ptr<leveldb_proto::ProtoDatabase<ContentStorageProto>> + storage_database_; + + SEQUENCE_CHECKER(sequence_checker_); + + base::WeakPtrFactory<FeedContentDatabase> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(FeedContentDatabase); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_FEED_CONTENT_DATABASE_H_ diff --git a/chromium/components/feed/core/feed_content_database_unittest.cc b/chromium/components/feed/core/feed_content_database_unittest.cc new file mode 100644 index 00000000000..b30f4522cb7 --- /dev/null +++ b/chromium/components/feed/core/feed_content_database_unittest.cc @@ -0,0 +1,332 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_content_database.h" + +#include <map> + +#include "base/test/scoped_task_environment.h" +#include "components/feed/core/feed_content_mutation.h" +#include "components/feed/core/proto/content_storage.pb.h" +#include "components/leveldb_proto/testing/fake_db.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using leveldb_proto::test::FakeDB; +using testing::_; + +namespace feed { + +namespace { + +const char kContentKeyPrefix[] = "ContentKey"; +const char kContentKey1[] = "ContentKey1"; +const char kContentKey2[] = "ContentKey2"; +const char kContentKey3[] = "ContentKey3"; +const char kContentData1[] = "Content Data1"; +const char kContentData2[] = "Content Data2"; + +} // namespace + +class FeedContentDatabaseTest : public testing::Test { + public: + FeedContentDatabaseTest() : content_db_(nullptr) {} + + void CreateDatabase(bool init_database) { + // The FakeDBs are owned by |feed_db_|, so clear our pointers before + // resetting |feed_db_| itself. + content_db_ = nullptr; + // Explicitly destroy any existing database before creating a new one. + feed_db_.reset(); + + auto storage_db = + std::make_unique<FakeDB<ContentStorageProto>>(&content_db_storage_); + + content_db_ = storage_db.get(); + feed_db_ = std::make_unique<FeedContentDatabase>(base::FilePath(), + std::move(storage_db)); + if (init_database) { + content_db_->InitCallback(true); + ASSERT_TRUE(db()->IsInitialized()); + } + } + + void InjectContentStorageProto(const std::string& key, + const std::string& data) { + ContentStorageProto storage_proto; + storage_proto.set_key(key); + storage_proto.set_content_data(data); + content_db_storage_[key] = storage_proto; + } + + void RunUntilIdle() { scoped_task_environment_.RunUntilIdle(); } + + FakeDB<ContentStorageProto>* storage_db() { return content_db_; } + + FeedContentDatabase* db() { return feed_db_.get(); } + + MOCK_METHOD1(OnContentEntriesReceived, + void(std::vector<std::pair<std::string, std::string>>)); + MOCK_METHOD1(OnContentKeyReceived, void(std::vector<std::string>)); + MOCK_METHOD1(OnStorageCommitted, void(bool)); + + private: + base::test::ScopedTaskEnvironment scoped_task_environment_; + + std::map<std::string, ContentStorageProto> content_db_storage_; + + // Owned by |feed_db_|. + FakeDB<ContentStorageProto>* content_db_; + + std::unique_ptr<FeedContentDatabase> feed_db_; + + DISALLOW_COPY_AND_ASSIGN(FeedContentDatabaseTest); +}; + +TEST_F(FeedContentDatabaseTest, Init) { + ASSERT_FALSE(db()); + + CreateDatabase(/*init_database=*/false); + + storage_db()->InitCallback(true); + EXPECT_TRUE(db()->IsInitialized()); +} + +TEST_F(FeedContentDatabaseTest, LoadContentAfterInitSuccess) { + CreateDatabase(/*init_database=*/true); + + EXPECT_CALL(*this, OnContentEntriesReceived(_)); + db()->LoadContent( + {kContentKey1}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, LoadContentsEntries) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1| and |kContentKey2|. + InjectContentStorageProto(kContentKey1, kContentData1); + InjectContentStorageProto(kContentKey2, kContentData2); + + // Try to Load |kContentKey2| and |kContentKey3|, only |kContentKey2| should + // return. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + ASSERT_EQ(results.size(), 1U); + EXPECT_EQ(results[0].first, kContentKey2); + EXPECT_EQ(results[0].second, kContentData2); + }); + db()->LoadContent( + {kContentKey2, kContentKey3}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, LoadContentsEntriesByPrefix) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1|, |kContentKey2|. + InjectContentStorageProto(kContentKey1, kContentData1); + InjectContentStorageProto(kContentKey2, kContentData2); + + // Try to Load "ContentKey", both |kContentKey1| and |kContentKey2| should + // return. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + ASSERT_EQ(results.size(), 2U); + EXPECT_EQ(results[0].first, kContentKey1); + EXPECT_EQ(results[0].second, kContentData1); + EXPECT_EQ(results[1].first, kContentKey2); + EXPECT_EQ(results[1].second, kContentData2); + }); + db()->LoadContentByPrefix( + kContentKeyPrefix, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, LoadAllContentKeys) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1|, |kContentKey2|. + InjectContentStorageProto(kContentKey1, kContentData1); + InjectContentStorageProto(kContentKey2, kContentData2); + + EXPECT_CALL(*this, OnContentKeyReceived(_)) + .WillOnce([](std::vector<std::string> results) { + ASSERT_EQ(results.size(), 2U); + EXPECT_EQ(results[0], kContentKey1); + EXPECT_EQ(results[1], kContentKey2); + }); + db()->LoadAllContentKeys(base::BindOnce( + &FeedContentDatabaseTest::OnContentKeyReceived, base::Unretained(this))); + storage_db()->LoadKeysCallback(true); +} + +TEST_F(FeedContentDatabaseTest, SaveContent) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1|, |kContentKey2|. + std::unique_ptr<ContentMutation> content_mutation = + std::make_unique<ContentMutation>(); + content_mutation->AppendUpsertOperation(kContentKey1, kContentData1); + content_mutation->AppendUpsertOperation(kContentKey2, kContentData2); + EXPECT_CALL(*this, OnStorageCommitted(true)); + db()->CommitContentMutation( + std::move(content_mutation), + base::BindOnce(&FeedContentDatabaseTest::OnStorageCommitted, + base::Unretained(this))); + storage_db()->UpdateCallback(true); + storage_db()->UpdateCallback(true); + + // Make sure they're there. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + ASSERT_EQ(results.size(), 2U); + EXPECT_EQ(results[0].first, kContentKey1); + EXPECT_EQ(results[0].second, kContentData1); + EXPECT_EQ(results[1].first, kContentKey2); + EXPECT_EQ(results[1].second, kContentData2); + }); + db()->LoadContent( + {kContentKey1, kContentKey2}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, DeleteContent) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1| and |kContentKey2|. + InjectContentStorageProto(kContentKey1, kContentData1); + InjectContentStorageProto(kContentKey2, kContentData2); + + // Delete |kContentKey2| and |kContentKey3| + std::unique_ptr<ContentMutation> content_mutation = + std::make_unique<ContentMutation>(); + content_mutation->AppendDeleteOperation(kContentKey2); + content_mutation->AppendDeleteOperation(kContentKey3); + EXPECT_CALL(*this, OnStorageCommitted(true)); + db()->CommitContentMutation( + std::move(content_mutation), + base::BindOnce(&FeedContentDatabaseTest::OnStorageCommitted, + base::Unretained(this))); + storage_db()->UpdateCallback(true); + storage_db()->UpdateCallback(true); + + // Make sure only |kContentKey2| got deleted. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + EXPECT_EQ(results.size(), 1U); + EXPECT_EQ(results[0].first, kContentKey1); + EXPECT_EQ(results[0].second, kContentData1); + }); + db()->LoadContent( + {kContentKey1, kContentKey2}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, DeleteContentByPrefix) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1| and |kContentKey2|. + InjectContentStorageProto(kContentKey1, kContentData1); + InjectContentStorageProto(kContentKey2, kContentData2); + + // Delete |kContentKey1| and |kContentKey2| + std::unique_ptr<ContentMutation> content_mutation = + std::make_unique<ContentMutation>(); + content_mutation->AppendDeleteByPrefixOperation(kContentKeyPrefix); + EXPECT_CALL(*this, OnStorageCommitted(true)); + db()->CommitContentMutation( + std::move(content_mutation), + base::BindOnce(&FeedContentDatabaseTest::OnStorageCommitted, + base::Unretained(this))); + storage_db()->UpdateCallback(true); + + // Make sure |kContentKey1| and |kContentKey2| got deleted. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + EXPECT_EQ(results.size(), 0U); + }); + db()->LoadContent( + {kContentKey1, kContentKey2}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, DeleteAllContent) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1| and |kContentKey2|. + InjectContentStorageProto(kContentKey1, kContentData1); + InjectContentStorageProto(kContentKey2, kContentData2); + + // Delete all content, meaning |kContentKey1| and |kContentKey2| are expected + // to be deleted. + std::unique_ptr<ContentMutation> content_mutation = + std::make_unique<ContentMutation>(); + content_mutation->AppendDeleteAllOperation(); + EXPECT_CALL(*this, OnStorageCommitted(true)); + db()->CommitContentMutation( + std::move(content_mutation), + base::BindOnce(&FeedContentDatabaseTest::OnStorageCommitted, + base::Unretained(this))); + storage_db()->UpdateCallback(true); + + // Make sure |kContentKey1| and |kContentKey2| got deleted. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + EXPECT_EQ(results.size(), 0U); + }); + db()->LoadContent( + {kContentKey1, kContentKey2}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +TEST_F(FeedContentDatabaseTest, SaveAndDeleteContent) { + CreateDatabase(/*init_database=*/true); + + // Store |kContentKey1|, |kContentKey2|. + std::unique_ptr<ContentMutation> content_mutation = + std::make_unique<ContentMutation>(); + content_mutation->AppendUpsertOperation(kContentKey1, kContentData1); + content_mutation->AppendUpsertOperation(kContentKey2, kContentData2); + content_mutation->AppendDeleteOperation(kContentKey2); + content_mutation->AppendDeleteOperation(kContentKey3); + EXPECT_CALL(*this, OnStorageCommitted(true)); + db()->CommitContentMutation( + std::move(content_mutation), + base::BindOnce(&FeedContentDatabaseTest::OnStorageCommitted, + base::Unretained(this))); + storage_db()->UpdateCallback(true); + storage_db()->UpdateCallback(true); + storage_db()->UpdateCallback(true); + storage_db()->UpdateCallback(true); + + // Make sure only |kContentKey2| got deleted. + EXPECT_CALL(*this, OnContentEntriesReceived(_)) + .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { + EXPECT_EQ(results.size(), 1U); + EXPECT_EQ(results[0].first, kContentKey1); + EXPECT_EQ(results[0].second, kContentData1); + }); + db()->LoadContent( + {kContentKey1, kContentKey2}, + base::BindOnce(&FeedContentDatabaseTest::OnContentEntriesReceived, + base::Unretained(this))); + storage_db()->LoadCallback(true); +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_content_mutation.cc b/chromium/components/feed/core/feed_content_mutation.cc new file mode 100644 index 00000000000..06b1f7080e9 --- /dev/null +++ b/chromium/components/feed/core/feed_content_mutation.cc @@ -0,0 +1,48 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_content_mutation.h" + +#include <utility> + +#include "base/logging.h" +#include "components/feed/core/feed_content_operation.h" + +namespace feed { + +ContentMutation::ContentMutation() = default; + +ContentMutation::~ContentMutation() = default; + +void ContentMutation::AppendDeleteOperation(std::string key) { + operations_list_.emplace_back( + ContentOperation::CreateDeleteOperation(std::move(key))); +} + +void ContentMutation::AppendDeleteAllOperation() { + operations_list_.emplace_back(ContentOperation::CreateDeleteAllOperation()); +} + +void ContentMutation::AppendDeleteByPrefixOperation(std::string prefix) { + operations_list_.emplace_back( + ContentOperation::CreateDeleteByPrefixOperation(std::move(prefix))); +} + +void ContentMutation::AppendUpsertOperation(std::string key, + std::string value) { + operations_list_.emplace_back(ContentOperation::CreateUpsertOperation( + std::move(key), std::move(value))); +} + +bool ContentMutation::Empty() { + return operations_list_.empty(); +} + +ContentOperation ContentMutation::TakeFristOperation() { + ContentOperation operation = std::move(operations_list_.front()); + operations_list_.pop_front(); + return operation; +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_content_mutation.h b/chromium/components/feed/core/feed_content_mutation.h new file mode 100644 index 00000000000..d37ef282f57 --- /dev/null +++ b/chromium/components/feed/core/feed_content_mutation.h @@ -0,0 +1,47 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_FEED_CONTENT_MUTATION_H_ +#define COMPONENTS_FEED_CORE_FEED_CONTENT_MUTATION_H_ + +#include <list> +#include <string> + +#include "base/macros.h" + +namespace feed { + +class ContentOperation; + +// Native counterpart of ContentMutation.java. +// To commit a set of ContentOperation into FeedContentDatabase, user need to +// create a ContentMutation, next use Append* methods to append operations +// into the mutation, and then pass the ContentMutation to +// |FeedContentDatabase::CommitContentMutation| to commit operations. +class ContentMutation { + public: + ContentMutation(); + ~ContentMutation(); + + void AppendDeleteOperation(std::string key); + void AppendDeleteAllOperation(); + void AppendDeleteByPrefixOperation(std::string prefix); + void AppendUpsertOperation(std::string key, std::string value); + + // Check if mutation has ContentOperation left. + bool Empty(); + + // This will remove the first ContentOperation in |operations_list_| and + // return it to caller. + ContentOperation TakeFristOperation(); + + private: + std::list<ContentOperation> operations_list_; + + DISALLOW_COPY_AND_ASSIGN(ContentMutation); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_FEED_CONTENT_MUTATION_H_ diff --git a/chromium/components/feed/core/feed_content_mutation_unittest.cc b/chromium/components/feed/core/feed_content_mutation_unittest.cc new file mode 100644 index 00000000000..89a88789284 --- /dev/null +++ b/chromium/components/feed/core/feed_content_mutation_unittest.cc @@ -0,0 +1,80 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_content_mutation.h" + +#include "components/feed/core/feed_content_operation.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { + +namespace { + +const char kContentkey[] = "ContentKey"; +const char kContentValue[] = "Value"; +const char kContentPrefix[] = "Content"; + +} // namespace + +class FeedContentMutationTest : public testing::Test { + public: + FeedContentMutationTest() = default; + + private: + DISALLOW_COPY_AND_ASSIGN(FeedContentMutationTest); +}; + +TEST_F(FeedContentMutationTest, AppendDeleteOperation) { + ContentMutation mutation; + EXPECT_TRUE(mutation.Empty()); + + mutation.AppendDeleteOperation(kContentkey); + EXPECT_FALSE(mutation.Empty()); + + ContentOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), ContentOperation::CONTENT_DELETE); + EXPECT_EQ(operation.key(), kContentkey); +} + +TEST_F(FeedContentMutationTest, AppendDeleteAllOperation) { + ContentMutation mutation; + EXPECT_TRUE(mutation.Empty()); + + mutation.AppendDeleteAllOperation(); + EXPECT_FALSE(mutation.Empty()); + + ContentOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), ContentOperation::CONTENT_DELETE_ALL); +} + +TEST_F(FeedContentMutationTest, AppendDeleteByPrefixOperation) { + ContentMutation mutation; + EXPECT_TRUE(mutation.Empty()); + + mutation.AppendDeleteByPrefixOperation(kContentPrefix); + EXPECT_FALSE(mutation.Empty()); + + ContentOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), ContentOperation::CONTENT_DELETE_BY_PREFIX); + EXPECT_EQ(operation.prefix(), kContentPrefix); +} + +TEST_F(FeedContentMutationTest, AppendUpsertOperation) { + ContentMutation mutation; + EXPECT_TRUE(mutation.Empty()); + + mutation.AppendUpsertOperation(kContentkey, kContentValue); + EXPECT_FALSE(mutation.Empty()); + + ContentOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), ContentOperation::CONTENT_UPSERT); + EXPECT_EQ(operation.key(), kContentkey); + EXPECT_EQ(operation.value(), kContentValue); +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_content_operation.cc b/chromium/components/feed/core/feed_content_operation.cc new file mode 100644 index 00000000000..650600cda05 --- /dev/null +++ b/chromium/components/feed/core/feed_content_operation.cc @@ -0,0 +1,69 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_content_operation.h" + +#include <utility> + +#include "base/logging.h" + +namespace feed { + +// static +ContentOperation ContentOperation::CreateDeleteOperation(std::string key) { + return ContentOperation(CONTENT_DELETE, std::move(key), std::string(), + std::string()); +} + +// static +ContentOperation ContentOperation::CreateDeleteAllOperation() { + return ContentOperation(CONTENT_DELETE_ALL, std::string(), std::string(), + std::string()); +} + +// static +ContentOperation ContentOperation::CreateDeleteByPrefixOperation( + std::string prefix) { + return ContentOperation(CONTENT_DELETE_BY_PREFIX, std::string(), + std::string(), std::move(prefix)); +} + +// static +ContentOperation ContentOperation::CreateUpsertOperation(std::string key, + std::string value) { + return ContentOperation(CONTENT_UPSERT, std::move(key), std::move(value), + std::string()); +} + +ContentOperation::ContentOperation(ContentOperation&& operation) = default; + +ContentOperation::Type ContentOperation::type() { + return type_; +} + +const std::string& ContentOperation::key() { + DCHECK(type_ == CONTENT_UPSERT || type_ == CONTENT_DELETE); + return key_; +} + +const std::string& ContentOperation::value() { + DCHECK_EQ(type_, CONTENT_UPSERT); + return value_; +} + +const std::string& ContentOperation::prefix() { + DCHECK_EQ(type_, CONTENT_DELETE_BY_PREFIX); + return prefix_; +} + +ContentOperation::ContentOperation(Type type, + std::string key, + std::string value, + std::string prefix) + : type_(type), + key_(std::move(key)), + value_(std::move(value)), + prefix_(std::move(prefix)) {} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_content_operation.h b/chromium/components/feed/core/feed_content_operation.h new file mode 100644 index 00000000000..103e902de8b --- /dev/null +++ b/chromium/components/feed/core/feed_content_operation.h @@ -0,0 +1,53 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_FEED_CONTENT_OPERATION_H_ +#define COMPONENTS_FEED_CORE_FEED_CONTENT_OPERATION_H_ + +#include <string> + +#include "base/macros.h" + +namespace feed { + +// Native counterpart of ContentOperation.java. +class ContentOperation { + public: + enum Type { + CONTENT_DELETE, + CONTENT_DELETE_ALL, + CONTENT_DELETE_BY_PREFIX, + CONTENT_UPSERT, + }; + + static ContentOperation CreateDeleteOperation(std::string key); + static ContentOperation CreateDeleteAllOperation(); + static ContentOperation CreateDeleteByPrefixOperation(std::string prefix); + static ContentOperation CreateUpsertOperation(std::string key, + std::string value); + + ContentOperation(ContentOperation&& operation); + + Type type(); + const std::string& key(); + const std::string& value(); + const std::string& prefix(); + + private: + ContentOperation(Type type, + std::string key, + std::string value, + std::string prefix); + + const Type type_; + const std::string key_; + const std::string value_; + const std::string prefix_; + + DISALLOW_COPY_AND_ASSIGN(ContentOperation); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_FEED_CONTENT_OPERATION_H_ diff --git a/chromium/components/feed/core/feed_image_database.cc b/chromium/components/feed/core/feed_image_database.cc index d0ec8acfc1c..358e9466be1 100644 --- a/chromium/components/feed/core/feed_image_database.cc +++ b/chromium/components/feed/core/feed_image_database.cc @@ -8,8 +8,8 @@ #include "base/logging.h" #include "base/sequenced_task_runner.h" #include "base/sys_info.h" -#include "base/task_scheduler/post_task.h" -#include "base/task_scheduler/task_traits.h" +#include "base/task/post_task.h" +#include "base/task/task_traits.h" #include "base/threading/thread_task_runner_handle.h" #include "base/time/time.h" #include "components/feed/core/proto/cached_image.pb.h" @@ -36,7 +36,7 @@ FeedImageDatabase::FeedImageDatabase(const base::FilePath& database_dir) database_dir, std::make_unique<leveldb_proto::ProtoDatabaseImpl<CachedImageProto>>( base::CreateSequencedTaskRunnerWithTraits( - {base::MayBlock(), base::TaskPriority::BACKGROUND, + {base::MayBlock(), base::TaskPriority::BEST_EFFORT, base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN}))) {} FeedImageDatabase::FeedImageDatabase( diff --git a/chromium/components/feed/core/feed_image_manager.cc b/chromium/components/feed/core/feed_image_manager.cc index 59b4960978f..f4e81e5c1ec 100644 --- a/chromium/components/feed/core/feed_image_manager.cc +++ b/chromium/components/feed/core/feed_image_manager.cc @@ -7,6 +7,9 @@ #include <utility> #include "base/bind.h" +#include "base/metrics/histogram_macros.h" +#include "base/threading/thread_task_runner_handle.h" +#include "base/timer/elapsed_timer.h" #include "components/feed/core/time_serialization.h" #include "components/image_fetcher/core/image_decoder.h" #include "components/image_fetcher/core/image_fetcher.h" @@ -44,6 +47,9 @@ constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotation = } })"); +void ReportFetchResult(FeedImageFetchResult result) { + UMA_HISTOGRAM_ENUMERATION("NewTabPage.Feed.ImageFetchResult", result); +} } // namespace FeedImageManager::FeedImageManager( @@ -72,11 +78,16 @@ void FeedImageManager::FetchImagesFromDatabase(size_t url_index, ImageFetchedCallback callback) { if (url_index >= urls.size()) { // Already reached the last entry. Return an empty image. - std::move(callback).Run(gfx::Image()); + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), gfx::Image())); return; } - std::string image_id = urls[url_index]; + const std::string& image_id = urls[url_index]; + // Only take the first instance of the url so we get the worst-case time. + if (url_timers_.find(image_id) == url_timers_.end()) { + url_timers_.insert(std::make_pair(image_id, base::ElapsedTimer())); + } image_database_->LoadImage( image_id, base::BindOnce(&FeedImageManager::OnImageFetchedFromDatabase, weak_ptr_factory_.GetWeakPtr(), url_index, @@ -105,21 +116,37 @@ void FeedImageManager::OnImageDecodedFromDatabase(size_t url_index, std::vector<std::string> urls, ImageFetchedCallback callback, const gfx::Image& image) { + const std::string& image_id = urls[url_index]; if (image.IsEmpty()) { // If decoding the image failed, delete the DB entry. - image_database_->DeleteImage(urls[url_index]); + image_database_->DeleteImage(image_id); FetchImageFromNetwork(url_index, std::move(urls), std::move(callback)); return; } - std::move(callback).Run(image); + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), image)); + + // Report success if the url exists. + // This check is for concurrent access to the same url. + if (url_timers_.find(image_id) != url_timers_.end()) { + UMA_HISTOGRAM_TIMES("NewTabPage.Feed.ImageLoadFromCacheTime", + url_timers_[image_id].Elapsed()); + ClearUmaTimer(image_id); + ReportFetchResult(FeedImageFetchResult::kSuccessCached); + } } void FeedImageManager::FetchImageFromNetwork(size_t url_index, std::vector<std::string> urls, ImageFetchedCallback callback) { - GURL url(urls[url_index]); + const std::string& image_id = urls[url_index]; + GURL url(image_id); if (!url.is_valid()) { + // Report failure. + ReportFetchResult(FeedImageFetchResult::kFailure); + ClearUmaTimer(image_id); + // url is not valid, go to next URL. FetchImagesFromDatabase(url_index + 1, std::move(urls), std::move(callback)); @@ -141,6 +168,10 @@ void FeedImageManager::OnImageFetchedFromNetwork( const std::string& image_data, const image_fetcher::RequestMetadata& request_metadata) { if (image_data.empty()) { + // Report failure. + ReportFetchResult(FeedImageFetchResult::kFailure); + ClearUmaTimer(urls[url_index]); + // Fetching image failed, let's move to the next url. FetchImagesFromDatabase(url_index + 1, std::move(urls), std::move(callback)); @@ -160,15 +191,31 @@ void FeedImageManager::OnImageDecodedFromNetwork(size_t url_index, ImageFetchedCallback callback, const std::string& image_data, const gfx::Image& image) { - // Decoding urls[url_index] failed, let's move to the next url. + std::string image_id = urls[url_index]; if (image.IsEmpty()) { + // Report failure. + ReportFetchResult(FeedImageFetchResult::kFailure); + ClearUmaTimer(image_id); + + // Decoding failed, let's move to the next url. FetchImagesFromDatabase(url_index + 1, std::move(urls), std::move(callback)); return; } - image_database_->SaveImage(urls[url_index], image_data); - std::move(callback).Run(image); + image_database_->SaveImage(image_id, image_data); + + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), image)); + + // Report success if the url exists. + // This check is for concurrent access to the same url. + if (url_timers_.find(image_id) != url_timers_.end()) { + UMA_HISTOGRAM_TIMES("NewTabPage.Feed.ImageLoadFromNetworkTime", + url_timers_[image_id].Elapsed()); + ClearUmaTimer(image_id); + ReportFetchResult(FeedImageFetchResult::kSuccessFetched); + } } void FeedImageManager::DoGarbageCollectionIfNeeded() { @@ -204,4 +251,8 @@ void FeedImageManager::StopGarbageCollection() { garbage_collection_timer_.Stop(); } +void FeedImageManager::ClearUmaTimer(const std::string& url) { + url_timers_.erase(url); +} + } // namespace feed diff --git a/chromium/components/feed/core/feed_image_manager.h b/chromium/components/feed/core/feed_image_manager.h index 13af33b8dcc..a9681ced3ae 100644 --- a/chromium/components/feed/core/feed_image_manager.h +++ b/chromium/components/feed/core/feed_image_manager.h @@ -9,10 +9,15 @@ #include <string> #include <vector> +#include "base/containers/flat_map.h" #include "base/memory/weak_ptr.h" #include "base/timer/timer.h" #include "components/feed/core/feed_image_database.h" +namespace base { +class ElapsedTimer; +} // namespace base + namespace gfx { class Image; } // namespace gfx @@ -26,6 +31,15 @@ namespace feed { using ImageFetchedCallback = base::OnceCallback<void(const gfx::Image&)>; +// Enum for the result of the fetch, reported through UMA. +// New values should be added at the end and things should not be renumbered. +enum class FeedImageFetchResult { + kSuccessCached = 0, + kSuccessFetched = 1, + kFailure = 2, + kMaxValue = kFailure, +}; + // FeedImageManager takes care of fetching images from the network and caching // them in the database. class FeedImageManager { @@ -84,6 +98,8 @@ class FeedImageManager { void OnGarbageCollectionDone(base::Time garbage_collected_day, bool success); void StopGarbageCollection(); + void ClearUmaTimer(const std::string& url); + // The day which image database already ran garbage collection against on. base::Time image_garbage_collected_day_; @@ -92,6 +108,9 @@ class FeedImageManager { std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher_; std::unique_ptr<FeedImageDatabase> image_database_; + // Track time it takes to get images. + base::flat_map<std::string, base::ElapsedTimer> url_timers_; + base::WeakPtrFactory<FeedImageManager> weak_ptr_factory_; DISALLOW_COPY_AND_ASSIGN(FeedImageManager); diff --git a/chromium/components/feed/core/feed_image_manager_unittest.cc b/chromium/components/feed/core/feed_image_manager_unittest.cc index 26315409256..35eb596cbb3 100644 --- a/chromium/components/feed/core/feed_image_manager_unittest.cc +++ b/chromium/components/feed/core/feed_image_manager_unittest.cc @@ -11,6 +11,7 @@ #include "base/bind.h" #include "base/files/scoped_temp_dir.h" +#include "base/test/metrics/histogram_tester.h" #include "base/test/mock_callback.h" #include "base/test/scoped_task_environment.h" #include "base/threading/sequenced_task_runner_handle.h" @@ -24,6 +25,7 @@ #include "ui/gfx/image/image.h" #include "ui/gfx/image/image_unittest_util.h" +using base::HistogramTester; using testing::_; namespace feed { @@ -34,6 +36,13 @@ const char kImageURL2[] = "http://cake.com/"; const char kImageData[] = "pie image"; const char kImageData2[] = "cake image"; +const char kUmaImageLoadSuccessHistogramName[] = + "NewTabPage.Feed.ImageFetchResult"; +const char kUmaCacheLoadHistogramName[] = + "NewTabPage.Feed.ImageLoadFromCacheTime"; +const char kUmaNetworkLoadHistogramName[] = + "NewTabPage.Feed.ImageLoadFromNetworkTime"; + class FakeImageDecoder : public image_fetcher::ImageDecoder { public: void DecodeImage( @@ -116,6 +125,8 @@ class FeedImageManagerTest : public testing::Test { return &test_url_loader_factory_; } + HistogramTester& histogram() { return histogram_; } + MOCK_METHOD1(OnImageLoaded, void(const std::string&)); private: @@ -126,6 +137,7 @@ class FeedImageManagerTest : public testing::Test { base::ScopedTempDir database_dir_; FakeImageDecoder* fake_image_decoder_; base::test::ScopedTaskEnvironment scoped_task_environment_; + HistogramTester histogram_; DISALLOW_COPY_AND_ASSIGN(FeedImageManagerTest); }; @@ -260,4 +272,83 @@ TEST_F(FeedImageManagerTest, GarbageCollectionRunOnStart) { EXPECT_TRUE(garbage_collection_timer().IsRunning()); } +TEST_F(FeedImageManagerTest, InvalidUrlHistogramFailure) { + base::MockCallback<ImageFetchedCallback> image_callback; + feed_image_manager()->FetchImage(std::vector<std::string>({""}), + image_callback.Get()); + + RunUntilIdle(); + + histogram().ExpectTotalCount(kUmaCacheLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaNetworkLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaImageLoadSuccessHistogramName, 1); + histogram().ExpectBucketCount(kUmaImageLoadSuccessHistogramName, + FeedImageFetchResult::kFailure, 1); +} + +TEST_F(FeedImageManagerTest, FetchImageFromCachHistogram) { + // Save the image in the database. + image_database()->SaveImage(kImageURL, kImageData); + RunUntilIdle(); + + base::MockCallback<ImageFetchedCallback> image_callback; + feed_image_manager()->FetchImage(std::vector<std::string>({kImageURL}), + image_callback.Get()); + + RunUntilIdle(); + + histogram().ExpectTotalCount(kUmaCacheLoadHistogramName, 1); + histogram().ExpectTotalCount(kUmaNetworkLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaImageLoadSuccessHistogramName, 1); + histogram().ExpectBucketCount(kUmaImageLoadSuccessHistogramName, + FeedImageFetchResult::kSuccessCached, 1); +} + +TEST_F(FeedImageManagerTest, FetchImageFromNetworkHistogram) { + test_url_loader_factory()->AddResponse(kImageURL, kImageData); + base::MockCallback<ImageFetchedCallback> image_callback; + feed_image_manager()->FetchImage(std::vector<std::string>({kImageURL}), + image_callback.Get()); + + RunUntilIdle(); + + histogram().ExpectTotalCount(kUmaCacheLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaNetworkLoadHistogramName, 1); + histogram().ExpectTotalCount(kUmaImageLoadSuccessHistogramName, 1); + histogram().ExpectBucketCount(kUmaImageLoadSuccessHistogramName, + FeedImageFetchResult::kSuccessFetched, 1); +} + +TEST_F(FeedImageManagerTest, FetchImageFromNetworkEmptyHistogram) { + test_url_loader_factory()->AddResponse(kImageURL, ""); + base::MockCallback<ImageFetchedCallback> image_callback; + feed_image_manager()->FetchImage(std::vector<std::string>({kImageURL}), + image_callback.Get()); + + RunUntilIdle(); + + histogram().ExpectTotalCount(kUmaCacheLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaNetworkLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaImageLoadSuccessHistogramName, 1); + histogram().ExpectBucketCount(kUmaImageLoadSuccessHistogramName, + FeedImageFetchResult::kFailure, 1); +} + +TEST_F(FeedImageManagerTest, NetworkDecodingErrorHistogram) { + test_url_loader_factory()->AddResponse(kImageURL, kImageData); + fake_image_decoder()->SetDecodingValid(false); + + base::MockCallback<ImageFetchedCallback> image_callback; + feed_image_manager()->FetchImage(std::vector<std::string>({kImageURL}), + image_callback.Get()); + + RunUntilIdle(); + + histogram().ExpectTotalCount(kUmaCacheLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaNetworkLoadHistogramName, 0); + histogram().ExpectTotalCount(kUmaImageLoadSuccessHistogramName, 1); + histogram().ExpectBucketCount(kUmaImageLoadSuccessHistogramName, + FeedImageFetchResult::kFailure, 1); +} + } // namespace feed diff --git a/chromium/components/feed/core/feed_journal_database.cc b/chromium/components/feed/core/feed_journal_database.cc new file mode 100644 index 00000000000..2b1e5b53bdb --- /dev/null +++ b/chromium/components/feed/core/feed_journal_database.cc @@ -0,0 +1,109 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_journal_database.h" + +#include <utility> + +#include "base/sys_info.h" +#include "base/task/post_task.h" +#include "components/feed/core/proto/journal_storage.pb.h" +#include "components/leveldb_proto/proto_database_impl.h" + +namespace feed { + +namespace { +using StorageEntryVector = + leveldb_proto::ProtoDatabase<JournalStorageProto>::KeyEntryVector; + +// Statistics are logged to UMA with this string as part of histogram name. They +// can all be found under LevelDB.*.FeedJournalDatabase. Changing this needs to +// synchronize with histograms.xml, AND will also become incompatible with older +// browsers still reporting the previous values. +const char kJournalDatabaseUMAClientName[] = "FeedJournalDatabase"; + +const char kJournalDatabaseFolder[] = "journal"; + +const size_t kDatabaseWriteBufferSizeBytes = 512 * 1024; +const size_t kDatabaseWriteBufferSizeBytesForLowEndDevice = 128 * 1024; + +} // namespace + +FeedJournalDatabase::FeedJournalDatabase(const base::FilePath& database_folder) + : FeedJournalDatabase( + database_folder, + std::make_unique< + leveldb_proto::ProtoDatabaseImpl<JournalStorageProto>>( + base::CreateSequencedTaskRunnerWithTraits( + {base::MayBlock(), base::TaskPriority::BEST_EFFORT, + base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN}))) {} + +FeedJournalDatabase::FeedJournalDatabase( + const base::FilePath& database_folder, + std::unique_ptr<leveldb_proto::ProtoDatabase<JournalStorageProto>> + storage_database) + : database_status_(UNINITIALIZED), + storage_database_(std::move(storage_database)), + weak_ptr_factory_(this) { + leveldb_env::Options options = leveldb_proto::CreateSimpleOptions(); + if (base::SysInfo::IsLowEndDevice()) { + options.write_buffer_size = kDatabaseWriteBufferSizeBytesForLowEndDevice; + } else { + options.write_buffer_size = kDatabaseWriteBufferSizeBytes; + } + + base::FilePath storage_folder = + database_folder.AppendASCII(kJournalDatabaseFolder); + storage_database_->Init( + kJournalDatabaseUMAClientName, storage_folder, options, + base::BindOnce(&FeedJournalDatabase::OnDatabaseInitialized, + weak_ptr_factory_.GetWeakPtr())); +} + +FeedJournalDatabase::~FeedJournalDatabase() = default; + +bool FeedJournalDatabase::IsInitialized() const { + return INITIALIZED == database_status_; +} + +void FeedJournalDatabase::LoadJournal(const std::string& key, + JournalLoadCallback callback) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + storage_database_->GetEntry( + key, base::BindOnce(&FeedJournalDatabase::OnGetEntryForLoadJournal, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void FeedJournalDatabase::OnGetEntryForLoadJournal( + JournalLoadCallback callback, + bool success, + std::unique_ptr<JournalStorageProto> journal) { + std::vector<std::string> results; + + if (!success || !journal) { + DVLOG_IF(1, !success) << "FeedJournalDatabase load journal failed."; + std::move(callback).Run(std::move(results)); + return; + } + + for (int i = 0; i < journal->journal_data_size(); ++i) { + results.emplace_back(journal->journal_data(i)); + } + + std::move(callback).Run(std::move(results)); +} + +void FeedJournalDatabase::OnDatabaseInitialized(bool success) { + DCHECK_EQ(database_status_, UNINITIALIZED); + + if (success) { + database_status_ = INITIALIZED; + } else { + database_status_ = INIT_FAILURE; + DVLOG(1) << "FeedJournalDatabase init failed."; + } +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_journal_database.h b/chromium/components/feed/core/feed_journal_database.h new file mode 100644 index 00000000000..d5af51d692d --- /dev/null +++ b/chromium/components/feed/core/feed_journal_database.h @@ -0,0 +1,81 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_FEED_JOURNAL_DATABASE_H_ +#define COMPONENTS_FEED_CORE_FEED_JOURNAL_DATABASE_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "base/files/file_path.h" +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/sequenced_task_runner.h" +#include "components/leveldb_proto/proto_database.h" + +namespace feed { + +class JournalStorageProto; + +// FeedJournalDatabase is leveldb backend store for Feed's journal storage data. +// Feed's journal data are key-value pairs. +class FeedJournalDatabase { + public: + enum State { + UNINITIALIZED, + INITIALIZED, + INIT_FAILURE, + }; + + // Returns the journal data as a vector of strings when calling loading data + // or keys. + using JournalLoadCallback = + base::OnceCallback<void(std::vector<std::string>)>; + + // Initializes the database with |database_folder|. + explicit FeedJournalDatabase(const base::FilePath& database_folder); + + // Initializes the database with |database_folder|. Creates storage using the + // given |storage_database| for local storage. Useful for testing. + FeedJournalDatabase( + const base::FilePath& database_folder, + std::unique_ptr<leveldb_proto::ProtoDatabase<JournalStorageProto>> + storage_database); + + ~FeedJournalDatabase(); + + // Returns true if initialization has finished successfully, else false. + // While this is false, initialization may already started, or initialization + // failed. + bool IsInitialized() const; + + // Loads the journal data for the |key| and passes it to |callback|. + void LoadJournal(const std::string& key, JournalLoadCallback callback); + + private: + void OnGetEntryForLoadJournal(JournalLoadCallback callback, + bool success, + std::unique_ptr<JournalStorageProto> journal); + + // Callback methods given to |storage_database_| for async responses. + void OnDatabaseInitialized(bool success); + + // Status of the database initialization. + State database_status_; + + // The database for storing journal storage information. + std::unique_ptr<leveldb_proto::ProtoDatabase<JournalStorageProto>> + storage_database_; + + SEQUENCE_CHECKER(sequence_checker_); + + base::WeakPtrFactory<FeedJournalDatabase> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(FeedJournalDatabase); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_FEED_JOURNAL_DATABASE_H_ diff --git a/chromium/components/feed/core/feed_journal_database_unittest.cc b/chromium/components/feed/core/feed_journal_database_unittest.cc new file mode 100644 index 00000000000..ffb523b2af0 --- /dev/null +++ b/chromium/components/feed/core/feed_journal_database_unittest.cc @@ -0,0 +1,138 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_journal_database.h" + +#include <map> +#include <utility> + +#include "base/test/scoped_task_environment.h" +#include "components/feed/core/proto/journal_storage.pb.h" +#include "components/leveldb_proto/testing/fake_db.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using leveldb_proto::test::FakeDB; +using testing::Mock; +using testing::NotNull; +using testing::_; + +namespace feed { + +namespace { + +const char kJournalKey1[] = "JournalKey1"; +const char kJournalKey2[] = "JournalKey2"; +const char kJournalData1[] = "Journal Data1"; +const char kJournalData2[] = "Journal Data2"; +const char kJournalData3[] = "Journal Data3"; +const char kJournalData4[] = "Journal Data4"; + +} // namespace + +class FeedJournalDatabaseTest : public testing::Test { + public: + FeedJournalDatabaseTest() : journal_db_(nullptr) {} + + void CreateDatabase(bool init_database) { + // The FakeDBs are owned by |feed_db_|, so clear our pointers before + // resetting |feed_db_| itself. + journal_db_ = nullptr; + // Explicitly destroy any existing database before creating a new one. + feed_db_.reset(); + + auto storage_db = + std::make_unique<FakeDB<JournalStorageProto>>(&journal_db_storage_); + + journal_db_ = storage_db.get(); + feed_db_ = std::make_unique<FeedJournalDatabase>(base::FilePath(), + std::move(storage_db)); + if (init_database) { + journal_db_->InitCallback(true); + ASSERT_TRUE(db()->IsInitialized()); + } + } + + void InjectJournalStorageProto(const std::string& key, + const std::vector<std::string>& entries) { + JournalStorageProto storage_proto; + storage_proto.set_key(key); + for (const std::string& entry : entries) { + storage_proto.add_journal_data(entry); + } + journal_db_storage_[key] = storage_proto; + } + + void RunUntilIdle() { scoped_task_environment_.RunUntilIdle(); } + + FakeDB<JournalStorageProto>* storage_db() { return journal_db_; } + + FeedJournalDatabase* db() { return feed_db_.get(); } + + MOCK_METHOD1(OnJournalEntryReceived, void(std::vector<std::string>)); + + private: + base::test::ScopedTaskEnvironment scoped_task_environment_; + + std::map<std::string, JournalStorageProto> journal_db_storage_; + + // Owned by |feed_db_|. + FakeDB<JournalStorageProto>* journal_db_; + + std::unique_ptr<FeedJournalDatabase> feed_db_; + + DISALLOW_COPY_AND_ASSIGN(FeedJournalDatabaseTest); +}; + +TEST_F(FeedJournalDatabaseTest, Init) { + ASSERT_FALSE(db()); + + CreateDatabase(/*init_database=*/false); + + EXPECT_FALSE(db()->IsInitialized()); + storage_db()->InitCallback(true); + EXPECT_TRUE(db()->IsInitialized()); +} + +TEST_F(FeedJournalDatabaseTest, LoadJournalEntry) { + CreateDatabase(/*init_database=*/true); + EXPECT_TRUE(db()->IsInitialized()); + + // Store |kJournalKey1| and |kJournalKey2|. + InjectJournalStorageProto(kJournalKey1, + {kJournalData1, kJournalData2, kJournalData3}); + InjectJournalStorageProto(kJournalKey2, {kJournalData4}); + + // Try to Load |kJournalKey1|. + EXPECT_CALL(*this, OnJournalEntryReceived(_)) + .WillOnce([](std::vector<std::string> results) { + ASSERT_EQ(results.size(), 3U); + EXPECT_EQ(results[0], kJournalData1); + EXPECT_EQ(results[1], kJournalData2); + EXPECT_EQ(results[2], kJournalData3); + }); + db()->LoadJournal( + kJournalKey1, + base::BindOnce(&FeedJournalDatabaseTest::OnJournalEntryReceived, + base::Unretained(this))); + storage_db()->GetCallback(true); +} + +TEST_F(FeedJournalDatabaseTest, LoadNonExistingJournalEntry) { + CreateDatabase(/*init_database=*/true); + EXPECT_TRUE(db()->IsInitialized()); + + // Try to Load |kJournalKey1|. + EXPECT_CALL(*this, OnJournalEntryReceived(_)) + .WillOnce([](std::vector<std::string> results) { + ASSERT_EQ(results.size(), 0U); + }); + db()->LoadJournal( + kJournalKey1, + base::BindOnce(&FeedJournalDatabaseTest::OnJournalEntryReceived, + base::Unretained(this))); + storage_db()->GetCallback(true); +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_journal_mutation.cc b/chromium/components/feed/core/feed_journal_mutation.cc new file mode 100644 index 00000000000..7e4e6184ffa --- /dev/null +++ b/chromium/components/feed/core/feed_journal_mutation.cc @@ -0,0 +1,46 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_journal_mutation.h" + +#include <utility> + +#include "components/feed/core/feed_journal_operation.h" + +namespace feed { + +JournalMutation::JournalMutation(std::string journal_name) + : journal_name_(std::move(journal_name)) {} + +JournalMutation::~JournalMutation() = default; + +void JournalMutation::AddAppendOperation(std::string value) { + operations_list_.emplace_back( + JournalOperation::CreateAppendOperation(std::move(value))); +} + +void JournalMutation::AddCopyOperation(std::string to_journal_name) { + operations_list_.emplace_back( + JournalOperation::CreateCopyOperation(std::move(to_journal_name))); +} + +void JournalMutation::AddDeleteOperation() { + operations_list_.emplace_back(JournalOperation::CreateDeleteOperation()); +} + +const std::string& JournalMutation::journal_name() { + return journal_name_; +} + +bool JournalMutation::Empty() { + return operations_list_.empty(); +} + +JournalOperation JournalMutation::TakeFristOperation() { + JournalOperation operation = std::move(operations_list_.front()); + operations_list_.pop_front(); + return operation; +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_journal_mutation.h b/chromium/components/feed/core/feed_journal_mutation.h new file mode 100644 index 00000000000..25cb3c0753d --- /dev/null +++ b/chromium/components/feed/core/feed_journal_mutation.h @@ -0,0 +1,50 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_FEED_JOURNAL_MUTATION_H_ +#define COMPONENTS_FEED_CORE_FEED_JOURNAL_MUTATION_H_ + +#include <list> +#include <string> + +#include "base/macros.h" + +namespace feed { + +class JournalOperation; + +// Native counterpart of JournalMutation.java. +// To commit a set of JournalOperation into FeedJournalDatabase, first, +// JournalMutation need to be created, then use Add* methods to add operations +// into the mutation, and pass the JournalMutation to +// FeedJournalDatabase::CommitJournalMutation to commit operations. +class JournalMutation { + public: + explicit JournalMutation(std::string journal_name); + ~JournalMutation(); + + void AddAppendOperation(std::string value); + void AddCopyOperation(std::string to_journal_name); + void AddDeleteOperation(); + + const std::string& journal_name(); + + // Check if mutation has JournalOperation left. + bool Empty(); + + // This will remove the first JournalOperation in |operations_list_| and + // return it to caller. + JournalOperation TakeFristOperation(); + + private: + const std::string journal_name_; + + std::list<JournalOperation> operations_list_; + + DISALLOW_COPY_AND_ASSIGN(JournalMutation); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_FEED_JOURNAL_MUTATION_H_ diff --git a/chromium/components/feed/core/feed_journal_mutation_unittest.cc b/chromium/components/feed/core/feed_journal_mutation_unittest.cc new file mode 100644 index 00000000000..f6c83467e0c --- /dev/null +++ b/chromium/components/feed/core/feed_journal_mutation_unittest.cc @@ -0,0 +1,69 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_journal_mutation.h" + +#include "components/feed/core/feed_journal_operation.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { + +namespace { + +const char kJournalName1[] = "Journal1"; +const char kJournalName2[] = "Journal2"; +const char kJournalValue[] = "value"; + +} // namespace + +class FeedJournalMutationTest : public testing::Test { + public: + FeedJournalMutationTest() = default; + + private: + DISALLOW_COPY_AND_ASSIGN(FeedJournalMutationTest); +}; + +TEST_F(FeedJournalMutationTest, AddAppendOperation) { + JournalMutation mutation(kJournalName1); + EXPECT_EQ(mutation.journal_name(), kJournalName1); + EXPECT_TRUE(mutation.Empty()); + + mutation.AddAppendOperation(kJournalValue); + EXPECT_FALSE(mutation.Empty()); + + JournalOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), JournalOperation::JOURNAL_APPEND); + EXPECT_EQ(operation.value(), kJournalValue); +} + +TEST_F(FeedJournalMutationTest, AddCopyOperation) { + JournalMutation mutation(kJournalName1); + EXPECT_EQ(mutation.journal_name(), kJournalName1); + EXPECT_TRUE(mutation.Empty()); + + mutation.AddCopyOperation(kJournalName2); + EXPECT_FALSE(mutation.Empty()); + + JournalOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), JournalOperation::JOURNAL_COPY); + EXPECT_EQ(operation.to_journal_name(), kJournalName2); +} + +TEST_F(FeedJournalMutationTest, AddDeleteOperation) { + JournalMutation mutation(kJournalName1); + EXPECT_EQ(mutation.journal_name(), kJournalName1); + EXPECT_TRUE(mutation.Empty()); + + mutation.AddDeleteOperation(); + EXPECT_FALSE(mutation.Empty()); + + JournalOperation operation = mutation.TakeFristOperation(); + EXPECT_TRUE(mutation.Empty()); + EXPECT_EQ(operation.type(), JournalOperation::JOURNAL_DELETE); +} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_journal_operation.cc b/chromium/components/feed/core/feed_journal_operation.cc new file mode 100644 index 00000000000..4e561427818 --- /dev/null +++ b/chromium/components/feed/core/feed_journal_operation.cc @@ -0,0 +1,53 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/feed_journal_operation.h" + +#include <utility> + +#include "base/logging.h" + +namespace feed { + +// static +JournalOperation JournalOperation::CreateAppendOperation(std::string value) { + return JournalOperation(JOURNAL_APPEND, std::move(value), std::string()); +} + +// static +JournalOperation JournalOperation::CreateCopyOperation( + std::string to_journal_name) { + return JournalOperation(JOURNAL_COPY, std::string(), + std::move(to_journal_name)); +} + +// static +JournalOperation JournalOperation::CreateDeleteOperation() { + return JournalOperation(JOURNAL_DELETE, std::string(), std::string()); +} + +JournalOperation::JournalOperation(JournalOperation&& operation) = default; + +JournalOperation::Type JournalOperation::type() { + return type_; +} + +const std::string& JournalOperation::value() { + DCHECK_EQ(type_, JOURNAL_APPEND); + return value_; +} + +const std::string& JournalOperation::to_journal_name() { + DCHECK_EQ(type_, JOURNAL_COPY); + return to_journal_name_; +} + +JournalOperation::JournalOperation(Type type, + std::string value, + std::string to_journal_name) + : type_(type), + value_(std::move(value)), + to_journal_name_(std::move(to_journal_name)) {} + +} // namespace feed diff --git a/chromium/components/feed/core/feed_journal_operation.h b/chromium/components/feed/core/feed_journal_operation.h new file mode 100644 index 00000000000..7e447950498 --- /dev/null +++ b/chromium/components/feed/core/feed_journal_operation.h @@ -0,0 +1,49 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_FEED_JOURNAL_OPERATION_H_ +#define COMPONENTS_FEED_CORE_FEED_JOURNAL_OPERATION_H_ + +#include <string> + +#include "base/macros.h" + +namespace feed { + +// Native counterpart of JournalOperation.java. +class JournalOperation { + public: + enum Type { + JOURNAL_APPEND, + JOURNAL_COPY, + JOURNAL_DELETE, + }; + + static JournalOperation CreateAppendOperation(std::string value); + static JournalOperation CreateCopyOperation(std::string to_journal_name); + static JournalOperation CreateDeleteOperation(); + + JournalOperation(JournalOperation&& operation); + + Type type(); + const std::string& value(); + const std::string& to_journal_name(); + + private: + JournalOperation(Type type, std::string value, std::string to_journal_name); + + const Type type_; + + // Used for JOURNAL_APPEND, |value_| will be append to the end of journal. + const std::string value_; + + // Used for JOURNAL_COPY, copy current journal to |to_journal_name_|. + const std::string to_journal_name_; + + DISALLOW_COPY_AND_ASSIGN(JournalOperation); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_FEED_JOURNAL_OPERATION_H_ diff --git a/chromium/components/feed/core/feed_networking_host.cc b/chromium/components/feed/core/feed_networking_host.cc index cd94af0a4a7..510ce964844 100644 --- a/chromium/components/feed/core/feed_networking_host.cc +++ b/chromium/components/feed/core/feed_networking_host.cc @@ -8,6 +8,7 @@ #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" +#include "base/strings/stringprintf.h" #include "base/values.h" #include "components/variations/net/variations_http_headers.h" #include "google_apis/gaia/google_service_auth_error.h" @@ -31,6 +32,8 @@ namespace { constexpr char kApiKeyQueryParam[] = "key"; constexpr char kAuthenticationScope[] = "https://www.googleapis.com/auth/googlenow"; +constexpr char kAuthorizationRequestHeaderFormat[] = "Bearer %s"; + constexpr char kContentEncoding[] = "Content-Encoding"; constexpr char kContentType[] = "application/octet-stream"; constexpr char kGzip[] = "gzip"; @@ -59,15 +62,15 @@ class NetworkFetch { void StartAccessTokenFetch(); void AccessTokenFetchFinished(GoogleServiceAuthError error, identity::AccessTokenInfo access_token_info); - void StartLoader(const std::string& access_token); - std::unique_ptr<network::SimpleURLLoader> MakeLoader( - const std::string& access_token); + void StartLoader(); + std::unique_ptr<network::SimpleURLLoader> MakeLoader(); net::HttpRequestHeaders MakeHeaders(const std::string& auth_header) const; void PopulateRequestBody(network::SimpleURLLoader* loader); void OnSimpleLoaderComplete(std::unique_ptr<std::string> response); const GURL url_; const std::string request_type_; + std::string access_token_; const std::vector<uint8_t> request_body_; IdentityManager* const identity_manager_; std::unique_ptr<identity::PrimaryAccountAccessTokenFetcher> token_fetcher_; @@ -96,7 +99,7 @@ void NetworkFetch::Start(FeedNetworkingHost::ResponseCallback done_callback) { done_callback_ = std::move(done_callback); if (!identity_manager_->HasPrimaryAccount()) { - StartLoader(std::string()); + StartLoader(); return; } @@ -119,19 +122,24 @@ void NetworkFetch::AccessTokenFetchFinished( identity::AccessTokenInfo access_token_info) { UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.TokenFetchStatus", error.state(), GoogleServiceAuthError::NUM_STATES); - StartLoader(access_token_info.token); + access_token_ = access_token_info.token; + StartLoader(); } -void NetworkFetch::StartLoader(const std::string& access_token) { - simple_loader_ = MakeLoader(access_token); +void NetworkFetch::StartLoader() { + simple_loader_ = MakeLoader(); simple_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( loader_factory_, base::BindOnce(&NetworkFetch::OnSimpleLoaderComplete, base::Unretained(this))); } -std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader( - const std::string& access_token) { - net::HttpRequestHeaders headers = MakeHeaders(access_token); +std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader() { + std::string auth_header = + access_token_.empty() + ? std::string() + : base::StringPrintf(kAuthorizationRequestHeaderFormat, + access_token_.c_str()); + net::HttpRequestHeaders headers = MakeHeaders(auth_header); // TODO(pnoland): Add data use measurement once it's supported for simple // url loader. net::NetworkTrafficAnnotationTag traffic_annotation = @@ -160,15 +168,14 @@ std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader( } })"); GURL url(url_); - if (access_token.empty() && !api_key_.empty()) + if (access_token_.empty() && !api_key_.empty()) url = net::AppendQueryParameter(url_, kApiKeyQueryParam, api_key_); auto resource_request = std::make_unique<network::ResourceRequest>(); resource_request->url = url; - resource_request->load_flags = - net::LOAD_BYPASS_CACHE | net::LOAD_DO_NOT_SAVE_COOKIES | - net::LOAD_DO_NOT_SEND_COOKIES | net::LOAD_DO_NOT_SEND_AUTH_DATA; + resource_request->load_flags = net::LOAD_BYPASS_CACHE; + resource_request->allow_credentials = false; resource_request->headers = headers; resource_request->method = request_type_; @@ -223,6 +230,14 @@ void NetworkFetch::OnSimpleLoaderComplete( if (response) { status_code = simple_loader_->ResponseInfo()->headers->response_code(); + if (status_code == net::HTTP_UNAUTHORIZED) { + OAuth2TokenService::ScopeSet scopes{kAuthenticationScope}; + std::string account_id = + identity_manager_->GetPrimaryAccountInfo().account_id; + identity_manager_->RemoveAccessTokenFromCache(account_id, scopes, + access_token_); + } + const uint8_t* begin = reinterpret_cast<const uint8_t*>(response->data()); const uint8_t* end = begin + response->size(); response_body.assign(begin, end); diff --git a/chromium/components/feed/core/feed_networking_host_unittest.cc b/chromium/components/feed/core/feed_networking_host_unittest.cc index 5c51361e07f..27d736431bb 100644 --- a/chromium/components/feed/core/feed_networking_host_unittest.cc +++ b/chromium/components/feed/core/feed_networking_host_unittest.cc @@ -8,6 +8,7 @@ #include "base/strings/string_number_conversions.h" #include "base/strings/string_split.h" +#include "base/test/bind_test_util.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/test_mock_time_task_runner.h" #include "net/http/http_response_headers.h" @@ -127,6 +128,10 @@ class FeedNetworkingHostTest : public testing::Test { response_string); } + network::TestURLLoaderFactory* test_factory() { + return &test_factory_; + } + private: scoped_refptr<base::TestMockTimeTaskRunner> mock_task_runner_; identity::IdentityTestEnvironment identity_test_env_; @@ -222,8 +227,30 @@ TEST_F(FeedNetworkingHostTest, ShouldReportNonProtocolErrorCodes) { } } -// TODO(pnoland): Add a test that verifies request headers -// specify gzip. +TEST_F(FeedNetworkingHostTest, ShouldSetHeadersCorrectly) { + MockResponseDoneCallback done_callback; + net::HttpRequestHeaders headers; + base::RunLoop interceptor_run_loop; + base::HistogramTester histogram_tester; + + test_factory()->SetInterceptor( + base::BindLambdaForTesting([&](const network::ResourceRequest& request) { + headers = request.headers; + interceptor_run_loop.Quit(); + })); + + SendRequestAndRespond("http://foobar.com/feed", "POST", "", "", + net::HTTP_OK, network::URLLoaderCompletionStatus(), + &done_callback); + + std::string content_encoding; + std::string authorization; + EXPECT_TRUE(headers.GetHeader("content-encoding", &content_encoding)); + EXPECT_TRUE(headers.GetHeader("Authorization", &authorization)); + + EXPECT_EQ(content_encoding, "gzip"); + EXPECT_EQ(authorization, "Bearer access_token"); +} TEST_F(FeedNetworkingHostTest, ShouldReportSizeHistograms) { std::string uncompressed_request_string(2048, 'a'); diff --git a/chromium/components/feed/core/feed_scheduler_host.cc b/chromium/components/feed/core/feed_scheduler_host.cc index 30230d7cef1..557884654ca 100644 --- a/chromium/components/feed/core/feed_scheduler_host.cc +++ b/chromium/components/feed/core/feed_scheduler_host.cc @@ -4,7 +4,6 @@ #include "components/feed/core/feed_scheduler_host.h" -#include <map> #include <string> #include <utility> #include <vector> @@ -48,7 +47,7 @@ const base::FeatureParam<std::string> kDisableTriggerTypes{ // default value. ParamPair LookupParam(UserClass user_class, TriggerType trigger) { switch (user_class) { - case UserClass::kRareNtpUser: + case UserClass::kRareSuggestionsViewer: switch (trigger) { case TriggerType::kNtpShown: return {"ntp_shown_hours_rare_ntp_user", 4.0}; @@ -57,7 +56,7 @@ ParamPair LookupParam(UserClass user_class, TriggerType trigger) { case TriggerType::kFixedTimer: return {"fixed_timer_hours_rare_ntp_user", 96.0}; } - case UserClass::kActiveNtpUser: + case UserClass::kActiveSuggestionsViewer: switch (trigger) { case TriggerType::kNtpShown: return {"ntp_shown_hours_active_ntp_user", 4.0}; @@ -120,9 +119,9 @@ void TryRun(base::OnceClosure closure) { // entries in histogram suffix "UserClasses". std::string UserClassToHistogramSuffix(UserClassifier::UserClass user_class) { switch (user_class) { - case UserClassifier::UserClass::kRareNtpUser: + case UserClassifier::UserClass::kRareSuggestionsViewer: return "RareNTPUser"; - case UserClassifier::UserClass::kActiveNtpUser: + case UserClassifier::UserClass::kActiveSuggestionsViewer: return "ActiveNTPUser"; case UserClassifier::UserClass::kActiveSuggestionsConsumer: return "ActiveSuggestionsConsumer"; @@ -159,6 +158,19 @@ FeedSchedulerHost::FeedSchedulerHost(PrefService* profile_prefs, if (eula_accepted_notifier_) { eula_accepted_notifier_->Init(this); } + + throttlers_.emplace(UserClassifier::UserClass::kRareSuggestionsViewer, + std::make_unique<RefreshThrottler>( + UserClassifier::UserClass::kRareSuggestionsViewer, + profile_prefs_, clock_)); + throttlers_.emplace(UserClassifier::UserClass::kActiveSuggestionsViewer, + std::make_unique<RefreshThrottler>( + UserClassifier::UserClass::kActiveSuggestionsViewer, + profile_prefs_, clock_)); + throttlers_.emplace(UserClassifier::UserClass::kActiveSuggestionsConsumer, + std::make_unique<RefreshThrottler>( + UserClassifier::UserClass::kActiveSuggestionsConsumer, + profile_prefs_, clock_)); } FeedSchedulerHost::~FeedSchedulerHost() = default; @@ -231,7 +243,7 @@ NativeRequestBehavior FeedSchedulerHost::ShouldSessionRequestData( } } - user_classifier_.OnEvent(UserClassifier::Event::kNtpOpened); + OnSuggestionsShown(); DVLOG(2) << "Specifying NativeRequestBehavior of " << static_cast<int>(behavior); UMA_HISTOGRAM_ENUMERATION("Feed.Scheduler.RequestBehavior", behavior); @@ -291,6 +303,10 @@ void FeedSchedulerHost::OnSuggestionConsumed() { user_classifier_.OnEvent(UserClassifier::Event::kSuggestionsUsed); } +void FeedSchedulerHost::OnSuggestionsShown() { + user_classifier_.OnEvent(UserClassifier::Event::kSuggestionsViewed); +} + void FeedSchedulerHost::OnHistoryCleared() { // Due to privacy, we should not fetch for a while (unless the user explicitly // asks for new suggestions) to give sync the time to propagate the changes in @@ -360,7 +376,13 @@ bool FeedSchedulerHost::ShouldRefresh(TriggerType trigger) { return false; } - // TODO(skym): Check with throttler. + auto throttlerIter = throttlers_.find(user_class); + if (throttlerIter == throttlers_.end() || + !throttlerIter->second->RequestQuota()) { + DVLOG(2) << "Throttler stopped refresh from trigger " + << static_cast<int>(trigger); + return false; + } switch (trigger) { case TriggerType::kNtpShown: diff --git a/chromium/components/feed/core/feed_scheduler_host.h b/chromium/components/feed/core/feed_scheduler_host.h index a1e2c2eae68..a6d1595ca55 100644 --- a/chromium/components/feed/core/feed_scheduler_host.h +++ b/chromium/components/feed/core/feed_scheduler_host.h @@ -9,9 +9,11 @@ #include <set> #include "base/callback.h" +#include "base/containers/flat_map.h" #include "base/gtest_prod_util.h" #include "base/macros.h" #include "base/memory/weak_ptr.h" +#include "components/feed/core/refresh_throttler.h" #include "components/feed/core/user_classifier.h" #include "components/web_resource/eula_accepted_notifier.h" @@ -102,10 +104,14 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { // upgrades that change the way tasks are stored. void OnTaskReschedule(); - // Called when a suggestion is consumed to update what kind of user the - // scheduler should be optimizing for. + // Should be called when a suggestion is consumed. This is a signal the + // scheduler users to track the kind of user, and optimize refresh frequency. void OnSuggestionConsumed(); + // Should be called when suggestions are shown. This is a signal the scheduler + // users to track the kind of user, and optimize refresh frequency. + void OnSuggestionsShown(); + // When the user clears history, the scheduler will clear out some stored data // and stop requesting refreshes for a period of time. void OnHistoryCleared(); @@ -181,6 +187,11 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { bool time_until_first_shown_trigger_reported_ = false; bool time_until_first_foregrounded_trigger_reported_ = false; + // In the case the user transitions between user classes, hold onto a + // throttler for any situation. + base::flat_map<UserClassifier::UserClass, std::unique_ptr<RefreshThrottler>> + throttlers_; + DISALLOW_COPY_AND_ASSIGN(FeedSchedulerHost); }; diff --git a/chromium/components/feed/core/feed_scheduler_host_unittest.cc b/chromium/components/feed/core/feed_scheduler_host_unittest.cc index 7d5e4fba086..3ce5a5a9f6e 100644 --- a/chromium/components/feed/core/feed_scheduler_host_unittest.cc +++ b/chromium/components/feed/core/feed_scheduler_host_unittest.cc @@ -4,6 +4,7 @@ #include "components/feed/core/feed_scheduler_host.h" +#include <algorithm> #include <string> #include <vector> @@ -12,6 +13,7 @@ #include "base/test/metrics/histogram_tester.h" #include "base/test/simple_test_clock.h" #include "components/feed/core/pref_names.h" +#include "components/feed/core/refresh_throttler.h" #include "components/feed/core/time_serialization.h" #include "components/feed/core/user_classifier.h" #include "components/feed/feed_feature_list.h" @@ -44,6 +46,7 @@ class FeedSchedulerHostTest : public ::testing::Test { protected: FeedSchedulerHostTest() : weak_factory_(this) { FeedSchedulerHost::RegisterProfilePrefs(profile_prefs_.registry()); + RefreshThrottler::RegisterProfilePrefs(profile_prefs_.registry()); UserClassifier::RegisterProfilePrefs(profile_prefs_.registry()); local_state()->registry()->RegisterBooleanPref(::prefs::kEulaAccepted, true); @@ -874,4 +877,18 @@ TEST_F(FeedSchedulerHostTest, TimeUntilFirstMetrics) { EXPECT_EQ(2, histogram_tester.GetBucketCount(forgroundedHistogram, 0)); } +TEST_F(FeedSchedulerHostTest, RefreshThrottler) { + variations::testing::VariationParamsManager variation_params( + kInterestFeedContentSuggestions.name, + {{"quota_SuggestionFetcherActiveNTPUser", "3"}}, + {kInterestFeedContentSuggestions.name}); + NewScheduler(); + + for (int i = 0; i < 5; i++) { + scheduler()->OnForegrounded(); + ResetRefreshState(base::Time()); + EXPECT_EQ(std::min(i + 1, 3), refresh_call_count()); + } +} + } // namespace feed diff --git a/chromium/components/feed/core/feed_storage_database.cc b/chromium/components/feed/core/feed_storage_database.cc deleted file mode 100644 index f440efde917..00000000000 --- a/chromium/components/feed/core/feed_storage_database.cc +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/feed/core/feed_storage_database.h" - -#include <unordered_set> - -#include "base/bind.h" -#include "base/files/file_path.h" -#include "base/strings/string_util.h" -#include "base/sys_info.h" -#include "base/task_scheduler/post_task.h" -#include "components/feed/core/proto/feed_storage.pb.h" -#include "components/leveldb_proto/proto_database_impl.h" - -namespace feed { - -namespace { -using StorageEntryVector = - leveldb_proto::ProtoDatabase<FeedStorageProto>::KeyEntryVector; - -// Statistics are logged to UMA with this string as part of histogram name. They -// can all be found under LevelDB.*.FeedStorageDatabase. Changing this needs to -// synchronize with histograms.xml, AND will also become incompatible with older -// browsers still reporting the previous values. -const char kStorageDatabaseUMAClientName[] = "FeedStorageDatabase"; - -const char kStorageDatabaseFolder[] = "storage"; - -const size_t kDatabaseWriteBufferSizeBytes = 512 * 1024; -const size_t kDatabaseWriteBufferSizeBytesForLowEndDevice = 128 * 1024; - -// Key prefixes for content's storage key and journal's storage key. Because we -// put both content data and journal data into one storage, we need to add -// prefixes to their keys to distinguish between content keys and journal keys. -const char kContentStoragePrefix[] = "cs-"; -const char kJournalStoragePrefix[] = "js-"; - -// Formats content key to storage key by adding a prefix. -std::string FormatContentKeyToStorageKey(const std::string& content_key) { - return kContentStoragePrefix + content_key; -} - -// Formats journal key to storage key by adding a prefix. -std::string FormatJournalKeyToStorageKey(const std::string& journal_key) { - return kJournalStoragePrefix + journal_key; -} - -// Check if the |storage_key| is for content data. -bool IsValidContentKey(const std::string& storage_key) { - return base::StartsWith(storage_key, kContentStoragePrefix, - base::CompareCase::SENSITIVE); -} - -// Parse content key from storage key. Return an empty string if |storage_key| -// is not recognized as content key. ex. journal's storage key. -std::string ParseContentKey(const std::string& storage_key) { - if (!IsValidContentKey(storage_key)) { - return std::string(); - } - - return storage_key.substr(strlen(kContentStoragePrefix)); -} - -bool DatabaseKeyFilter(const std::unordered_set<std::string>& key_set, - const std::string& key) { - return key_set.find(key) != key_set.end(); -} - -bool DatabasePrefixFilter(const std::string& key_prefix, - const std::string& key) { - return base::StartsWith(key, key_prefix, base::CompareCase::SENSITIVE); -} - -} // namespace - -FeedStorageDatabase::FeedStorageDatabase(const base::FilePath& database_folder) - : FeedStorageDatabase( - database_folder, - std::make_unique<leveldb_proto::ProtoDatabaseImpl<FeedStorageProto>>( - base::CreateSequencedTaskRunnerWithTraits( - {base::MayBlock(), base::TaskPriority::BACKGROUND, - base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN}))) {} - -FeedStorageDatabase::FeedStorageDatabase( - const base::FilePath& database_folder, - std::unique_ptr<leveldb_proto::ProtoDatabase<FeedStorageProto>> - storage_database) - : database_status_(UNINITIALIZED), - storage_database_(std::move(storage_database)), - weak_ptr_factory_(this) { - leveldb_env::Options options = leveldb_proto::CreateSimpleOptions(); - if (base::SysInfo::IsLowEndDevice()) { - options.write_buffer_size = kDatabaseWriteBufferSizeBytesForLowEndDevice; - } else { - options.write_buffer_size = kDatabaseWriteBufferSizeBytes; - } - - base::FilePath storage_folder = - database_folder.AppendASCII(kStorageDatabaseFolder); - storage_database_->Init( - kStorageDatabaseUMAClientName, storage_folder, options, - base::BindOnce(&FeedStorageDatabase::OnDatabaseInitialized, - weak_ptr_factory_.GetWeakPtr())); -} - -FeedStorageDatabase::~FeedStorageDatabase() = default; - -bool FeedStorageDatabase::IsInitialized() const { - return INITIALIZED == database_status_; -} - -void FeedStorageDatabase::LoadContent(const std::vector<std::string>& keys, - ContentLoadCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - std::unordered_set<std::string> key_set; - for (const auto& key : keys) { - key_set.insert(FormatContentKeyToStorageKey(key)); - } - - storage_database_->LoadEntriesWithFilter( - base::BindRepeating(&DatabaseKeyFilter, std::move(key_set)), - base::BindOnce(&FeedStorageDatabase::OnLoadEntriesForLoadContent, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::LoadContentByPrefix(const std::string& prefix, - ContentLoadCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - std::string key_prefix = FormatContentKeyToStorageKey(prefix); - - storage_database_->LoadEntriesWithFilter( - base::BindRepeating(&DatabasePrefixFilter, std::move(key_prefix)), - base::BindOnce(&FeedStorageDatabase::OnLoadEntriesForLoadContent, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::LoadAllContentKeys(ContentKeyCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - storage_database_->LoadKeys( - base::BindOnce(&FeedStorageDatabase::OnLoadKeysForLoadAllContentKeys, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::SaveContent(std::vector<KeyAndData> pairs, - ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - auto contents_to_save = std::make_unique<StorageEntryVector>(); - for (auto entry : pairs) { - FeedStorageProto proto; - proto.set_key(std::move(entry.first)); - proto.set_content_data(std::move(entry.second)); - contents_to_save->emplace_back(FormatContentKeyToStorageKey(proto.key()), - std::move(proto)); - } - - storage_database_->UpdateEntries( - std::move(contents_to_save), std::make_unique<std::vector<std::string>>(), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::DeleteContent( - const std::vector<std::string>& keys_to_delete, - ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - auto content_to_delete = std::make_unique<std::vector<std::string>>(); - for (const auto& key : keys_to_delete) { - content_to_delete->emplace_back(FormatContentKeyToStorageKey(key)); - } - storage_database_->UpdateEntries( - std::make_unique<StorageEntryVector>(), std::move(content_to_delete), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::DeleteContentByPrefix( - const std::string& prefix_to_delete, - ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - std::string key_prefix = FormatContentKeyToStorageKey(prefix_to_delete); - storage_database_->UpdateEntriesWithRemoveFilter( - std::make_unique<StorageEntryVector>(), - base::BindRepeating(&DatabasePrefixFilter, std::move(key_prefix)), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::DeleteAllContent(ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - std::string key_prefix = FormatContentKeyToStorageKey(std::string()); - storage_database_->UpdateEntriesWithRemoveFilter( - std::make_unique<StorageEntryVector>(), - base::BindRepeating(&DatabasePrefixFilter, std::move(key_prefix)), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::LoadJournal(const std::string& key, - JournalLoadCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - storage_database_->GetEntry( - FormatJournalKeyToStorageKey(key), - base::BindOnce(&FeedStorageDatabase::OnGetEntryForLoadJournal, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::LoadAllJournals(LoadAllJournalsCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - storage_database_->LoadEntriesWithFilter( - base::BindRepeating(&DatabasePrefixFilter, kJournalStoragePrefix), - base::BindOnce(&FeedStorageDatabase::OnLoadEntriesForLoadAllJournals, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::AppendToJournal(const std::string& key, - std::vector<std::string> entries, - ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - storage_database_->GetEntry( - FormatJournalKeyToStorageKey(key), - base::BindOnce(&FeedStorageDatabase::OnGetEntryAppendToJournal, - weak_ptr_factory_.GetWeakPtr(), std::move(callback), key, - std::move(entries))); -} - -void FeedStorageDatabase::CopyJournal(const std::string& from_key, - const std::string& to_key, - ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - storage_database_->GetEntry( - FormatJournalKeyToStorageKey(from_key), - base::BindOnce(&FeedStorageDatabase::OnGetEntryForCopyJournal, - weak_ptr_factory_.GetWeakPtr(), std::move(callback), - to_key)); -} - -void FeedStorageDatabase::DeleteJournal(const std::string& key, - ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - auto journals_to_delete = std::make_unique<std::vector<std::string>>(); - journals_to_delete->push_back(FormatJournalKeyToStorageKey(key)); - - storage_database_->UpdateEntries( - std::make_unique<StorageEntryVector>(), std::move(journals_to_delete), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::DeleteAllJournals(ConfirmationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - std::string key_prefix = FormatJournalKeyToStorageKey(std::string()); - storage_database_->UpdateEntriesWithRemoveFilter( - std::make_unique<StorageEntryVector>(), - base::BindRepeating(&DatabasePrefixFilter, std::move(key_prefix)), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::OnDatabaseInitialized(bool success) { - DCHECK_EQ(database_status_, UNINITIALIZED); - - if (success) { - database_status_ = INITIALIZED; - } else { - database_status_ = INIT_FAILURE; - DVLOG(1) << "FeedStorageDatabase init failed."; - } -} - -void FeedStorageDatabase::OnLoadEntriesForLoadContent( - ContentLoadCallback callback, - bool success, - std::unique_ptr<std::vector<FeedStorageProto>> content) { - std::vector<KeyAndData> results; - - if (!success || !content) { - DVLOG_IF(1, !success) << "FeedStorageDatabase load content failed."; - std::move(callback).Run(std::move(results)); - return; - } - - for (const auto& proto : *content) { - DCHECK(proto.has_key()); - DCHECK(proto.has_content_data()); - - results.emplace_back(proto.key(), proto.content_data()); - } - - std::move(callback).Run(std::move(results)); -} - -void FeedStorageDatabase::OnLoadKeysForLoadAllContentKeys( - ContentKeyCallback callback, - bool success, - std::unique_ptr<std::vector<std::string>> keys) { - std::vector<std::string> results; - - if (!success || !keys) { - DVLOG_IF(1, !success) << "FeedStorageDatabase load content keys failed."; - std::move(callback).Run(std::move(results)); - return; - } - - // Filter out journal keys, only keep content keys. - for (const std::string& key : *keys) { - if (IsValidContentKey(key)) - results.emplace_back(ParseContentKey(key)); - } - - std::move(callback).Run(std::move(results)); -} - -void FeedStorageDatabase::OnGetEntryForLoadJournal( - JournalLoadCallback callback, - bool success, - std::unique_ptr<FeedStorageProto> journal) { - std::vector<std::string> results; - - if (!success || !journal) { - DVLOG_IF(1, !success) << "FeedStorageDatabase load journal failed."; - std::move(callback).Run(std::move(results)); - return; - } - - for (int i = 0; i < journal->journal_data_size(); ++i) { - results.emplace_back(journal->journal_data(i)); - } - - std::move(callback).Run(std::move(results)); -} - -void FeedStorageDatabase::OnGetEntryAppendToJournal( - ConfirmationCallback callback, - const std::string& key, - std::vector<std::string> entries, - bool success, - std::unique_ptr<FeedStorageProto> journal) { - if (!success) { - DVLOG(1) << "FeedStorageDatabase load journal failed."; - std::move(callback).Run(success); - return; - } - - if (journal == nullptr) { - // The journal does not exist, create a new one. - journal = std::make_unique<FeedStorageProto>(); - journal->set_key(key); - } - DCHECK_EQ(journal->key(), key); - - for (const std::string& entry : entries) { - journal->add_journal_data(entry); - } - auto journals_to_save = std::make_unique<StorageEntryVector>(); - journals_to_save->emplace_back(FormatJournalKeyToStorageKey(key), - std::move(*journal)); - - storage_database_->UpdateEntries( - std::move(journals_to_save), std::make_unique<std::vector<std::string>>(), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::OnGetEntryForCopyJournal( - ConfirmationCallback callback, - const std::string& to_key, - bool success, - std::unique_ptr<FeedStorageProto> journal) { - if (!success || !journal) { - DVLOG_IF(1, !success) << "FeedStorageDatabase load journal failed."; - std::move(callback).Run(success); - return; - } - - journal->set_key(to_key); - auto journal_to_save = std::make_unique<StorageEntryVector>(); - journal_to_save->emplace_back(FormatJournalKeyToStorageKey(to_key), - std::move(*journal)); - - storage_database_->UpdateEntries( - std::move(journal_to_save), std::make_unique<std::vector<std::string>>(), - base::BindOnce(&FeedStorageDatabase::OnStorageCommitted, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedStorageDatabase::OnLoadEntriesForLoadAllJournals( - LoadAllJournalsCallback callback, - bool success, - std::unique_ptr<std::vector<FeedStorageProto>> entries) { - std::vector<std::vector<std::string>> results; - - if (!success || !entries) { - DVLOG_IF(1, !success) << "FeedStorageDatabase load journals failed."; - std::move(callback).Run(std::move(results)); - return; - } - - for (const auto& entry : *entries) { - DCHECK(entry.has_key()); - DCHECK_NE(entry.journal_data_size(), 0); - - std::vector<std::string> journal; - journal.reserve(entry.journal_data_size()); - for (int i = 0; i < entry.journal_data_size(); ++i) { - journal.emplace_back(entry.journal_data(i)); - } - results.push_back(journal); - } - - std::move(callback).Run(std::move(results)); -} - -void FeedStorageDatabase::OnStorageCommitted(ConfirmationCallback callback, - bool success) { - DVLOG_IF(1, !success) << "FeedStorageDatabase committed failed."; - std::move(callback).Run(success); -} - -} // namespace feed diff --git a/chromium/components/feed/core/feed_storage_database.h b/chromium/components/feed/core/feed_storage_database.h deleted file mode 100644 index dc657c6e89d..00000000000 --- a/chromium/components/feed/core/feed_storage_database.h +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_FEED_CORE_FEED_STORAGE_DATABASE_H_ -#define COMPONENTS_FEED_CORE_FEED_STORAGE_DATABASE_H_ - -#include <memory> -#include <string> -#include <utility> -#include <vector> - -#include "base/memory/weak_ptr.h" -#include "base/sequenced_task_runner.h" -#include "components/leveldb_proto/proto_database.h" - -namespace feed { - -class FeedStorageProto; - -// FeedStorageDatabase is leveldb backed store for feed's content storage data -// and jounal storage data. -class FeedStorageDatabase { - public: - enum State { - UNINITIALIZED, - INITIALIZED, - INIT_FAILURE, - }; - - using KeyAndData = std::pair<std::string, std::string>; - - // Returns the storage data as a vector of key-value pairs when calling - // loading data. - using ContentLoadCallback = base::OnceCallback<void(std::vector<KeyAndData>)>; - - // Returns the content keys as a vector when calling loading all content keys. - using ContentKeyCallback = base::OnceCallback<void(std::vector<std::string>)>; - - // Returns the journal data as a vector of strings when calling loading data. - using JournalLoadCallback = - base::OnceCallback<void(std::vector<std::string>)>; - - // Returns a vector of journal data when calling loading all journals. - using LoadAllJournalsCallback = - base::OnceCallback<void(std::vector<std::vector<std::string>>)>; - - // Returns whether the commit operation succeeded. - using ConfirmationCallback = base::OnceCallback<void(bool)>; - - // Initializes the database with |database_folder|. - explicit FeedStorageDatabase(const base::FilePath& database_folder); - - // Initializes the database with |database_folder|. Creates storage using the - // given |storage_database| for local storage. Useful for testing. - FeedStorageDatabase( - const base::FilePath& database_folder, - std::unique_ptr<leveldb_proto::ProtoDatabase<FeedStorageProto>> - storage_database); - - ~FeedStorageDatabase(); - - // Returns true if initialization has finished successfully, else false. - // While this is false, initialization may already started, or initialization - // failed. - bool IsInitialized() const; - - // Loads the content data for the |keys| and passes them to |callback|. - void LoadContent(const std::vector<std::string>& keys, - ContentLoadCallback callback); - - // Loads the content data whose key matches |prefix|, and passes them to - // |callback|. - void LoadContentByPrefix(const std::string& prefix, - ContentLoadCallback callback); - - // Loads all content keys in the storage, and passes them to |callback|. - void LoadAllContentKeys(ContentKeyCallback callback); - - // Inserts or updates the content data |pairs|, |callback| will be called when - // the data are saved or if there is an error. The fields in |pairs| will be - // std::move. - void SaveContent(std::vector<KeyAndData> pairs, - ConfirmationCallback callback); - - // Deletes the content data for |keys_to_delete|, |callback| will be called - // when the data are deleted or if there is an error. - void DeleteContent(const std::vector<std::string>& keys_to_delete, - ConfirmationCallback callback); - - // Deletes the content data whose key matches |prefix_to_delete|, |callback| - // will be called when the content are deleted or if there is an error. - void DeleteContentByPrefix(const std::string& prefix_to_delete, - ConfirmationCallback callback); - - // Delete all content, |callback| will be called when all content is deleted - // or if there is an error. - void DeleteAllContent(ConfirmationCallback callback); - - // Loads the journal data for the |key| and passes it to |callback|. - void LoadJournal(const std::string& key, JournalLoadCallback callback); - - // Loads all journals in the storage, and passes them to |callback|. - void LoadAllJournals(LoadAllJournalsCallback callback); - - // Appends |entries| to a journal whose key is |key|, if there the journal do - // not exist, create one. |callback| will be called when the data are saved or - // if there is an error. - void AppendToJournal(const std::string& key, - std::vector<std::string> entries, - ConfirmationCallback callback); - - // Creates a new journal with name |to_key|, and copys all data from the - // journal with |from_key| to it. |callback| will be called when the data are - // saved or if there is an error. - void CopyJournal(const std::string& from_key, - const std::string& to_key, - ConfirmationCallback callback); - - // Deletes the journal with |key|, |callback| will be called when the journal - // is deleted or if there is an error. - void DeleteJournal(const std::string& key, ConfirmationCallback callback); - - // Delete all journals, |callback| will be called when all journals are - // deleted or if there is an error. - void DeleteAllJournals(ConfirmationCallback callback); - - private: - // Callback methods given to |storage_database_| for async responses. - void OnDatabaseInitialized(bool success); - void OnLoadEntriesForLoadContent( - ContentLoadCallback callback, - bool success, - std::unique_ptr<std::vector<FeedStorageProto>> content); - void OnLoadKeysForLoadAllContentKeys( - ContentKeyCallback callback, - bool success, - std::unique_ptr<std::vector<std::string>> keys); - void OnGetEntryForLoadJournal(JournalLoadCallback callback, - bool success, - std::unique_ptr<FeedStorageProto> journal); - void OnGetEntryAppendToJournal(ConfirmationCallback callback, - const std::string& key, - std::vector<std::string> entries, - bool success, - std::unique_ptr<FeedStorageProto> journal); - void OnGetEntryForCopyJournal(ConfirmationCallback callback, - const std::string& to_key, - bool success, - std::unique_ptr<FeedStorageProto> journal); - void OnLoadEntriesForLoadAllJournals( - LoadAllJournalsCallback callback, - bool success, - std::unique_ptr<std::vector<FeedStorageProto>> entries); - void OnStorageCommitted(ConfirmationCallback callback, bool success); - - State database_status_; - - std::unique_ptr<leveldb_proto::ProtoDatabase<FeedStorageProto>> - storage_database_; - - SEQUENCE_CHECKER(sequence_checker_); - - base::WeakPtrFactory<FeedStorageDatabase> weak_ptr_factory_; - - DISALLOW_COPY_AND_ASSIGN(FeedStorageDatabase); -}; - -} // namespace feed - -#endif // COMPONENTS_FEED_CORE_FEED_STORAGE_DATABASE_H_ diff --git a/chromium/components/feed/core/feed_storage_database_unittest.cc b/chromium/components/feed/core/feed_storage_database_unittest.cc deleted file mode 100644 index 71a6ee191d2..00000000000 --- a/chromium/components/feed/core/feed_storage_database_unittest.cc +++ /dev/null @@ -1,607 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/feed/core/feed_storage_database.h" - -#include <map> - -#include "base/test/scoped_task_environment.h" -#include "components/feed/core/proto/feed_storage.pb.h" -#include "components/feed/core/time_serialization.h" -#include "components/leveldb_proto/testing/fake_db.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" - -using leveldb_proto::test::FakeDB; -using testing::Mock; -using testing::NotNull; -using testing::_; - -namespace feed { - -namespace { -const std::string kContentKeyPrefix = "ContentKey"; -const std::string kContentKey1 = "ContentKey1"; -const std::string kContentKey2 = "ContentKey2"; -const std::string kContentKey3 = "ContentKey3"; -const std::string kContentData1 = "Content Data1"; -const std::string kContentData2 = "Content Data2"; -const std::string kContentData3 = "Content Data3"; -const std::string kJournalKey1 = "JournalKey1"; -const std::string kJournalKey2 = "JournalKey2"; -const std::string kJournalKey3 = "JournalKey3"; -const std::string kJournalData1 = "Journal Data1"; -const std::string kJournalData2 = "Journal Data2"; -const std::string kJournalData3 = "Journal Data3"; -const std::string kJournalData4 = "Journal Data4"; -const std::string kJournalData5 = "Journal Data5"; -const std::string kJournalData6 = "Journal Data6"; -} // namespace - -class FeedStorageDatabaseTest : public testing::Test { - public: - FeedStorageDatabaseTest() : storage_db_(nullptr) {} - - void CreateDatabase(bool init_database) { - // The FakeDBs are owned by |feed_db_|, so clear our pointers before - // resetting |feed_db_| itself. - storage_db_ = nullptr; - // Explicitly destroy any existing database before creating a new one. - feed_db_.reset(); - - auto storage_db = - std::make_unique<FakeDB<FeedStorageProto>>(&storage_db_storage_); - - storage_db_ = storage_db.get(); - feed_db_ = std::make_unique<FeedStorageDatabase>(base::FilePath(), - std::move(storage_db)); - if (init_database) { - storage_db_->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - } - } - - void InjectContentStorageProto(const std::string& key, - const std::string& data) { - FeedStorageProto storage_proto; - storage_proto.set_key(key); - storage_proto.set_content_data(data); - storage_db_storage_["cs-" + key] = storage_proto; - } - - void InjectJournalStorageProto(const std::string& key, - const std::vector<std::string>& entries) { - FeedStorageProto storage_proto; - storage_proto.set_key(key); - for (const std::string& entry : entries) { - storage_proto.add_journal_data(entry); - } - storage_db_storage_["js-" + key] = storage_proto; - } - - void RunUntilIdle() { scoped_task_environment_.RunUntilIdle(); } - - FakeDB<FeedStorageProto>* storage_db() { return storage_db_; } - - FeedStorageDatabase* db() { return feed_db_.get(); } - - MOCK_METHOD1(OnContentEntriesReceived, - void(std::vector<std::pair<std::string, std::string>>)); - MOCK_METHOD1(OnContentKeyReceived, void(std::vector<std::string>)); - MOCK_METHOD1(OnJournalEntryReceived, void(std::vector<std::string>)); - MOCK_METHOD1(OnJournalEntriesReceived, - void(std::vector<std::vector<std::string>>)); - MOCK_METHOD1(OnStorageCommitted, void(bool)); - - private: - base::test::ScopedTaskEnvironment scoped_task_environment_; - - std::map<std::string, FeedStorageProto> storage_db_storage_; - - // Owned by |feed_db_|. - FakeDB<FeedStorageProto>* storage_db_; - - std::unique_ptr<FeedStorageDatabase> feed_db_; - - DISALLOW_COPY_AND_ASSIGN(FeedStorageDatabaseTest); -}; - -TEST_F(FeedStorageDatabaseTest, Init) { - ASSERT_FALSE(db()); - - CreateDatabase(/*init_database=*/false); - - storage_db()->InitCallback(true); - EXPECT_TRUE(db()->IsInitialized()); -} - -TEST_F(FeedStorageDatabaseTest, LoadContentAfterInitSuccess) { - CreateDatabase(/*init_database=*/true); - - EXPECT_CALL(*this, OnContentEntriesReceived(_)); - db()->LoadContent( - {kContentKey1}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, LoadContentsEntries) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1| and |kContentKey2|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - // Try to Load |kContentKey2| and |kContentKey3|, only |kContentKey2| should - // return. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - ASSERT_EQ(results.size(), 1U); - EXPECT_EQ(results[0].first, kContentKey2); - EXPECT_EQ(results[0].second, kContentData2); - }); - db()->LoadContent( - {kContentKey2, kContentKey3}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, LoadContentsEntriesByPrefix) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1|, |kContentKey2|, |kJournalKey1|, |kJournalKey2|, - // |kJournalKey3|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - // Try to Load "ContentKey", both |kContentKey1| and |kContentKey2| should - // return. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - ASSERT_EQ(results.size(), 2U); - EXPECT_EQ(results[0].first, kContentKey1); - EXPECT_EQ(results[0].second, kContentData1); - EXPECT_EQ(results[1].first, kContentKey2); - EXPECT_EQ(results[1].second, kContentData2); - }); - db()->LoadContentByPrefix( - kContentKeyPrefix, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, LoadAllContentKeys) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1|, |kContentKey2|, |kJournalKey1|, |kJournalKey2|, - // |kJournalKey3|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - EXPECT_CALL(*this, OnContentKeyReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 2U); - EXPECT_EQ(results[0], kContentKey1); - EXPECT_EQ(results[1], kContentKey2); - }); - db()->LoadAllContentKeys(base::BindOnce( - &FeedStorageDatabaseTest::OnContentKeyReceived, base::Unretained(this))); - storage_db()->LoadKeysCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, SaveContent) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1|, |kContentKey2|, |kJournalKey1|, |kJournalKey2|, - // |kJournalKey3|. - std::vector<std::pair<std::string, std::string>> entries; - entries.push_back(std::make_pair(kContentKey1, kContentData1)); - entries.push_back(std::make_pair(kContentKey2, kContentData2)); - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->SaveContent(std::move(entries), - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->UpdateCallback(true); - - // Make sure they're there. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - ASSERT_EQ(results.size(), 2U); - EXPECT_EQ(results[0].first, kContentKey1); - EXPECT_EQ(results[0].second, kContentData1); - EXPECT_EQ(results[1].first, kContentKey2); - EXPECT_EQ(results[1].second, kContentData2); - }); - db()->LoadContent( - {kContentKey1, kContentKey2}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, DeleteContent) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1| and |kContentKey2|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - - // Delete |kContentKey2| and |kContentKey3| - std::vector<std::string> keys; - keys.push_back(kContentKey2); - keys.push_back(kContentKey3); - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->DeleteContent( - std::move(keys), - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->UpdateCallback(true); - - // Make sure only |kContentKey2| got deleted. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - EXPECT_EQ(results.size(), 1U); - EXPECT_EQ(results[0].first, kContentKey1); - EXPECT_EQ(results[0].second, kContentData1); - }); - db()->LoadContent( - {kContentKey1, kContentKey2}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, DeleteContentByPrefix) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1| and |kContentKey2|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - - // Delete |kContentKey1| and |kContentKey2| - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->DeleteContentByPrefix( - kContentKeyPrefix, - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->UpdateCallback(true); - - // Make sure |kContentKey1| and |kContentKey2| got deleted. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - EXPECT_EQ(results.size(), 0U); - }); - db()->LoadContent( - {kContentKey1, kContentKey2}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, DeleteAllContent) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1| and |kContentKey2|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - - // Store |kJournalKey1|, |kJournalKey2|, |kJournalKey3|. - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - // Delete all content, meaning |kContentKey1| and |kContentKey2| are expected - // to be deleted. - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->DeleteAllContent(base::BindOnce( - &FeedStorageDatabaseTest::OnStorageCommitted, base::Unretained(this))); - storage_db()->UpdateCallback(true); - - // Make sure |kContentKey1| and |kContentKey2| got deleted. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - EXPECT_EQ(results.size(), 0U); - }); - db()->LoadContent( - {kContentKey1, kContentKey2}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); - - // Make sure all journals are there. - EXPECT_CALL(*this, OnJournalEntriesReceived(_)) - .WillOnce([](std::vector<std::vector<std::string>> results) { - ASSERT_EQ(results.size(), 3U); - }); - db()->LoadAllJournals( - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, LoadJournalEntry) { - CreateDatabase(/*init_database=*/true); - - // Store |kJournalKey1|. - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - - // Try to Load |kJournalKey1|. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 3U); - EXPECT_EQ(results[0], kJournalData1); - EXPECT_EQ(results[1], kJournalData2); - EXPECT_EQ(results[2], kJournalData3); - }); - db()->LoadJournal( - kJournalKey1, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, LoadNonExistingJournalEntry) { - CreateDatabase(/*init_database=*/true); - - // Try to Load |kJournalKey1|. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 0U); - }); - db()->LoadJournal( - kJournalKey1, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, LoadAllJournals) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1|, |kContentKey2|, |kJournalKey1|, |kJournalKey2|, - // |kJournalKey3|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - EXPECT_CALL(*this, OnJournalEntriesReceived(_)) - .WillOnce([](std::vector<std::vector<std::string>> results) { - ASSERT_EQ(results.size(), 3U); - EXPECT_EQ(results[0][0], kJournalData1); - EXPECT_EQ(results[0][1], kJournalData2); - EXPECT_EQ(results[0][2], kJournalData3); - EXPECT_EQ(results[1][0], kJournalData4); - EXPECT_EQ(results[1][1], kJournalData5); - EXPECT_EQ(results[2][0], kJournalData6); - }); - db()->LoadAllJournals( - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, AppendToJournal_WhenJournalExists) { - CreateDatabase(/*init_database=*/true); - - // Save |kContentKey1| - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->AppendToJournal( - kJournalKey1, {kJournalData1, kJournalData2}, - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->GetCallback(true); - storage_db()->UpdateCallback(true); - - // Make sure they're there. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 2U); - EXPECT_EQ(results[0], kJournalData1); - EXPECT_EQ(results[1], kJournalData2); - }); - db()->LoadJournal( - kJournalKey1, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); - - Mock::VerifyAndClearExpectations(this); - - // Append more for |kContentKey1| - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->AppendToJournal( - kJournalKey1, {kJournalData3, kJournalData4, kJournalData5}, - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->GetCallback(true); - storage_db()->UpdateCallback(true); - - // Check new instances are there. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 5U); - EXPECT_EQ(results[0], kJournalData1); - EXPECT_EQ(results[1], kJournalData2); - EXPECT_EQ(results[2], kJournalData3); - EXPECT_EQ(results[3], kJournalData4); - EXPECT_EQ(results[4], kJournalData5); - }); - db()->LoadJournal( - kJournalKey1, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, AppendToJournal_WhenJournalMissing) { - CreateDatabase(/*init_database=*/true); - - // Append data for |kContentKey1| - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->AppendToJournal( - kJournalKey1, {kJournalData1, kJournalData2, kJournalData3}, - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->GetCallback(true); - storage_db()->UpdateCallback(true); - - // Check new data are there. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 3U); - EXPECT_EQ(results[0], kJournalData1); - EXPECT_EQ(results[1], kJournalData2); - EXPECT_EQ(results[2], kJournalData3); - }); - db()->LoadJournal( - kJournalKey1, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, CopyJournal) { - CreateDatabase(/*init_database=*/true); - - // Save |kContentKey1|. - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - - // Copy |kContentKey1| to |kContentKey2|. - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->CopyJournal(kJournalKey1, kJournalKey2, - base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->GetCallback(true); - storage_db()->UpdateCallback(true); - - // Check new journal is there. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 3U); - EXPECT_EQ(results[0], kJournalData1); - EXPECT_EQ(results[1], kJournalData2); - EXPECT_EQ(results[2], kJournalData3); - }); - db()->LoadJournal( - kJournalKey2, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); - - Mock::VerifyAndClearExpectations(this); - - // Check first journal is still there. - EXPECT_CALL(*this, OnJournalEntryReceived(_)) - .WillOnce([](std::vector<std::string> results) { - ASSERT_EQ(results.size(), 3U); - EXPECT_EQ(results[0], kJournalData1); - EXPECT_EQ(results[1], kJournalData2); - EXPECT_EQ(results[2], kJournalData3); - }); - db()->LoadJournal( - kJournalKey1, - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntryReceived, - base::Unretained(this))); - storage_db()->GetCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, DeleteJournal) { - CreateDatabase(/*init_database=*/true); - - // Store |kJournalKey1|, |kJournalKey2|, |kJournalKey3|. - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - // Delete |kJournalKey2|. - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->DeleteJournal( - kJournalKey2, base::BindOnce(&FeedStorageDatabaseTest::OnStorageCommitted, - base::Unretained(this))); - storage_db()->UpdateCallback(true); - - // Make sure |kJournalKey2| got deleted. - EXPECT_CALL(*this, OnJournalEntriesReceived(_)) - .WillOnce([](std::vector<std::vector<std::string>> results) { - ASSERT_EQ(results.size(), 2U); - EXPECT_EQ(results[0][0], kJournalData1); - EXPECT_EQ(results[0][1], kJournalData2); - EXPECT_EQ(results[0][2], kJournalData3); - EXPECT_EQ(results[1][0], kJournalData6); - }); - db()->LoadAllJournals( - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -TEST_F(FeedStorageDatabaseTest, DeleteAllJournals) { - CreateDatabase(/*init_database=*/true); - - // Store |kContentKey1| and |kContentKey2|. - InjectContentStorageProto(kContentKey1, kContentData1); - InjectContentStorageProto(kContentKey2, kContentData2); - - // Store |kJournalKey1|, |kJournalKey2|, |kJournalKey3|. - InjectJournalStorageProto(kJournalKey1, - {kJournalData1, kJournalData2, kJournalData3}); - InjectJournalStorageProto(kJournalKey2, {kJournalData4, kJournalData5}); - InjectJournalStorageProto(kJournalKey3, {kJournalData6}); - - // Delete all journals, meaning |kJournalKey1|, |kJournalKey2| and - // |kJournalKey3| are expected to be deleted. - EXPECT_CALL(*this, OnStorageCommitted(true)); - db()->DeleteAllJournals(base::BindOnce( - &FeedStorageDatabaseTest::OnStorageCommitted, base::Unretained(this))); - storage_db()->UpdateCallback(true); - - // Make sure all journals got deleted. - EXPECT_CALL(*this, OnJournalEntriesReceived(_)) - .WillOnce([](std::vector<std::vector<std::string>> results) { - ASSERT_EQ(results.size(), 0U); - }); - db()->LoadAllJournals( - base::BindOnce(&FeedStorageDatabaseTest::OnJournalEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); - - // Make sure all content are still there. - EXPECT_CALL(*this, OnContentEntriesReceived(_)) - .WillOnce([](std::vector<std::pair<std::string, std::string>> results) { - ASSERT_EQ(results.size(), 2U); - EXPECT_EQ(results[0].first, kContentKey1); - EXPECT_EQ(results[0].second, kContentData1); - EXPECT_EQ(results[1].first, kContentKey2); - EXPECT_EQ(results[1].second, kContentData2); - }); - db()->LoadContent( - {kContentKey1, kContentKey2}, - base::BindOnce(&FeedStorageDatabaseTest::OnContentEntriesReceived, - base::Unretained(this))); - storage_db()->LoadCallback(true); -} - -} // namespace feed diff --git a/chromium/components/feed/core/pref_names.cc b/chromium/components/feed/core/pref_names.cc index 1976a47dfd9..fb14849ba0e 100644 --- a/chromium/components/feed/core/pref_names.cc +++ b/chromium/components/feed/core/pref_names.cc @@ -12,13 +12,16 @@ const char kBackgroundRefreshPeriod[] = "feed.background_refresh_period"; const char kLastFetchAttemptTime[] = "feed.last_fetch_attempt"; -const char kUserClassifierAverageNTPOpenedPerHour[] = - "feed.user_classifier.average_ntp_opened_per_hour"; +const char kThrottlerRequestCount[] = "feed.refresh_throttler.count"; +const char kThrottlerRequestsDay[] = "feed.refresh_throttler.day"; + +const char kUserClassifierAverageSuggestionsViwedPerHour[] = + "feed.user_classifier.average_suggestions_veiwed_per_hour"; const char kUserClassifierAverageSuggestionsUsedPerHour[] = "feed.user_classifier.average_suggestions_used_per_hour"; -const char kUserClassifierLastTimeToOpenNTP[] = - "feed.user_classifier.last_time_to_open_ntp"; +const char kUserClassifierLastTimeToViewSuggestions[] = + "feed.user_classifier.last_time_to_view_suggestions"; const char kUserClassifierLastTimeToUseSuggestions[] = "feed.user_classifier.last_time_to_use_suggestions"; diff --git a/chromium/components/feed/core/pref_names.h b/chromium/components/feed/core/pref_names.h index cd613634e08..d7a1d0f9c15 100644 --- a/chromium/components/feed/core/pref_names.h +++ b/chromium/components/feed/core/pref_names.h @@ -15,16 +15,23 @@ extern const char kBackgroundRefreshPeriod[]; // The pref name for the last time when a background fetch was attempted. extern const char kLastFetchAttemptTime[]; +// The pref name for today's count of RefreshThrottler requests, so far. +extern const char kThrottlerRequestCount[]; +// The pref name for the current day for the counter of RefreshThrottler's +// requests. +extern const char kThrottlerRequestsDay[]; + // The pref name for the discounted average number of browsing sessions per hour // that involve opening a new NTP. -extern const char kUserClassifierAverageNTPOpenedPerHour[]; +extern const char kUserClassifierAverageSuggestionsViwedPerHour[]; // The pref name for the discounted average number of browsing sessions per hour // that involve using content suggestions (i.e. opening one or clicking on the // "More" button). extern const char kUserClassifierAverageSuggestionsUsedPerHour[]; -// The pref name for the last time a new NTP was opened. -extern const char kUserClassifierLastTimeToOpenNTP[]; +// The pref name for the last time a surface was shown that displayed +// suggestions to the user. +extern const char kUserClassifierLastTimeToViewSuggestions[]; // The pref name for the last time content suggestions were used by the user. extern const char kUserClassifierLastTimeToUseSuggestions[]; diff --git a/chromium/components/feed/core/proto/BUILD.gn b/chromium/components/feed/core/proto/BUILD.gn index 98626a99163..2dfff8a2e5e 100644 --- a/chromium/components/feed/core/proto/BUILD.gn +++ b/chromium/components/feed/core/proto/BUILD.gn @@ -7,6 +7,7 @@ import("//third_party/protobuf/proto_library.gni") proto_library("proto") { sources = [ "cached_image.proto", - "feed_storage.proto", + "content_storage.proto", + "journal_storage.proto", ] } diff --git a/chromium/components/feed/core/proto/feed_storage.proto b/chromium/components/feed/core/proto/content_storage.proto index 6796b2b798c..750a15cb610 100644 --- a/chromium/components/feed/core/proto/feed_storage.proto +++ b/chromium/components/feed/core/proto/content_storage.proto @@ -8,13 +8,11 @@ option optimize_for = LITE_RUNTIME; package feed; -message FeedStorageProto { - // original key for data. +// Used for storing content data in content storage. +message ContentStorageProto { + // Original key for data. optional string key = 1; // Content data. optional bytes content_data = 2; - - // Journal data. - repeated bytes journal_data = 3; } diff --git a/chromium/components/feed/core/proto/journal_storage.proto b/chromium/components/feed/core/proto/journal_storage.proto new file mode 100644 index 00000000000..9c5331a0972 --- /dev/null +++ b/chromium/components/feed/core/proto/journal_storage.proto @@ -0,0 +1,18 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package feed; + +// Used for storing journal data in journal storage. +message JournalStorageProto { + // Original key for data. + optional string key = 1; + + // Journal data. + repeated bytes journal_data = 2; +} diff --git a/chromium/components/feed/core/refresh_throttler.cc b/chromium/components/feed/core/refresh_throttler.cc new file mode 100644 index 00000000000..06712f9b4b9 --- /dev/null +++ b/chromium/components/feed/core/refresh_throttler.cc @@ -0,0 +1,146 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/refresh_throttler.h" + +#include <limits> +#include <set> +#include <utility> +#include <vector> + +#include "base/metrics/field_trial_params.h" +#include "base/metrics/histogram.h" +#include "base/metrics/histogram_base.h" +#include "base/strings/stringprintf.h" +#include "base/time/clock.h" +#include "components/feed/core/pref_names.h" +#include "components/feed/feed_feature_list.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" + +namespace feed { + +namespace { + +// Values correspond to ntp_snippets::RequestStatus and histograms.xml +enum class RequestStatus { + kObsolete1 = 0, + kQuotaGranted = 1, + kQuotaExceeded = 2, + kObsolete2 = 3, + kStatusCount = 4 +}; + +// When adding a new type here, extend also the "RequestThrottlerTypes" +// <histogram_suffixes> in histograms.xml with the |name| string. First value in +// the pair is the name, second is the default requests per day. +std::pair<std::string, int> GetThrottlerParams( + UserClassifier::UserClass user_class) { + switch (user_class) { + case UserClassifier::UserClass::kRareSuggestionsViewer: + return {"SuggestionFetcherRareNTPUser", 5}; + case UserClassifier::UserClass::kActiveSuggestionsViewer: + return {"SuggestionFetcherActiveNTPUser", 20}; + case UserClassifier::UserClass::kActiveSuggestionsConsumer: + return {"SuggestionFetcherActiveSuggestionsConsumer", 20}; + } +} + +} // namespace + +RefreshThrottler::RefreshThrottler(UserClassifier::UserClass user_class, + PrefService* pref_service, + base::Clock* clock) + : pref_service_(pref_service), clock_(clock) { + DCHECK(pref_service); + DCHECK(clock); + + std::pair<std::string, int> throttler_params = GetThrottlerParams(user_class); + name_ = throttler_params.first; + max_requests_per_day_ = base::GetFieldTrialParamByFeatureAsInt( + kInterestFeedContentSuggestions, + base::StringPrintf("quota_%s", name_.c_str()), throttler_params.second); + + // Since the histogram names are dynamic, we cannot use the standard macros + // and we need to lookup the histograms, instead. + int status_count = static_cast<int>(RequestStatus::kStatusCount); + // Corresponds to UMA_HISTOGRAM_ENUMERATION(name, sample, |status_count|). + histogram_request_status_ = base::LinearHistogram::FactoryGet( + base::StringPrintf("NewTabPage.RequestThrottler.RequestStatus_%s", + name_.c_str()), + 1, status_count, status_count + 1, + base::HistogramBase::kUmaTargetedHistogramFlag); + // Corresponds to UMA_HISTOGRAM_COUNTS_100(name, sample). + histogram_per_day_ = base::Histogram::FactoryGet( + base::StringPrintf("NewTabPage.RequestThrottler.PerDay_%s", + name_.c_str()), + 1, 100, 50, base::HistogramBase::kUmaTargetedHistogramFlag); +} + +// static +void RefreshThrottler::RegisterProfilePrefs(PrefRegistrySimple* registry) { + registry->RegisterIntegerPref(prefs::kThrottlerRequestCount, 0); + registry->RegisterIntegerPref(prefs::kThrottlerRequestsDay, 0); +} + +bool RefreshThrottler::RequestQuota() { + ResetCounterIfDayChanged(); + + // Increment |new_count| in a overflow safe fashion. + int new_count = GetCount(); + if (new_count < std::numeric_limits<int>::max()) { + new_count++; + } + SetCount(new_count); + bool available = (new_count <= GetQuota()); + + histogram_request_status_->Add( + static_cast<int>(available ? RequestStatus::kQuotaGranted + : RequestStatus::kQuotaExceeded)); + + return available; +} + +void RefreshThrottler::ResetCounterIfDayChanged() { + // Grant new quota on local midnight to spread out when clients that start + // making un-throttled requests to server. + int now_day = clock_->Now().LocalMidnight().since_origin().InDays(); + + if (!HasDay()) { + // The counter is used for the first time in this profile. + SetDay(now_day); + } else if (now_day != GetDay()) { + // Day has changed - report the number of requests from the previous day. + histogram_per_day_->Add(GetCount()); + // Reset the counters. + SetCount(0); + SetDay(now_day); + } +} + +int RefreshThrottler::GetQuota() const { + return max_requests_per_day_; +} + +int RefreshThrottler::GetCount() const { + return pref_service_->GetInteger(prefs::kThrottlerRequestCount); +} + +void RefreshThrottler::SetCount(int count) { + pref_service_->SetInteger(prefs::kThrottlerRequestCount, count); +} + +int RefreshThrottler::GetDay() const { + return pref_service_->GetInteger(prefs::kThrottlerRequestsDay); +} + +void RefreshThrottler::SetDay(int day) { + pref_service_->SetInteger(prefs::kThrottlerRequestsDay, day); +} + +bool RefreshThrottler::HasDay() const { + return pref_service_->HasPrefPath(prefs::kThrottlerRequestsDay); +} + +} // namespace feed diff --git a/chromium/components/feed/core/refresh_throttler.h b/chromium/components/feed/core/refresh_throttler.h new file mode 100644 index 00000000000..c659ad4c1e3 --- /dev/null +++ b/chromium/components/feed/core/refresh_throttler.h @@ -0,0 +1,78 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_REFRESH_THROTTLER_H_ +#define COMPONENTS_FEED_CORE_REFRESH_THROTTLER_H_ + +#include <string> + +#include "base/macros.h" +#include "components/feed/core/user_classifier.h" + +class PrefRegistrySimple; +class PrefService; + +namespace base { +class Clock; +class HistogramBase; +} // namespace base + +namespace feed { + +// Counts requests to perform refreshes, compares them to a daily quota, and +// reports them to UMA. In the application code, create one local instance for +// each given throttler name, identified by the UserClass. Reports to the same +// histograms that previous NTP implementation used: +// - "NewTabPage.RequestThrottler.RequestStatus_|name|" - status of each +// request; +// - "NewTabPage.RequestThrottler.PerDay_|name|" - the daily count of requests. +class RefreshThrottler { + public: + RefreshThrottler(UserClassifier::UserClass user_class, + PrefService* pref_service, + base::Clock* clock); + + // Registers profile prefs, called from browser_prefs.cc. + static void RegisterProfilePrefs(PrefRegistrySimple* registry); + + // Returns whether quota is available for another request, persists the usage + // of said quota, and reports this information to UMA. + bool RequestQuota(); + + private: + // Also emits the PerDay histogram if the day changed. + void ResetCounterIfDayChanged(); + + int GetQuota() const; + int GetCount() const; + void SetCount(int count); + int GetDay() const; + void SetDay(int day); + bool HasDay() const; + + // Provides durable storage. + PrefService* pref_service_; + + // Used to access current time, injected for testing. + base::Clock* clock_; + + // The name used by this throttler, based off UserClass, which will be used as + // a suffix when constructing histogram or finch param names. + std::string name_; + + // The total requests allowed before RequestQuota() starts returning false, + // reset every time |clock_| changes days. Read from a variation param during + // initialization. + int max_requests_per_day_; + + // The histograms for reporting the requests of the given |type_|. + base::HistogramBase* histogram_request_status_; + base::HistogramBase* histogram_per_day_; + + DISALLOW_COPY_AND_ASSIGN(RefreshThrottler); +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_REFRESH_THROTTLER_H_ diff --git a/chromium/components/feed/core/refresh_throttler_unittest.cc b/chromium/components/feed/core/refresh_throttler_unittest.cc new file mode 100644 index 00000000000..6631d69c1fe --- /dev/null +++ b/chromium/components/feed/core/refresh_throttler_unittest.cc @@ -0,0 +1,82 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/refresh_throttler.h" + +#include <limits> +#include <memory> + +#include "base/test/simple_test_clock.h" +#include "components/feed/core/pref_names.h" +#include "components/feed/core/user_classifier.h" +#include "components/feed/feed_feature_list.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/testing_pref_service.h" +#include "components/variations/variations_params_manager.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { + +class RefreshThrottlerTest : public testing::Test { + public: + RefreshThrottlerTest() { + RefreshThrottler::RegisterProfilePrefs(test_prefs_.registry()); + test_clock_.SetNow(base::Time::Now()); + + variations::testing::VariationParamsManager variation_params( + kInterestFeedContentSuggestions.name, + {{"quota_SuggestionFetcherActiveNTPUser", "2"}}, + {kInterestFeedContentSuggestions.name}); + + throttler_ = std::make_unique<RefreshThrottler>( + UserClassifier::UserClass::kActiveSuggestionsViewer, &test_prefs_, + &test_clock_); + } + + protected: + TestingPrefServiceSimple test_prefs_; + base::SimpleTestClock test_clock_; + std::unique_ptr<RefreshThrottler> throttler_; + + private: + DISALLOW_COPY_AND_ASSIGN(RefreshThrottlerTest); +}; + +TEST_F(RefreshThrottlerTest, QuotaExceeded) { + EXPECT_TRUE(throttler_->RequestQuota()); + EXPECT_TRUE(throttler_->RequestQuota()); + EXPECT_FALSE(throttler_->RequestQuota()); +} + +TEST_F(RefreshThrottlerTest, QuotaIsPerDay) { + EXPECT_TRUE(throttler_->RequestQuota()); + EXPECT_TRUE(throttler_->RequestQuota()); + EXPECT_FALSE(throttler_->RequestQuota()); + + test_clock_.Advance(base::TimeDelta::FromDays(1)); + EXPECT_TRUE(throttler_->RequestQuota()); +} + +TEST_F(RefreshThrottlerTest, RollOver) { + // Exhaust our quota so the for loop can verify everything as false. + EXPECT_TRUE(throttler_->RequestQuota()); + EXPECT_TRUE(throttler_->RequestQuota()); + + test_clock_.SetNow(test_clock_.Now().LocalMidnight()); + for (int i = 0; i < 24; i++) { + EXPECT_FALSE(throttler_->RequestQuota()); + test_clock_.Advance(base::TimeDelta::FromHours(1)); + } + EXPECT_TRUE(throttler_->RequestQuota()); +} + +TEST_F(RefreshThrottlerTest, Overflow) { + test_prefs_.SetInteger(prefs::kThrottlerRequestCount, + std::numeric_limits<int>::max()); + EXPECT_FALSE(throttler_->RequestQuota()); + EXPECT_EQ(std::numeric_limits<int>::max(), + test_prefs_.GetInteger(prefs::kThrottlerRequestCount)); +} + +} // namespace feed diff --git a/chromium/components/feed/core/user_classifier.cc b/chromium/components/feed/core/user_classifier.cc index 5e36c13a04a..a8b3cdf4acd 100644 --- a/chromium/components/feed/core/user_classifier.cc +++ b/chromium/components/feed/core/user_classifier.cc @@ -45,11 +45,12 @@ const char kMinHoursParam[] = "user_classifier_min_hours"; const double kActiveConsumerClicksAtLeastOncePerHours = 96; const char kActiveConsumerClicksAtLeastOncePerHoursParam[] = "user_classifier_active_consumer_clicks_at_least_once_per_hours"; -const double kRareUserOpensNTPAtMostOncePerHours = 96; -const char kRareUserOpensNTPAtMostOncePerHoursParam[] = - "user_classifier_rare_user_opens_ntp_at_most_once_per_hours"; +const double kRareUserViewsAtMostOncePerHours = 96; +const char kRareUserViewsAtMostOncePerHoursParam[] = + "user_classifier_rare_user_views_at_most_once_per_hours"; -// Histograms for logging the estimated average hours to next event. +// Histograms for logging the estimated average hours to next event. During +// launch these must match legacy histogram names. const char kHistogramAverageHoursToOpenNTP[] = "NewTabPage.UserClassifier.AverageHoursToOpenNTP"; const char kHistogramAverageHoursToUseSuggestions[] = @@ -57,18 +58,19 @@ const char kHistogramAverageHoursToUseSuggestions[] = // List of all Events used for iteration. const UserClassifier::Event kEvents[] = { - UserClassifier::Event::kNtpOpened, UserClassifier::Event::kSuggestionsUsed}; + UserClassifier::Event::kSuggestionsViewed, + UserClassifier::Event::kSuggestionsUsed}; // Arrays of pref names, indexed by Event's int value. -const char* kRateKeys[] = {prefs::kUserClassifierAverageNTPOpenedPerHour, +const char* kRateKeys[] = {prefs::kUserClassifierAverageSuggestionsViwedPerHour, prefs::kUserClassifierAverageSuggestionsUsedPerHour}; -const char* kLastTimeKeys[] = {prefs::kUserClassifierLastTimeToOpenNTP, +const char* kLastTimeKeys[] = {prefs::kUserClassifierLastTimeToViewSuggestions, prefs::kUserClassifierLastTimeToUseSuggestions}; // Default lengths of the intervals for new users for the events. const double kInitialHoursBetweenEvents[] = {24, 120}; const char* kInitialHoursBetweenEventsParams[] = { - "user_classifier_default_interval_ntp_opened", + "user_classifier_default_interval_suggestions_viewed", "user_classifier_default_interval_suggestions_used"}; // This verifies that each of the arrays has exactly the same number of values @@ -185,11 +187,11 @@ UserClassifier::UserClassifier(PrefService* pref_service, base::Clock* clock) kInterestFeedContentSuggestions, kActiveConsumerClicksAtLeastOncePerHoursParam, kActiveConsumerClicksAtLeastOncePerHours)), - rare_user_opens_ntp_at_most_once_per_hours_( + rare_viewer_opens_surface_at_most_once_per_hours_( variations::GetVariationParamByFeatureAsDouble( kInterestFeedContentSuggestions, - kRareUserOpensNTPAtMostOncePerHoursParam, - kRareUserOpensNTPAtMostOncePerHours)) { + kRareUserViewsAtMostOncePerHoursParam, + kRareUserViewsAtMostOncePerHours)) { // The pref_service_ can be null in tests. if (!pref_service_) { return; @@ -233,7 +235,7 @@ void UserClassifier::OnEvent(Event event) { // We use kMaxHours as the max value below as the maximum value for the // histograms must be constant. switch (event) { - case Event::kNtpOpened: + case Event::kSuggestionsViewed: UMA_HISTOGRAM_CUSTOM_COUNTS(kHistogramAverageHoursToOpenNTP, avg, 1, kMaxHours, 50); break; @@ -253,12 +255,12 @@ double UserClassifier::GetEstimatedAvgTime(Event event) const { UserClassifier::UserClass UserClassifier::GetUserClass() const { // The pref_service_ can be null in tests. if (!pref_service_) { - return UserClass::kActiveNtpUser; + return UserClass::kActiveSuggestionsViewer; } - if (GetEstimatedAvgTime(Event::kNtpOpened) >= - rare_user_opens_ntp_at_most_once_per_hours_) { - return UserClass::kRareNtpUser; + if (GetEstimatedAvgTime(Event::kSuggestionsViewed) >= + rare_viewer_opens_surface_at_most_once_per_hours_) { + return UserClass::kRareSuggestionsViewer; } if (GetEstimatedAvgTime(Event::kSuggestionsUsed) <= @@ -266,17 +268,17 @@ UserClassifier::UserClass UserClassifier::GetUserClass() const { return UserClass::kActiveSuggestionsConsumer; } - return UserClass::kActiveNtpUser; + return UserClass::kActiveSuggestionsViewer; } std::string UserClassifier::GetUserClassDescriptionForDebugging() const { switch (GetUserClass()) { - case UserClass::kRareNtpUser: - return "Rare user of the NTP"; - case UserClass::kActiveNtpUser: - return "Active user of the NTP"; + case UserClass::kRareSuggestionsViewer: + return "Rare viewer of Feed articles"; + case UserClass::kActiveSuggestionsViewer: + return "Active viewer of Feed articles"; case UserClass::kActiveSuggestionsConsumer: - return "Active consumer of NTP articles"; + return "Active consumer of Feed articles"; } NOTREACHED(); return std::string(); diff --git a/chromium/components/feed/core/user_classifier.h b/chromium/components/feed/core/user_classifier.h index 309f4741c66..9e9b72ef4df 100644 --- a/chromium/components/feed/core/user_classifier.h +++ b/chromium/components/feed/core/user_classifier.h @@ -27,8 +27,10 @@ class UserClassifier { // Different groupings of usage. A user will belong to exactly one of these at // any given point in time. Can change at runtime. enum class UserClass { - kRareNtpUser, // Almost never opens the NTP. - kActiveNtpUser, // Uses NTP but not articles. + kRareSuggestionsViewer, // Almost never opens surfaces that show + // suggestions, like the NTP. + kActiveSuggestionsViewer, // Frequently shown suggestions, but does not + // usually open them. kActiveSuggestionsConsumer, // Frequently opens news articles. }; @@ -44,10 +46,11 @@ class UserClassifier { // NOTE: if you add any element, add it also in the static arrays in .cc and // create another histogram. enum class Event { - kNtpOpened = 0, // When the user opens a new NTP - this indicates potential - // use of content suggestions. - kSuggestionsUsed = 1, // When the user clicks on some suggestions or on - // some "More" button. + kSuggestionsViewed = 0, // When the user opens a surface that is showing + // suggestions, such as the NTP. This indicates + // potential use of content suggestions. + kSuggestionsUsed = 1, // When the user clicks on some suggestions or on + // the "More" button. kMaxValue = kSuggestionsUsed }; @@ -101,7 +104,7 @@ class UserClassifier { // Params of the classification. const double active_consumer_clicks_at_least_once_per_hours_; - const double rare_user_opens_ntp_at_most_once_per_hours_; + const double rare_viewer_opens_surface_at_most_once_per_hours_; DISALLOW_COPY_AND_ASSIGN(UserClassifier); }; diff --git a/chromium/components/feed/core/user_classifier_unittest.cc b/chromium/components/feed/core/user_classifier_unittest.cc index 705e69fc75a..8a8eb0302fe 100644 --- a/chromium/components/feed/core/user_classifier_unittest.cc +++ b/chromium/components/feed/core/user_classifier_unittest.cc @@ -56,10 +56,10 @@ class FeedUserClassifierTest : public testing::Test { DISALLOW_COPY_AND_ASSIGN(FeedUserClassifierTest); }; -TEST_F(FeedUserClassifierTest, ShouldBeActiveNtpUserInitially) { +TEST_F(FeedUserClassifierTest, ShouldBeActiveSuggestionsViewerInitially) { UserClassifier* user_classifier = CreateUserClassifier(); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveNtpUser)); + Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); } TEST_F(FeedUserClassifierTest, @@ -69,7 +69,7 @@ TEST_F(FeedUserClassifierTest, // After one click still only an active user. user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveNtpUser)); + Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); // After a few more clicks, become an active consumer. for (int i = 0; i < 5; i++) { @@ -95,7 +95,7 @@ TEST_F(FeedUserClassifierTest, test_clock()->Advance(base::TimeDelta::FromHours(1)); user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveNtpUser)); + Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); // One more click to become an active consumer. test_clock()->Advance(base::TimeDelta::FromHours(1)); @@ -104,38 +104,39 @@ TEST_F(FeedUserClassifierTest, Eq(UserClassifier::UserClass::kActiveSuggestionsConsumer)); } -TEST_F(FeedUserClassifierTest, ShouldBecomeRareNtpUserByNoActivity) { +TEST_F(FeedUserClassifierTest, + ShouldBecomeRareSuggestionsViewerUserByNoActivity) { UserClassifier* user_classifier = CreateUserClassifier(); // After two days of waiting still an active user. test_clock()->Advance(base::TimeDelta::FromDays(2)); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveNtpUser)); + Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); // Two more days to become a rare user. test_clock()->Advance(base::TimeDelta::FromDays(2)); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kRareNtpUser)); + Eq(UserClassifier::UserClass::kRareSuggestionsViewer)); } TEST_F(FeedUserClassifierTest, - ShouldBecomeRareNtpUserByNoActivityWithDecreasedParam) { + ShouldBecomeRareSuggestionsViewerByNoActivityWithDecreasedParam) { // Decrease the param to one half. variations::testing::VariationParamsManager variation_params( kInterestFeedContentSuggestions.name, - {{"user_classifier_rare_user_opens_ntp_at_most_once_per_hours", "48"}}, + {{"user_classifier_rare_user_views_at_most_once_per_hours", "48"}}, {kInterestFeedContentSuggestions.name}); UserClassifier* user_classifier = CreateUserClassifier(); // After one days of waiting still an active user. test_clock()->Advance(base::TimeDelta::FromDays(1)); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveNtpUser)); + Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); // One more day to become a rare user. test_clock()->Advance(base::TimeDelta::FromDays(1)); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kRareNtpUser)); + Eq(UserClassifier::UserClass::kRareSuggestionsViewer)); } class FeedUserClassifierEventTest @@ -300,7 +301,7 @@ INSTANTIATE_TEST_CASE_P( // distinguish the only instance we make here). FeedUserClassifierEventTest, testing::Values( - std::make_pair(UserClassifier::Event::kNtpOpened, + std::make_pair(UserClassifier::Event::kSuggestionsViewed, "NewTabPage.UserClassifier.AverageHoursToOpenNTP"), std::make_pair( UserClassifier::Event::kSuggestionsUsed, diff --git a/chromium/components/feed/features.gni b/chromium/components/feed/features.gni index d61856bd8f3..4d55a71441b 100644 --- a/chromium/components/feed/features.gni +++ b/chromium/components/feed/features.gni @@ -3,6 +3,6 @@ # found in the LICENSE file. declare_args() { - # Temporarily compile out Feed while M69 branches to avoid bloating binary. + # Temporarily compile out Feed while M70 branches to avoid bloating binary. enable_feed_in_chrome = false } |