diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2019-05-16 09:59:13 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2019-05-20 10:28:53 +0000 |
commit | 6c11fb357ec39bf087b8b632e2b1e375aef1b38b (patch) | |
tree | c8315530db18a8ee566521c39ab8a6af4f72bc03 /chromium/components/feed | |
parent | 3ffaed019d0772e59d6cdb2d0d32fe4834c31f72 (diff) | |
download | qtwebengine-chromium-6c11fb357ec39bf087b8b632e2b1e375aef1b38b.tar.gz |
BASELINE: Update Chromium to 74.0.3729.159
Change-Id: I8d2497da544c275415aedd94dd25328d555de811
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/feed')
27 files changed, 547 insertions, 1251 deletions
diff --git a/chromium/components/feed/OWNERS b/chromium/components/feed/OWNERS index 90e498d6f46..2efa28a3bd3 100644 --- a/chromium/components/feed/OWNERS +++ b/chromium/components/feed/OWNERS @@ -4,6 +4,5 @@ pnoland@chromium.org skym@chromium.org zea@chromium.org -per-file *Test.java=aluo@chromium.org # Team: chrome-jardin-team@google.com # COMPONENT: UI>Browser>ContentSuggestions>Feed diff --git a/chromium/components/feed/content/feed_host_service.cc b/chromium/components/feed/content/feed_host_service.cc index d90e7f201ff..91ee7949cbe 100644 --- a/chromium/components/feed/content/feed_host_service.cc +++ b/chromium/components/feed/content/feed_host_service.cc @@ -10,14 +10,12 @@ namespace feed { FeedHostService::FeedHostService( std::unique_ptr<FeedLoggingMetrics> logging_metrics, - std::unique_ptr<FeedImageManager> image_manager, std::unique_ptr<FeedNetworkingHost> networking_host, std::unique_ptr<FeedSchedulerHost> scheduler_host, std::unique_ptr<FeedContentDatabase> content_database, std::unique_ptr<FeedJournalDatabase> journal_database, std::unique_ptr<FeedOfflineHost> offline_host) : logging_metrics_(std::move(logging_metrics)), - image_manager_(std::move(image_manager)), networking_host_(std::move(networking_host)), scheduler_host_(std::move(scheduler_host)), content_database_(std::move(content_database)), @@ -26,10 +24,6 @@ FeedHostService::FeedHostService( FeedHostService::~FeedHostService() = default; -FeedImageManager* FeedHostService::GetImageManager() { - return image_manager_.get(); -} - FeedNetworkingHost* FeedHostService::GetNetworkingHost() { return networking_host_.get(); } diff --git a/chromium/components/feed/content/feed_host_service.h b/chromium/components/feed/content/feed_host_service.h index bae43d5a577..0550994f6bd 100644 --- a/chromium/components/feed/content/feed_host_service.h +++ b/chromium/components/feed/content/feed_host_service.h @@ -10,7 +10,6 @@ #include "base/macros.h" #include "components/feed/content/feed_offline_host.h" #include "components/feed/core/feed_content_database.h" -#include "components/feed/core/feed_image_manager.h" #include "components/feed/core/feed_journal_database.h" #include "components/feed/core/feed_logging_metrics.h" #include "components/feed/core/feed_networking_host.h" @@ -27,7 +26,6 @@ namespace feed { class FeedHostService : public KeyedService { public: FeedHostService(std::unique_ptr<FeedLoggingMetrics> logging_metrics, - std::unique_ptr<FeedImageManager> image_manager, std::unique_ptr<FeedNetworkingHost> networking_host, std::unique_ptr<FeedSchedulerHost> scheduler_host, std::unique_ptr<FeedContentDatabase> content_database, @@ -36,7 +34,6 @@ class FeedHostService : public KeyedService { ~FeedHostService() override; FeedLoggingMetrics* GetLoggingMetrics(); - FeedImageManager* GetImageManager(); FeedNetworkingHost* GetNetworkingHost(); FeedSchedulerHost* GetSchedulerHost(); FeedContentDatabase* GetContentDatabase(); @@ -45,7 +42,6 @@ class FeedHostService : public KeyedService { private: std::unique_ptr<FeedLoggingMetrics> logging_metrics_; - std::unique_ptr<FeedImageManager> image_manager_; std::unique_ptr<FeedNetworkingHost> networking_host_; std::unique_ptr<FeedSchedulerHost> scheduler_host_; std::unique_ptr<FeedContentDatabase> content_database_; diff --git a/chromium/components/feed/content/feed_offline_host.cc b/chromium/components/feed/content/feed_offline_host.cc index b47ebda1606..d1b0dfd9da6 100644 --- a/chromium/components/feed/content/feed_offline_host.cc +++ b/chromium/components/feed/content/feed_offline_host.cc @@ -25,12 +25,6 @@ using offline_pages::SuggestionsProvider; namespace { -// |url| is always set. Sometimes |original_url| is set. If |original_url| is -// set it is returned by this method, otherwise fall back to |url|. -const GURL& PreferOriginal(const OfflinePageItem& item) { - return item.original_url.is_empty() ? item.url : item.original_url; -} - // Aggregates multiple callbacks from OfflinePageModel, storing the offline url. // When all callbacks have been invoked, tracked by ref counting, then // |on_completeion_| is finally invoked, sending all results together. @@ -255,7 +249,7 @@ void FeedOfflineHost::OfflinePageModelLoaded(OfflinePageModel* model) { void FeedOfflineHost::OfflinePageAdded(OfflinePageModel* model, const OfflinePageItem& added_page) { DCHECK(!notify_status_change_.is_null()); - const std::string& url = PreferOriginal(added_page).spec(); + const std::string& url = added_page.GetOriginalUrl().spec(); CacheOfflinePageUrlAndId(url, added_page.offline_id); notify_status_change_.Run(url, true); } diff --git a/chromium/components/feed/content/feed_offline_host_unittest.cc b/chromium/components/feed/content/feed_offline_host_unittest.cc index 6d03c1354c7..d2e052ffaa8 100644 --- a/chromium/components/feed/content/feed_offline_host_unittest.cc +++ b/chromium/components/feed/content/feed_offline_host_unittest.cc @@ -57,7 +57,7 @@ class TestOfflinePageModel : public StubOfflinePageModel { std::string name_space) { OfflinePageItem item; item.url = GURL(url); - item.original_url = GURL(original_url); + item.original_url_if_different = GURL(original_url); item.offline_id = offline_id; item.creation_time = creation_time; item.client_id = offline_pages::ClientId(name_space, ""); @@ -373,7 +373,7 @@ TEST_F(FeedOfflineHostTest, GetCurrentArticleSuggestionsMultiple) { TEST_F(FeedOfflineHostTest, OfflinePageAdded) { OfflinePageItem added_page; added_page.url = GURL(kUrl1); - added_page.original_url = GURL(kUrl2); + added_page.original_url_if_different = GURL(kUrl2); added_page.offline_id = 4; host()->OfflinePageAdded(nullptr, added_page); diff --git a/chromium/components/feed/core/BUILD.gn b/chromium/components/feed/core/BUILD.gn index 68e34e068bb..4d7b5a36a2f 100644 --- a/chromium/components/feed/core/BUILD.gn +++ b/chromium/components/feed/core/BUILD.gn @@ -16,10 +16,6 @@ source_set("feed_core") { "feed_content_mutation.h", "feed_content_operation.cc", "feed_content_operation.h", - "feed_image_database.cc", - "feed_image_database.h", - "feed_image_manager.cc", - "feed_image_manager.h", "feed_journal_database.cc", "feed_journal_database.h", "feed_journal_mutation.cc", @@ -46,7 +42,6 @@ source_set("feed_core") { "//base", "//components/feed:feature_list", "//components/feed/core/proto", - "//components/image_fetcher/core:core", "//components/leveldb_proto", "//net", "//ui/base/mojo:mojo", @@ -80,8 +75,6 @@ source_set("core_unit_tests") { sources = [ "feed_content_database_unittest.cc", "feed_content_mutation_unittest.cc", - "feed_image_database_unittest.cc", - "feed_image_manager_unittest.cc", "feed_journal_database_unittest.cc", "feed_journal_mutation_unittest.cc", "feed_logging_metrics_unittest.cc", diff --git a/chromium/components/feed/core/feed_content_database.cc b/chromium/components/feed/core/feed_content_database.cc index 7e06660d433..a9b760aa1e4 100644 --- a/chromium/components/feed/core/feed_content_database.cc +++ b/chromium/components/feed/core/feed_content_database.cc @@ -36,6 +36,12 @@ const char kContentDatabaseFolder[] = "content"; const size_t kDatabaseWriteBufferSizeBytes = 64 * 1024; // 64KB const size_t kDatabaseWriteBufferSizeBytesForLowEndDevice = 32 * 1024; // 32KB +leveldb::ReadOptions CreateReadOptions() { + leveldb::ReadOptions opts; + opts.fill_cache = false; + return opts; +} + bool DatabaseKeyFilter(const std::unordered_set<std::string>& key_set, const std::string& key) { return key_set.find(key) != key_set.end(); @@ -97,6 +103,7 @@ void FeedContentDatabase::LoadContent(const std::vector<std::string>& keys, storage_database_->LoadEntriesWithFilter( base::BindRepeating(&DatabaseKeyFilter, std::move(key_set)), + CreateReadOptions(), /* target_prefix */ "", base::BindOnce(&FeedContentDatabase::OnLoadEntriesForLoadContent, weak_ptr_factory_.GetWeakPtr(), base::TimeTicks::Now(), std::move(callback))); @@ -108,6 +115,7 @@ void FeedContentDatabase::LoadContentByPrefix(const std::string& prefix, storage_database_->LoadEntriesWithFilter( base::BindRepeating(&DatabasePrefixFilter, std::move(prefix)), + CreateReadOptions(), /* target_prefix */ "", base::BindOnce(&FeedContentDatabase::OnLoadEntriesForLoadContent, weak_ptr_factory_.GetWeakPtr(), base::TimeTicks::Now(), std::move(callback))); diff --git a/chromium/components/feed/core/feed_content_database_unittest.cc b/chromium/components/feed/core/feed_content_database_unittest.cc index aa662291b44..3a8a6a06738 100644 --- a/chromium/components/feed/core/feed_content_database_unittest.cc +++ b/chromium/components/feed/core/feed_content_database_unittest.cc @@ -6,6 +6,7 @@ #include <map> +#include "base/bind.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/scoped_task_environment.h" #include "components/feed/core/feed_content_mutation.h" diff --git a/chromium/components/feed/core/feed_image_database.cc b/chromium/components/feed/core/feed_image_database.cc deleted file mode 100644 index 57601876e2e..00000000000 --- a/chromium/components/feed/core/feed_image_database.cc +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/feed/core/feed_image_database.h" - -#include "base/bind.h" -#include "base/logging.h" -#include "base/sequenced_task_runner.h" -#include "base/system/sys_info.h" -#include "base/task/post_task.h" -#include "base/task/task_traits.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/time/clock.h" -#include "base/time/default_clock.h" -#include "base/time/time.h" -#include "components/feed/core/proto/cached_image.pb.h" -#include "components/feed/core/time_serialization.h" -#include "components/leveldb_proto/public/proto_database_provider.h" - -namespace feed { - -namespace { -// Statistics are logged to UMA with this string as part of histogram name. They -// can all be found under LevelDB.*.FeedImageDatabase. Changing this needs to -// synchronize with histograms.xml, AND will also become incompatible with older -// browsers still reporting the previous values. -const char kImageDatabaseUMAClientName[] = "FeedImageDatabase"; - -const char kImageDatabaseFolder[] = "images"; - -const size_t kDatabaseWriteBufferSizeBytes = 512 * 1024; -const size_t kDatabaseWriteBufferSizeBytesForLowEndDevice = 128 * 1024; -} // namespace - -FeedImageDatabase::FeedImageDatabase(const base::FilePath& database_dir) - : FeedImageDatabase( - database_dir, - leveldb_proto::ProtoDatabaseProvider::CreateUniqueDB< - CachedImageProto>(base::CreateSequencedTaskRunnerWithTraits( - {base::MayBlock(), base::TaskPriority::BEST_EFFORT, - base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN})), - base::DefaultClock::GetInstance()) {} - -FeedImageDatabase::FeedImageDatabase( - const base::FilePath& database_dir, - std::unique_ptr<leveldb_proto::ProtoDatabase<CachedImageProto>> - image_database, - base::Clock* clock) - : database_status_(UNINITIALIZED), - image_database_(std::move(image_database)), - clock_(clock), - weak_ptr_factory_(this) { - leveldb_env::Options options = leveldb_proto::CreateSimpleOptions(); - if (base::SysInfo::IsLowEndDevice()) { - options.write_buffer_size = kDatabaseWriteBufferSizeBytesForLowEndDevice; - } else { - options.write_buffer_size = kDatabaseWriteBufferSizeBytes; - } - base::FilePath image_dir = database_dir.AppendASCII(kImageDatabaseFolder); - image_database_->Init( - kImageDatabaseUMAClientName, image_dir, options, - base::BindOnce(&FeedImageDatabase::OnDatabaseInitialized, - weak_ptr_factory_.GetWeakPtr())); -} - -FeedImageDatabase::~FeedImageDatabase() = default; - -bool FeedImageDatabase::IsInitialized() { - return INITIALIZED == database_status_; -} - -void FeedImageDatabase::SaveImage(const std::string& url, - const std::string& image_data) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // If database is not ready, ignore the request. - if (!IsInitialized()) - return; - - CachedImageProto image_proto; - image_proto.set_url(url); - image_proto.set_data(image_data); - image_proto.set_last_used_time(ToDatabaseTime(clock_->Now())); - - SaveImageImpl(url, image_proto); -} - -void FeedImageDatabase::LoadImage(const std::string& url, - FeedImageDatabaseCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - switch (database_status_) { - case INITIALIZED: - case INIT_FAILURE: - LoadImageImpl(url, std::move(callback)); - break; - case UNINITIALIZED: - pending_image_callbacks_.emplace_back(url, std::move(callback)); - break; - default: - NOTREACHED(); - } -} - -void FeedImageDatabase::DeleteImage(const std::string& url) { - DeleteImageImpl(url, base::BindOnce(&FeedImageDatabase::OnImageUpdated, - weak_ptr_factory_.GetWeakPtr())); -} - -void FeedImageDatabase::GarbageCollectImages( - base::Time expired_time, - FeedImageDatabaseOperationCallback callback) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - // If database is not initialized yet, ignore the request. - if (!IsInitialized()) { - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), false)); - return; - } - - image_database_->LoadEntries(base::BindOnce( - &FeedImageDatabase::GarbageCollectImagesImpl, - weak_ptr_factory_.GetWeakPtr(), expired_time, std::move(callback))); -} - -void FeedImageDatabase::OnDatabaseInitialized(bool success) { - DCHECK_EQ(database_status_, UNINITIALIZED); - - if (success) { - database_status_ = INITIALIZED; - } else { - database_status_ = INIT_FAILURE; - DVLOG(1) << "FeedImageDatabase init failed."; - } - - ProcessPendingImageLoads(); -} - -void FeedImageDatabase::ProcessPendingImageLoads() { - DCHECK_NE(database_status_, UNINITIALIZED); - - for (auto& image_callback : pending_image_callbacks_) - LoadImageImpl(image_callback.first, std::move(image_callback.second)); - - pending_image_callbacks_.clear(); -} - -void FeedImageDatabase::SaveImageImpl(std::string url, - const CachedImageProto& image_proto) { - auto entries_to_save = std::make_unique<ImageKeyEntryVector>(); - entries_to_save->emplace_back(std::move(url), image_proto); - - image_database_->UpdateEntries( - std::move(entries_to_save), std::make_unique<std::vector<std::string>>(), - base::BindOnce(&FeedImageDatabase::OnImageUpdated, - weak_ptr_factory_.GetWeakPtr())); -} - -void FeedImageDatabase::OnImageLoaded(std::string url, - FeedImageDatabaseCallback callback, - bool success, - std::unique_ptr<CachedImageProto> entry) { - if (!success || !entry) { - DVLOG_IF(1, !success) << "FeedImageDatabase load failed."; - std::move(callback).Run(std::string()); - return; - } - - DCHECK_EQ(url, entry->url()); - std::move(callback).Run(entry->data()); - - // Update timestamp for image. - entry->set_last_used_time(ToDatabaseTime(clock_->Now())); - SaveImageImpl(std::move(url), *entry); -} - -void FeedImageDatabase::LoadImageImpl(const std::string& url, - FeedImageDatabaseCallback callback) { - DCHECK_NE(database_status_, UNINITIALIZED); - - if (IsInitialized()) { - image_database_->GetEntry( - url, base::BindOnce(&FeedImageDatabase::OnImageLoaded, - weak_ptr_factory_.GetWeakPtr(), url, - std::move(callback))); - } else { - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), std::string())); - } -} - -void FeedImageDatabase::OnImageUpdated(bool success) { - DVLOG_IF(1, !success) << "FeedImageDatabase update failed."; -} - -void FeedImageDatabase::DeleteImageImpl( - const std::string& url, - FeedImageDatabaseOperationCallback callback) { - image_database_->UpdateEntries( - std::make_unique<ImageKeyEntryVector>(), - std::make_unique<std::vector<std::string>>(1, url), std::move(callback)); -} - -void FeedImageDatabase::GarbageCollectImagesImpl( - base::Time expired_time, - FeedImageDatabaseOperationCallback callback, - bool load_entries_success, - std::unique_ptr<std::vector<CachedImageProto>> image_entries) { - if (!load_entries_success) { - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(&FeedImageDatabase::OnGarbageCollectionDone, - weak_ptr_factory_.GetWeakPtr(), - std::move(callback), false)); - return; - } - - int64_t expired_database_time = ToDatabaseTime(expired_time); - auto keys_to_remove = std::make_unique<std::vector<std::string>>(); - for (const CachedImageProto& image : *image_entries) { - if (image.last_used_time() < expired_database_time) { - keys_to_remove->emplace_back(image.url()); - } - } - - if (keys_to_remove->empty()) { - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), true)); - return; - } - - image_database_->UpdateEntries( - std::make_unique<ImageKeyEntryVector>(), std::move(keys_to_remove), - base::BindOnce(&FeedImageDatabase::OnGarbageCollectionDone, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); -} - -void FeedImageDatabase::OnGarbageCollectionDone( - FeedImageDatabaseOperationCallback callback, - bool success) { - DVLOG_IF(1, !success) << "FeedImageDatabase garbage collection failed."; - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), success)); -} - -} // namespace feed diff --git a/chromium/components/feed/core/feed_image_database.h b/chromium/components/feed/core/feed_image_database.h deleted file mode 100644 index 90c8a39ecc5..00000000000 --- a/chromium/components/feed/core/feed_image_database.h +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_FEED_CORE_FEED_IMAGE_DATABASE_H_ -#define COMPONENTS_FEED_CORE_FEED_IMAGE_DATABASE_H_ - -#include <memory> -#include <string> -#include <utility> -#include <vector> - -#include "base/memory/weak_ptr.h" -#include "components/leveldb_proto/public/proto_database.h" - -namespace base { -class Clock; -} // namespace base - -namespace feed { - -class CachedImageProto; - -// FeedImageDatabase is leveldb backed store for feed's image data. -// FeedImageDatabase keeps images identified by URLs. -// Save and Load operations are asynchronous, every load operation will update -// last_used_time for the image for garbage collection purpose. -class FeedImageDatabase { - public: - enum State { - UNINITIALIZED, - INITIALIZED, - INIT_FAILURE, - }; - - // Returns the resulting raw image data as std::string of a |LoadImage| call. - using FeedImageDatabaseCallback = - base::OnceCallback<void(const std::string&)>; - - using FeedImageDatabaseOperationCallback = base::OnceCallback<void(bool)>; - - // Initializes the database with |database_dir|. - explicit FeedImageDatabase(const base::FilePath& database_dir); - // Initializes the database with |database_dir|. Creates storage using the - // given |image_database| for local storage. Useful for testing. - FeedImageDatabase( - const base::FilePath& database_dir, - std::unique_ptr<leveldb_proto::ProtoDatabase<CachedImageProto>> - image_database, - base::Clock* clock); - ~FeedImageDatabase(); - - // Returns true if initialization has finished successfully, else false. - // While this is false, initialization may already started, or initialization - // failed. - bool IsInitialized(); - - // Adds or updates the image data for the |url|. - // If the database is not initialized or in some error status, the call will - // be ignored. - void SaveImage(const std::string& url, const std::string& image_data); - - // Loads the image data for the |url| and passes it to |callback|. - // |callback| will be called in the same thread as this function called. - // If the image cannot be found in database, or database error, returns an - // empty CachedImageProto. If the database is not initialized yet, the - // request will be pending until the database has been initialized. - void LoadImage(const std::string& url, FeedImageDatabaseCallback callback); - - // Deletes the image data for the |url|. - void DeleteImage(const std::string& url); - - // Delete all images whose |last_used_time| is older than |expired_time| and - // passes the result to |callback|. |callback| will be called in the same - // thread as this function called. If database is not initialized, or failed - // to delete expired entry, false will be passed to |callback|. - void GarbageCollectImages(base::Time expired_time, - FeedImageDatabaseOperationCallback callback); - - private: - friend class FeedImageDatabaseTest; - - using ImageKeyEntryVector = - leveldb_proto::ProtoDatabase<CachedImageProto>::KeyEntryVector; - - // Initialization - void OnDatabaseInitialized(bool success); - void ProcessPendingImageLoads(); - - // Saving - void SaveImageImpl(std::string url, const CachedImageProto& image_proto); - void OnImageUpdated(bool success); - - // Loading - void LoadImageImpl(const std::string& url, - FeedImageDatabaseCallback callback); - void OnImageLoaded(std::string url, - FeedImageDatabaseCallback callback, - bool success, - std::unique_ptr<CachedImageProto> entry); - - // Deleting - void DeleteImageImpl(const std::string& url, - FeedImageDatabaseOperationCallback callback); - - // Garbage collection - void GarbageCollectImagesImpl( - base::Time expired_time, - FeedImageDatabaseOperationCallback callback, - bool load_entries_success, - std::unique_ptr<std::vector<CachedImageProto>> image_entries); - void OnGarbageCollectionDone(FeedImageDatabaseOperationCallback callback, - bool success); - - State database_status_; - - std::unique_ptr<leveldb_proto::ProtoDatabase<CachedImageProto>> - image_database_; - - // Used to access current time, injected for testing. - base::Clock* clock_; - - std::vector<std::pair<std::string, FeedImageDatabaseCallback>> - pending_image_callbacks_; - - // Used to check that functions are called on the correct sequence. - SEQUENCE_CHECKER(sequence_checker_); - - base::WeakPtrFactory<FeedImageDatabase> weak_ptr_factory_; - - DISALLOW_COPY_AND_ASSIGN(FeedImageDatabase); -}; - -} // namespace feed - -#endif // COMPONENTS_FEED_CORE_FEED_IMAGE_DATABASE_H_ diff --git a/chromium/components/feed/core/feed_image_database_unittest.cc b/chromium/components/feed/core/feed_image_database_unittest.cc deleted file mode 100644 index 340b8fe19bc..00000000000 --- a/chromium/components/feed/core/feed_image_database_unittest.cc +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/feed/core/feed_image_database.h" - -#include <map> - -#include "base/test/scoped_task_environment.h" -#include "base/test/simple_test_clock.h" -#include "components/feed/core/proto/cached_image.pb.h" -#include "components/feed/core/time_serialization.h" -#include "components/leveldb_proto/testing/fake_db.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" - -using leveldb_proto::test::FakeDB; -using testing::Mock; -using testing::_; - -namespace feed { - -namespace { - -// Fixed "now" to make tests more deterministic. -char kNowString[] = "2018-06-11 15:41"; - -constexpr char kImageURL[] = "http://pie.com/"; -constexpr char kImageData[] = "pie image"; - -} // namespace - -class FeedImageDatabaseTest : public testing::Test { - public: - FeedImageDatabaseTest() : image_db_(nullptr) { - base::Time now; - EXPECT_TRUE(base::Time::FromUTCString(kNowString, &now)); - test_clock_.SetNow(now); - } - - void CreateDatabase() { - // The FakeDBs are owned by |feed_db_|, so clear our pointers before - // resetting |feed_db_| itself. - image_db_ = nullptr; - // Explicitly destroy any existing database before creating a new one. - feed_db_.reset(); - - auto image_db = - std::make_unique<FakeDB<CachedImageProto>>(&image_db_storage_); - - image_db_ = image_db.get(); - feed_db_ = std::make_unique<FeedImageDatabase>( - base::FilePath(), std::move(image_db), &test_clock_); - } - - int64_t GetImageLastUsedTime(const std::string& url) { - return image_db_storage_[kImageURL].last_used_time(); - } - - void InjectImageProto(const std::string& url, - const std::string& data, - base::Time time) { - CachedImageProto image_proto; - image_proto.set_url(url); - image_proto.set_data(data); - image_proto.set_last_used_time(ToDatabaseTime(time)); - image_db_storage_[url] = image_proto; - } - - FakeDB<CachedImageProto>* image_db() { return image_db_; } - FeedImageDatabase* db() { return feed_db_.get(); } - base::SimpleTestClock* test_clock() { return &test_clock_; } - - void RunUntilIdle() { scoped_task_environment_.RunUntilIdle(); } - - MOCK_METHOD1(OnImageLoaded, void(const std::string&)); - MOCK_METHOD1(OnGarbageCollected, void(bool)); - - private: - base::test::ScopedTaskEnvironment scoped_task_environment_; - - std::map<std::string, CachedImageProto> image_db_storage_; - - // Owned by |feed_db_|. - FakeDB<CachedImageProto>* image_db_; - - base::SimpleTestClock test_clock_; - - std::unique_ptr<FeedImageDatabase> feed_db_; - - DISALLOW_COPY_AND_ASSIGN(FeedImageDatabaseTest); -}; - -TEST_F(FeedImageDatabaseTest, Init) { - ASSERT_FALSE(db()); - - CreateDatabase(); - EXPECT_FALSE(db()->IsInitialized()); - - image_db()->InitCallback(true); - - EXPECT_TRUE(db()->IsInitialized()); -} - -TEST_F(FeedImageDatabaseTest, LoadBeforeInitSuccess) { - CreateDatabase(); - EXPECT_FALSE(db()->IsInitialized()); - - // Start an image load before the database is initialized. - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - - EXPECT_CALL(*this, OnImageLoaded(_)); - - image_db()->InitCallback(true); - EXPECT_TRUE(db()->IsInitialized()); - image_db()->GetCallback(true); -} - -TEST_F(FeedImageDatabaseTest, LoadBeforeInitFailed) { - CreateDatabase(); - EXPECT_FALSE(db()->IsInitialized()); - - // Start an image load before the database is initialized. - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - - EXPECT_CALL(*this, OnImageLoaded(_)); - - image_db()->InitCallback(false); - EXPECT_FALSE(db()->IsInitialized()); - RunUntilIdle(); -} - -TEST_F(FeedImageDatabaseTest, LoadAfterInitSuccess) { - CreateDatabase(); - EXPECT_FALSE(db()->IsInitialized()); - - EXPECT_CALL(*this, OnImageLoaded(_)).Times(0); - - image_db()->InitCallback(true); - EXPECT_TRUE(db()->IsInitialized()); - - Mock::VerifyAndClearExpectations(this); - - EXPECT_CALL(*this, OnImageLoaded(_)); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); -} - -TEST_F(FeedImageDatabaseTest, LoadAfterInitFailed) { - CreateDatabase(); - EXPECT_FALSE(db()->IsInitialized()); - - EXPECT_CALL(*this, OnImageLoaded(_)).Times(0); - - image_db()->InitCallback(false); - EXPECT_FALSE(db()->IsInitialized()); - - Mock::VerifyAndClearExpectations(this); - - EXPECT_CALL(*this, OnImageLoaded(_)); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - RunUntilIdle(); -} - -TEST_F(FeedImageDatabaseTest, Save) { - CreateDatabase(); - image_db()->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - - // Store an image. - db()->SaveImage(kImageURL, kImageData); - image_db()->UpdateCallback(true); - - // Make sure they're there. - EXPECT_CALL(*this, OnImageLoaded(kImageData)); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); -} - -TEST_F(FeedImageDatabaseTest, SavePersist) { - CreateDatabase(); - image_db()->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - - // Store an image. - db()->SaveImage(kImageURL, kImageData); - image_db()->UpdateCallback(true); - - // They should still exist after recreating the database. - CreateDatabase(); - image_db()->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - - EXPECT_CALL(*this, OnImageLoaded(kImageData)); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); -} - -TEST_F(FeedImageDatabaseTest, LoadUpdatesTime) { - CreateDatabase(); - image_db()->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - - // Store an image. - InjectImageProto(kImageURL, kImageData, base::Time::UnixEpoch()); - - int64_t old_time = GetImageLastUsedTime(kImageURL); - // Make sure they're there. - EXPECT_CALL(*this, OnImageLoaded(kImageData)); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); - image_db()->UpdateCallback(true); - EXPECT_TRUE(old_time != GetImageLastUsedTime(kImageURL)); -} - -TEST_F(FeedImageDatabaseTest, Delete) { - CreateDatabase(); - image_db()->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - - // Store the image. - db()->SaveImage(kImageURL, kImageData); - image_db()->UpdateCallback(true); - - // Make sure the image is there. - EXPECT_CALL(*this, OnImageLoaded(kImageData)); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); - - Mock::VerifyAndClearExpectations(this); - - // Delete the image. - db()->DeleteImage(kImageURL); - image_db()->UpdateCallback(true); - - // Make sure the image is gone. - EXPECT_CALL(*this, OnImageLoaded(std::string())); - db()->LoadImage(kImageURL, - base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); -} - -TEST_F(FeedImageDatabaseTest, GarbageCollectImagesTest) { - CreateDatabase(); - image_db()->InitCallback(true); - ASSERT_TRUE(db()->IsInitialized()); - - base::Time now = test_clock()->Now(); - base::Time expired_time = now - base::TimeDelta::FromDays(30); - base::Time very_old_time = now - base::TimeDelta::FromDays(100); - - // Store images. - InjectImageProto("url1", "data1", very_old_time); - InjectImageProto("url2", "data2", now); - InjectImageProto("url3", "data3", very_old_time); - - // Garbage collect all except the second. - EXPECT_CALL(*this, OnGarbageCollected(true)); - db()->GarbageCollectImages( - expired_time, base::BindOnce(&FeedImageDatabaseTest::OnGarbageCollected, - base::Unretained(this))); - // This will first load all images, then delete the expired ones. - image_db()->LoadCallback(true); - image_db()->UpdateCallback(true); - RunUntilIdle(); - - // Make sure the images are gone. - EXPECT_CALL(*this, OnImageLoaded(std::string())); - db()->LoadImage("url1", base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); - - EXPECT_CALL(*this, OnImageLoaded(std::string())); - db()->LoadImage("url3", base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); - - // Make sure the second still exists. - EXPECT_CALL(*this, OnImageLoaded("data2")); - db()->LoadImage("url2", base::BindOnce(&FeedImageDatabaseTest::OnImageLoaded, - base::Unretained(this))); - image_db()->GetCallback(true); -} - -} // namespace feed diff --git a/chromium/components/feed/core/feed_image_manager.cc b/chromium/components/feed/core/feed_image_manager.cc deleted file mode 100644 index c7776e417a8..00000000000 --- a/chromium/components/feed/core/feed_image_manager.cc +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/feed/core/feed_image_manager.h" - -#include <utility> - -#include "base/bind.h" -#include "base/metrics/histogram_macros.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/timer/elapsed_timer.h" -#include "components/feed/core/time_serialization.h" -#include "components/image_fetcher/core/image_decoder.h" -#include "components/image_fetcher/core/image_fetcher.h" -#include "ui/gfx/geometry/size.h" -#include "ui/gfx/image/image.h" - -namespace feed { - -namespace { - -// Keep in sync with DIMENSION_UNKNOWN in third_party/feed/src/main/java/com/ -// google/android/libraries/feed/host/imageloader/ImageLoaderApi.java. -const int DIMENSION_UNKNOWN = -1; - -const int kDefaultGarbageCollectionExpiredDays = 30; -const int kLongGarbageCollectionInterval = 12 * 60 * 60; // 12 hours -const int kShortGarbageCollectionInterval = 5 * 60; // 5 minutes -constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotation = - net::DefineNetworkTrafficAnnotation("feed_image_fetcher", R"( - semantics { - sender: "Feed Library Image Fetch" - description: - "Retrieves images for content suggestions, for display on the " - "New Tab page." - trigger: - "Triggered when the user looks at a content suggestion (and its " - "thumbnail isn't cached yet)." - data: "None." - destination: GOOGLE_OWNED_SERVICE - } - policy { - cookies_allowed: NO - setting: - "This can be disabled from the New Tab Page by collapsing the " - "articles section." - chrome_policy { - NTPContentSuggestionsEnabled { - NTPContentSuggestionsEnabled: false - } - } - })"); - -void ReportFetchResult(FeedImageFetchResult result) { - UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.Image.FetchResult", - result); -} - -gfx::Size CreateGfxSize(int width_px, int height_px) { - DCHECK_GE(width_px, DIMENSION_UNKNOWN); - DCHECK_GE(height_px, DIMENSION_UNKNOWN); - - // Only resize the image when both |width_px| and |height_px| are available. - if (width_px == DIMENSION_UNKNOWN || height_px == DIMENSION_UNKNOWN) { - return gfx::Size(); - } - return gfx::Size(width_px, height_px); -} - -} // namespace - -FeedImageManager::FeedImageManager( - std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, - std::unique_ptr<FeedImageDatabase> image_database) - : image_garbage_collected_day_(FromDatabaseTime(0)), - image_fetcher_(std::move(image_fetcher)), - image_database_(std::move(image_database)), - weak_ptr_factory_(this) { - DoGarbageCollectionIfNeeded(); -} - -FeedImageManager::~FeedImageManager() { - StopGarbageCollection(); -} - -void FeedImageManager::FetchImage(std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback) { - DCHECK(image_database_); - - FetchImagesFromDatabase(0, std::move(urls), width_px, height_px, - std::move(callback)); -} - -void FeedImageManager::FetchImagesFromDatabase(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback) { - if (url_index >= urls.size()) { - // Already reached the last entry. Return an empty image. - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), gfx::Image(), -1)); - return; - } - - const std::string& image_id = urls[url_index]; - // Only take the first instance of the url so we get the worst-case time. - if (url_timers_.find(image_id) == url_timers_.end()) { - url_timers_.insert(std::make_pair(image_id, base::ElapsedTimer())); - } - image_database_->LoadImage( - image_id, - base::BindOnce(&FeedImageManager::OnImageFetchedFromDatabase, - weak_ptr_factory_.GetWeakPtr(), url_index, std::move(urls), - width_px, height_px, std::move(callback))); -} - -void FeedImageManager::OnImageFetchedFromDatabase( - size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const std::string& image_data) { - if (image_data.empty()) { - // Fetching from the DB failed; start a network fetch. - FetchImageFromNetwork(url_index, std::move(urls), width_px, height_px, - std::move(callback)); - return; - } - - image_fetcher_->GetImageDecoder()->DecodeImage( - image_data, CreateGfxSize(width_px, height_px), - base::BindRepeating(&FeedImageManager::OnImageDecodedFromDatabase, - weak_ptr_factory_.GetWeakPtr(), url_index, - std::move(urls), width_px, height_px, - base::Passed(std::move(callback)))); -} - -void FeedImageManager::OnImageDecodedFromDatabase(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const gfx::Image& image) { - const std::string& image_id = urls[url_index]; - if (image.IsEmpty()) { - // If decoding the image failed, delete the DB entry. - image_database_->DeleteImage(image_id); - FetchImageFromNetwork(url_index, std::move(urls), width_px, height_px, - std::move(callback)); - return; - } - - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), image, url_index)); - - // Report success if the url exists. - // This check is for concurrent access to the same url. - if (url_timers_.find(image_id) != url_timers_.end()) { - UMA_HISTOGRAM_TIMES("ContentSuggestions.Feed.Image.LoadFromCacheTime", - url_timers_[image_id].Elapsed()); - ClearUmaTimer(image_id); - ReportFetchResult(FeedImageFetchResult::kSuccessCached); - } -} - -void FeedImageManager::FetchImageFromNetwork(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback) { - const std::string& image_id = urls[url_index]; - GURL url(image_id); - if (!url.is_valid()) { - // Report failure. - ReportFetchResult(FeedImageFetchResult::kFailure); - ClearUmaTimer(image_id); - - // url is not valid, go to next URL. - FetchImagesFromDatabase(url_index + 1, std::move(urls), width_px, height_px, - std::move(callback)); - return; - } - - image_fetcher_->FetchImageData( - url.spec(), url, - base::BindOnce(&FeedImageManager::OnImageFetchedFromNetwork, - weak_ptr_factory_.GetWeakPtr(), url_index, std::move(urls), - width_px, height_px, std::move(callback)), - kTrafficAnnotation); -} - -void FeedImageManager::OnImageFetchedFromNetwork( - size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const std::string& image_data, - const image_fetcher::RequestMetadata& request_metadata) { - if (image_data.empty()) { - // Report failure. - ReportFetchResult(FeedImageFetchResult::kFailure); - ClearUmaTimer(urls[url_index]); - - // Fetching image failed, let's move to the next url. - FetchImagesFromDatabase(url_index + 1, std::move(urls), width_px, height_px, - std::move(callback)); - return; - } - - image_fetcher_->GetImageDecoder()->DecodeImage( - image_data, CreateGfxSize(width_px, height_px), - base::BindRepeating(&FeedImageManager::OnImageDecodedFromNetwork, - weak_ptr_factory_.GetWeakPtr(), url_index, - std::move(urls), width_px, height_px, - base::Passed(std::move(callback)), image_data)); -} - -void FeedImageManager::OnImageDecodedFromNetwork(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const std::string& image_data, - const gfx::Image& image) { - std::string image_id = urls[url_index]; - if (image.IsEmpty()) { - // Report failure. - ReportFetchResult(FeedImageFetchResult::kFailure); - ClearUmaTimer(image_id); - - // Decoding failed, let's move to the next url. - FetchImagesFromDatabase(url_index + 1, std::move(urls), width_px, height_px, - std::move(callback)); - return; - } - - image_database_->SaveImage(image_id, image_data); - - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), image, url_index)); - - // Report success if the url exists. - // This check is for concurrent access to the same url. - if (url_timers_.find(image_id) != url_timers_.end()) { - UMA_HISTOGRAM_TIMES("ContentSuggestions.Feed.Image.LoadFromNetworkTime", - url_timers_[image_id].Elapsed()); - ClearUmaTimer(image_id); - ReportFetchResult(FeedImageFetchResult::kSuccessFetched); - } -} - -void FeedImageManager::DoGarbageCollectionIfNeeded() { - // For saving resource purpose(ex. cpu, battery), We round up garbage - // collection age to day, so we only run GC once a day. - base::Time to_be_expired = - base::Time::Now().LocalMidnight() - - base::TimeDelta::FromDays(kDefaultGarbageCollectionExpiredDays); - if (image_garbage_collected_day_ != to_be_expired) { - image_database_->GarbageCollectImages( - to_be_expired, - base::BindOnce(&FeedImageManager::OnGarbageCollectionDone, - weak_ptr_factory_.GetWeakPtr(), to_be_expired)); - } -} - -void FeedImageManager::OnGarbageCollectionDone(base::Time garbage_collected_day, - bool success) { - base::TimeDelta gc_delay = - base::TimeDelta::FromSeconds(kShortGarbageCollectionInterval); - if (success) { - if (image_garbage_collected_day_ < garbage_collected_day) - image_garbage_collected_day_ = garbage_collected_day; - gc_delay = base::TimeDelta::FromSeconds(kLongGarbageCollectionInterval); - } - - garbage_collection_timer_.Start( - FROM_HERE, gc_delay, this, - &FeedImageManager::DoGarbageCollectionIfNeeded); -} - -void FeedImageManager::StopGarbageCollection() { - garbage_collection_timer_.Stop(); -} - -void FeedImageManager::ClearUmaTimer(const std::string& url) { - url_timers_.erase(url); -} - -} // namespace feed diff --git a/chromium/components/feed/core/feed_image_manager.h b/chromium/components/feed/core/feed_image_manager.h deleted file mode 100644 index a900d31aacb..00000000000 --- a/chromium/components/feed/core/feed_image_manager.h +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_FEED_CORE_FEED_IMAGE_MANAGER_H_ -#define COMPONENTS_FEED_CORE_FEED_IMAGE_MANAGER_H_ - -#include <memory> -#include <string> -#include <vector> - -#include "base/containers/flat_map.h" -#include "base/memory/weak_ptr.h" -#include "base/timer/timer.h" -#include "components/feed/core/feed_image_database.h" - -namespace base { -class ElapsedTimer; -} // namespace base - -namespace gfx { -class Image; -} // namespace gfx - -namespace image_fetcher { -class ImageFetcher; -struct RequestMetadata; -} // namespace image_fetcher - -namespace feed { - -using ImageFetchedCallback = - base::OnceCallback<void(const gfx::Image&, size_t)>; - -// Enum for the result of the fetch, reported through UMA. -// New values should be added at the end and things should not be renumbered. -enum class FeedImageFetchResult { - kSuccessCached = 0, - kSuccessFetched = 1, - kFailure = 2, - kMaxValue = kFailure, -}; - -// FeedImageManager takes care of fetching images from the network and caching -// them in the database. -class FeedImageManager { - public: - FeedImageManager(std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher, - std::unique_ptr<FeedImageDatabase> image_database); - ~FeedImageManager(); - - // Fetches an image from |urls|, and resize the image with |width_px| and - // |height_px|. FeedImageManager will go through URLs in |urls| one by one - // trying to fetch and decode them in order. If |width_px| and |height_px| are - // not available/legal, FeedImageManager will not resize the image. Upon - // success, a decoded image will be passed to |callback| as well as cached - // locally. |urls| should be supplied in priority order, and the first success - // will prevent any further processing. Failure to fetch or decode an image - // will cause FeedImageManager to process the next URL in |urls|. If - // FeedImageManager failed to fetch and decode all the URLs in |urls|, it will - // pass an empty image to |callback|. |callback| will be called exactly once. - void FetchImage(std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback); - - private: - friend class FeedImageManagerTest; - - // Database - void FetchImagesFromDatabase(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback); - void OnImageFetchedFromDatabase(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const std::string& image_data); - void OnImageDecodedFromDatabase(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const gfx::Image& image); - - // Network - void FetchImageFromNetwork(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback); - void OnImageFetchedFromNetwork( - size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const std::string& image_data, - const image_fetcher::RequestMetadata& request_metadata); - void OnImageDecodedFromNetwork(size_t url_index, - std::vector<std::string> urls, - int width_px, - int height_px, - ImageFetchedCallback callback, - const std::string& image_data, - const gfx::Image& image); - - // Garbage collection will be run when FeedImageManager starts up, and then - // once a day. Garbage collection will remove images, that have not been - // touched for 30 days. - void DoGarbageCollectionIfNeeded(); - void OnGarbageCollectionDone(base::Time garbage_collected_day, bool success); - void StopGarbageCollection(); - - void ClearUmaTimer(const std::string& url); - - // The day which image database already ran garbage collection against on. - base::Time image_garbage_collected_day_; - - base::OneShotTimer garbage_collection_timer_; - - std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher_; - std::unique_ptr<FeedImageDatabase> image_database_; - - // Track time it takes to get images. - base::flat_map<std::string, base::ElapsedTimer> url_timers_; - - base::WeakPtrFactory<FeedImageManager> weak_ptr_factory_; - - DISALLOW_COPY_AND_ASSIGN(FeedImageManager); -}; - -} // namespace feed - -#endif // COMPONENTS_FEED_CORE_FEED_IMAGE_MANAGER_H_ diff --git a/chromium/components/feed/core/feed_image_manager_unittest.cc b/chromium/components/feed/core/feed_image_manager_unittest.cc index 81ea896fcbd..418724a2dae 100644 --- a/chromium/components/feed/core/feed_image_manager_unittest.cc +++ b/chromium/components/feed/core/feed_image_manager_unittest.cc @@ -15,6 +15,7 @@ #include "base/test/mock_callback.h" #include "base/test/scoped_task_environment.h" #include "base/threading/sequenced_task_runner_handle.h" +#include "base/timer/timer.h" #include "components/image_fetcher/core/image_decoder.h" #include "components/image_fetcher/core/image_fetcher_impl.h" #include "services/network/public/cpp/shared_url_loader_factory.h" diff --git a/chromium/components/feed/core/feed_journal_database.cc b/chromium/components/feed/core/feed_journal_database.cc index 412ee24d5c1..f061a3d6100 100644 --- a/chromium/components/feed/core/feed_journal_database.cc +++ b/chromium/components/feed/core/feed_journal_database.cc @@ -6,6 +6,7 @@ #include <utility> +#include "base/bind.h" #include "base/metrics/histogram_macros.h" #include "base/system/sys_info.h" #include "base/task/post_task.h" diff --git a/chromium/components/feed/core/feed_journal_database_unittest.cc b/chromium/components/feed/core/feed_journal_database_unittest.cc index 85d7cd92de0..47c60b84539 100644 --- a/chromium/components/feed/core/feed_journal_database_unittest.cc +++ b/chromium/components/feed/core/feed_journal_database_unittest.cc @@ -7,6 +7,7 @@ #include <map> #include <utility> +#include "base/bind.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/scoped_task_environment.h" #include "components/feed/core/feed_journal_mutation.h" diff --git a/chromium/components/feed/core/feed_logging_metrics.cc b/chromium/components/feed/core/feed_logging_metrics.cc index 9b83b0aafb3..03989b12cfc 100644 --- a/chromium/components/feed/core/feed_logging_metrics.cc +++ b/chromium/components/feed/core/feed_logging_metrics.cc @@ -8,6 +8,7 @@ #include <string> #include <type_traits> +#include "base/bind.h" #include "base/metrics/histogram.h" #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" @@ -27,7 +28,11 @@ namespace { // identical bucket sizes and names with Zine is for comparing Feed with Zine // easily. After Zine is deprecated, we can change the values if we needed. +// Constants used as max sample sizes for histograms. +const int kMaxContentCount = 50; +const int kMaxFailureCount = 10; const int kMaxSuggestionsTotal = 50; +const int kMaxTokenCount = 10; // Keep in sync with MAX_SUGGESTIONS_PER_SECTION in NewTabPageUma.java. const int kMaxSuggestionsForArticle = 20; @@ -35,6 +40,42 @@ const int kMaxSuggestionsForArticle = 20; const char kHistogramArticlesUsageTimeLocal[] = "NewTabPage.ContentSuggestions.UsageTimeLocal"; +// Values correspond to +// third_party/feed/src/src/main/java/com/google/android/libraries/feed/host/ +// logging/SpinnerType.java, enums.xml and histograms.xml. +// These values are persisted to logs. Entries should not be renumbered and +// numeric values should never be reused. +enum class SpinnerType { + KInitialLoad = 1, + KZeroStateRefresh = 2, + KMoreButton = 3, + KSyntheticToken = 4, + KInfiniteFeed = 5, + kMaxValue = KInfiniteFeed +}; + +// Each suffix here should correspond to an entry under histogram suffix +// ContentSuggestionCategory in histograms.xml. +std::string GetSpinnerTypeSuffix(SpinnerType spinner_type) { + switch (spinner_type) { + case SpinnerType::KInitialLoad: + return "InitialLoad"; + case SpinnerType::KZeroStateRefresh: + return "ZeroStateRefresh"; + case SpinnerType::KMoreButton: + return "MoreButton"; + case SpinnerType::KSyntheticToken: + return "SyntheticToken"; + case SpinnerType::KInfiniteFeed: + return "InfiniteFeed"; + } + + // TODO(https://crbug.com/935602): Handle new values when adding new values on + // java side. + NOTREACHED(); + return std::string(); +} + // Records ContentSuggestions usage. Therefore the day is sliced into 20min // buckets. Depending on the current local time the count of the corresponding // bucket is increased. @@ -51,6 +92,7 @@ void RecordContentSuggestionsUsage(base::Time now) { std::string histogram_name( base::StringPrintf("%s.%s", kHistogramArticlesUsageTimeLocal, kWeekdayNames[now_exploded.day_of_week])); + // Since the |histogram_name| is dynamic, we can't use the regular macro. base::UmaHistogramExactLinear(histogram_name, bucket, kNumBuckets); UMA_HISTOGRAM_EXACT_LINEAR(kHistogramArticlesUsageTimeLocal, bucket, kNumBuckets); @@ -76,6 +118,37 @@ void RecordSuggestionPageVisited(bool return_to_ntp) { base::RecordAction(base::UserMetricsAction("MobileNTP.Snippets.VisitEnd")); } +void RecordUndoableActionUMA(const std::string& histogram_base, + int position, + bool committed) { + std::string histogram_name = + histogram_base + (committed ? ".Commit" : ".Undo"); + + // Since the |histogram_name| is dynamic, we can't use the regular macro. + base::UmaHistogramExactLinear(histogram_name, position, kMaxSuggestionsTotal); +} + +void CheckURLVisitedDone(int position, bool committed, bool visited) { + if (visited) { + RecordUndoableActionUMA("NewTabPage.ContentSuggestions.DismissedVisited", + position, committed); + } else { + RecordUndoableActionUMA("NewTabPage.ContentSuggestions.DismissedUnvisited", + position, committed); + } +} + +void RecordSpinnerTimeUMA(const char* base_name, + base::TimeDelta time, + int spinner_type) { + SpinnerType type = static_cast<SpinnerType>(spinner_type); + std::string suffix = GetSpinnerTypeSuffix(type); + std::string histogram_name( + base::StringPrintf("%s.%s", base_name, suffix.c_str())); + base::UmaHistogramTimes(histogram_name, time); + base::UmaHistogramTimes(base_name, time); +} + } // namespace FeedLoggingMetrics::FeedLoggingMetrics( @@ -178,10 +251,11 @@ void FeedLoggingMetrics::OnSuggestionMenuOpened(int position, ToUMAScore(score), 11); } -void FeedLoggingMetrics::OnSuggestionDismissed(int position, const GURL& url) { +void FeedLoggingMetrics::OnSuggestionDismissed(int position, + const GURL& url, + bool committed) { history_url_check_callback_.Run( - url, base::BindOnce(&FeedLoggingMetrics::CheckURLVisitedDone, - weak_ptr_factory_.GetWeakPtr(), position)); + url, base::BindOnce(&CheckURLVisitedDone, position, committed)); base::RecordAction(base::UserMetricsAction("Suggestions.Content.Dismissed")); } @@ -221,25 +295,116 @@ void FeedLoggingMetrics::OnMoreButtonClicked(int position) { kMaxSuggestionsForArticle + 1); } -void FeedLoggingMetrics::OnSpinnerShown(base::TimeDelta shown_time) { - base::UmaHistogramTimes( - "ContentSuggestions.Feed.FetchPendingSpinner.VisibleDuration", - shown_time); +void FeedLoggingMetrics::OnNotInterestedInSource(int position, bool committed) { + RecordUndoableActionUMA( + "ContentSuggestions.Feed.InterestHeader.NotInterestedInSource", position, + committed); } -void FeedLoggingMetrics::ReportScrolledAfterOpen() { - base::RecordAction(base::UserMetricsAction("Suggestions.ScrolledAfterOpen")); +void FeedLoggingMetrics::OnNotInterestedInTopic(int position, bool committed) { + RecordUndoableActionUMA( + "ContentSuggestions.Feed.InterestHeader.NotInterestedInTopic", position, + committed); } -void FeedLoggingMetrics::CheckURLVisitedDone(int position, bool visited) { - if (visited) { - UMA_HISTOGRAM_EXACT_LINEAR("NewTabPage.ContentSuggestions.DismissedVisited", - position, kMaxSuggestionsTotal); +void FeedLoggingMetrics::OnSpinnerStarted(int spinner_type) { + // TODO(https://crbug.com/935602): Handle new values when adding new values on + // java side. + SpinnerType type = static_cast<SpinnerType>(spinner_type); + UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.FetchPendingSpinner.Shown", + type); +} + +void FeedLoggingMetrics::OnSpinnerFinished(base::TimeDelta shown_time, + int spinner_type) { + RecordSpinnerTimeUMA( + "ContentSuggestions.Feed.FetchPendingSpinner.VisibleDuration", shown_time, + spinner_type); +} + +void FeedLoggingMetrics::OnSpinnerDestroyedWithoutCompleting( + base::TimeDelta shown_time, + int spinner_type) { + RecordSpinnerTimeUMA( + "ContentSuggestions.Feed.FetchPendingSpinner." + "VisibleDurationWithoutCompleting", + shown_time, spinner_type); +} + +void FeedLoggingMetrics::OnPietFrameRenderingEvent( + std::vector<int> piet_error_codes) { + for (auto error_code : piet_error_codes) { + base::UmaHistogramSparse( + "ContentSuggestions.Feed.Piet.FrameRenderingErrorCode", error_code); + } +} + +void FeedLoggingMetrics::OnInternalError(int internal_error) { + // TODO(https://crbug.com/935602): The max value here is fragile, figure out + // some way to test the @IntDef size. + UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.InternalError", + internal_error, 10); +} + +void FeedLoggingMetrics::OnTokenCompleted(bool was_synthetic, + int content_count, + int token_count) { + if (was_synthetic) { + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.TokenCompleted.ContentCount2.Synthetic", + content_count, kMaxContentCount); + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.TokenCompleted.TokenCount.Synthetic", + token_count, kMaxTokenCount); + } else { + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.TokenCompleted.ContentCount2.NotSynthetic", + content_count, kMaxContentCount); + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.TokenCompleted.TokenCount.NotSynthetic", + token_count, kMaxTokenCount); + } +} + +void FeedLoggingMetrics::OnTokenFailedToComplete(bool was_synthetic, + int failure_count) { + if (was_synthetic) { + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.TokenFailedToCompleted.Synthetic", + failure_count, kMaxFailureCount); } else { UMA_HISTOGRAM_EXACT_LINEAR( - "NewTabPage.ContentSuggestions.DismissedUnvisited", position, - kMaxSuggestionsTotal); + "ContentSuggestions.Feed.TokenFailedToCompleted.NotSynthetic", + failure_count, kMaxFailureCount); } } +void FeedLoggingMetrics::OnServerRequest(int request_reason) { + // TODO(https://crbug.com/935602): The max value here is fragile, figure out + // some way to test the @IntDef size. + UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.ServerRequest.Reason", + request_reason, 8); +} + +void FeedLoggingMetrics::OnZeroStateShown(int zero_state_show_reason) { + // TODO(https://crbug.com/935602): The max value here is fragile, figure out + // some way to test the @IntDef size. + UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.ZeroStateShown.Reason", + zero_state_show_reason, 3); +} + +void FeedLoggingMetrics::OnZeroStateRefreshCompleted(int new_content_count, + int new_token_count) { + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.ZeroStateRefreshCompleted.ContentCount", + new_content_count, kMaxContentCount); + UMA_HISTOGRAM_EXACT_LINEAR( + "ContentSuggestions.Feed.ZeroStateRefreshCompleted.TokenCount", + new_token_count, kMaxTokenCount); +} + +void FeedLoggingMetrics::ReportScrolledAfterOpen() { + base::RecordAction(base::UserMetricsAction("Suggestions.ScrolledAfterOpen")); +} + } // namespace feed diff --git a/chromium/components/feed/core/feed_logging_metrics.h b/chromium/components/feed/core/feed_logging_metrics.h index 994d19f8c5d..2b1984ccce7 100644 --- a/chromium/components/feed/core/feed_logging_metrics.h +++ b/chromium/components/feed/core/feed_logging_metrics.h @@ -7,6 +7,7 @@ #include <memory> #include <utility> +#include <vector> #include "base/callback.h" #include "base/macros.h" @@ -60,7 +61,7 @@ class FeedLoggingMetrics { base::Time publish_date, float score); - void OnSuggestionDismissed(int position, const GURL& url); + void OnSuggestionDismissed(int position, const GURL& url, bool committed); void OnSuggestionSwiped(); @@ -75,13 +76,34 @@ class FeedLoggingMetrics { void OnMoreButtonClicked(int position); - void OnSpinnerShown(base::TimeDelta shown_time); + void OnNotInterestedInSource(int position, bool committed); + + void OnNotInterestedInTopic(int position, bool committed); + + void OnSpinnerStarted(int spinner_type); + + void OnSpinnerFinished(base::TimeDelta shown_time, int spinner_type); + + void OnSpinnerDestroyedWithoutCompleting(base::TimeDelta shown_time, + int spinner_type); + + void OnPietFrameRenderingEvent(std::vector<int> piet_error_codes); + + void OnInternalError(int internal_error); + + void OnTokenCompleted(bool was_synthetic, int content_count, int token_count); + + void OnTokenFailedToComplete(bool was_synthetic, int failure_count); + + void OnServerRequest(int request_reason); + + void OnZeroStateShown(int zero_state_show_reason); + + void OnZeroStateRefreshCompleted(int new_content_count, int new_token_count); void ReportScrolledAfterOpen(); private: - void CheckURLVisitedDone(int position, bool visited); - const HistoryURLCheckCallback history_url_check_callback_; // Used to access current time, injected for testing. diff --git a/chromium/components/feed/core/feed_logging_metrics_unittest.cc b/chromium/components/feed/core/feed_logging_metrics_unittest.cc index 02436f2738f..37303c0f183 100644 --- a/chromium/components/feed/core/feed_logging_metrics_unittest.cc +++ b/chromium/components/feed/core/feed_logging_metrics_unittest.cc @@ -4,6 +4,7 @@ #include "components/feed/core/feed_logging_metrics.h" +#include "base/bind.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/simple_test_clock.h" #include "base/time/time.h" @@ -149,21 +150,55 @@ TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionWindowOpened) { /*count=*/4))); } -TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionDismissedIfVisited) { +TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionDismissedCommitIfVisited) { base::HistogramTester histogram_tester; - feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, kVisitedUrl); + feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, kVisitedUrl, + true); EXPECT_THAT(histogram_tester.GetAllSamples( - "NewTabPage.ContentSuggestions.DismissedVisited"), + "NewTabPage.ContentSuggestions.DismissedVisited.Commit"), ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); } -TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionDismissedIfNotVisited) { +TEST_F(FeedLoggingMetricsTest, + ShouldLogOnSuggestionDismissedCommitIfNotVisited) { base::HistogramTester histogram_tester; - feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, - GURL("http://non_visited.com")); + feed_logging_metrics()->OnSuggestionDismissed( + /*position=*/10, GURL("http://non_visited.com"), true); EXPECT_THAT(histogram_tester.GetAllSamples( - "NewTabPage.ContentSuggestions.DismissedVisited"), - IsEmpty()); + "NewTabPage.ContentSuggestions.DismissedUnvisited.Commit"), + ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); +} + +TEST_F(FeedLoggingMetricsTest, + ShouldLogOnSuggestionDismissedUndoIfUndoDismissAndVisited) { + base::HistogramTester histogram_tester; + feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, kVisitedUrl, + false); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.DismissedVisited.Undo"), + ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); +} + +TEST_F(FeedLoggingMetricsTest, + ShouldLogOnSuggestionDismissedUndoIfUndoDismissAndNotVisited) { + base::HistogramTester histogram_tester; + feed_logging_metrics()->OnSuggestionDismissed( + /*position=*/10, GURL("http://non_visited.com"), false); + EXPECT_THAT(histogram_tester.GetAllSamples( + "NewTabPage.ContentSuggestions.DismissedUnvisited.Undo"), + ElementsAre(base::Bucket(/*min=*/10, /*count=*/1))); +} + +TEST_F(FeedLoggingMetricsTest, ShouldReportOnPietFrameRenderingEvent) { + base::HistogramTester histogram_tester; + std::vector<int> error_codes({0, 1, 6, 7}); + feed_logging_metrics()->OnPietFrameRenderingEvent(error_codes); + EXPECT_THAT(histogram_tester.GetAllSamples( + "ContentSuggestions.Feed.Piet.FrameRenderingErrorCode"), + ElementsAre(base::Bucket(/*min=*/0, /*count=*/1), + base::Bucket(/*min=*/1, /*count=*/1), + base::Bucket(/*min=*/6, /*count=*/1), + base::Bucket(/*min=*/7, /*count=*/1))); } } // namespace feed diff --git a/chromium/components/feed/core/feed_networking_host.cc b/chromium/components/feed/core/feed_networking_host.cc index 046f7c95f2c..27a94812f05 100644 --- a/chromium/components/feed/core/feed_networking_host.cc +++ b/chromium/components/feed/core/feed_networking_host.cc @@ -6,6 +6,7 @@ #include <utility> +#include "base/bind.h" #include "base/metrics/field_trial_params.h" #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" @@ -65,11 +66,12 @@ class NetworkFetch { private: void StartAccessTokenFetch(); - void AccessTokenFetchFinished(GoogleServiceAuthError error, + void AccessTokenFetchFinished(base::TimeTicks token_start_ticks, + GoogleServiceAuthError error, identity::AccessTokenInfo access_token_info); void StartLoader(); std::unique_ptr<network::SimpleURLLoader> MakeLoader(); - net::HttpRequestHeaders MakeHeaders(const std::string& auth_header) const; + void SetRequestHeaders(network::ResourceRequest* request) const; void PopulateRequestBody(network::SimpleURLLoader* loader); void OnSimpleLoaderComplete(std::unique_ptr<std::string> response); @@ -84,7 +86,13 @@ class NetworkFetch { network::SharedURLLoaderFactory* loader_factory_; const std::string api_key_; const base::TickClock* tick_clock_; - base::TimeTicks start_ticks_; + + // Set when the NetworkFetch is constructed, before token and article fetch. + const base::TimeTicks entire_send_start_ticks_; + + // Should be set right before the article fetch, and after the token fetch if + // there is one. + base::TimeTicks loader_only_start_ticks_; DISALLOW_COPY_AND_ASSIGN(NetworkFetch); }; @@ -103,7 +111,7 @@ NetworkFetch::NetworkFetch(const GURL& url, loader_factory_(loader_factory), api_key_(api_key), tick_clock_(tick_clock), - start_ticks_(tick_clock_->NowTicks()) {} + entire_send_start_ticks_(tick_clock_->NowTicks()) {} void NetworkFetch::Start(FeedNetworkingHost::ResponseCallback done_callback) { done_callback_ = std::move(done_callback); @@ -123,20 +131,27 @@ void NetworkFetch::StartAccessTokenFetch() { token_fetcher_ = std::make_unique<identity::PrimaryAccountAccessTokenFetcher>( "feed", identity_manager_, scopes, base::BindOnce(&NetworkFetch::AccessTokenFetchFinished, - base::Unretained(this)), + base::Unretained(this), tick_clock_->NowTicks()), identity::PrimaryAccountAccessTokenFetcher::Mode::kWaitUntilAvailable); } void NetworkFetch::AccessTokenFetchFinished( + base::TimeTicks token_start_ticks, GoogleServiceAuthError error, identity::AccessTokenInfo access_token_info) { UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.Network.TokenFetchStatus", error.state(), GoogleServiceAuthError::NUM_STATES); + + base::TimeDelta token_duration = tick_clock_->NowTicks() - token_start_ticks; + UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.TokenDuration", + token_duration); + access_token_ = access_token_info.token; StartLoader(); } void NetworkFetch::StartLoader() { + loader_only_start_ticks_ = tick_clock_->NowTicks(); simple_loader_ = MakeLoader(); simple_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( loader_factory_, base::BindOnce(&NetworkFetch::OnSimpleLoaderComplete, @@ -144,12 +159,6 @@ void NetworkFetch::StartLoader() { } std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader() { - std::string auth_header = - access_token_.empty() - ? std::string() - : base::StringPrintf(kAuthorizationRequestHeaderFormat, - access_token_.c_str()); - net::HttpRequestHeaders headers = MakeHeaders(auth_header); // TODO(pnoland): Add data use measurement once it's supported for simple // url loader. net::NetworkTrafficAnnotationTag traffic_annotation = @@ -186,8 +195,8 @@ std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader() { resource_request->load_flags = net::LOAD_BYPASS_CACHE; resource_request->allow_credentials = false; - resource_request->headers = headers; resource_request->method = request_type_; + SetRequestHeaders(resource_request.get()); auto simple_loader = network::SimpleURLLoader::Create( std::move(resource_request), traffic_annotation); @@ -198,22 +207,23 @@ std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader() { return simple_loader; } -net::HttpRequestHeaders NetworkFetch::MakeHeaders( - const std::string& auth_header) const { - net::HttpRequestHeaders headers; - headers.SetHeader(net::HttpRequestHeaders::kContentType, kContentType); - headers.SetHeader(kContentEncoding, kGzip); - - bool is_authorized = !auth_header.empty(); - if (is_authorized) - headers.SetHeader(net::HttpRequestHeaders::kAuthorization, auth_header); +void NetworkFetch::SetRequestHeaders(network::ResourceRequest* request) const { + request->headers.SetHeader(net::HttpRequestHeaders::kContentType, + kContentType); + request->headers.SetHeader(kContentEncoding, kGzip); + + variations::SignedIn signed_in_status = variations::SignedIn::kNo; + if (!access_token_.empty()) { + std::string auth_header = base::StringPrintf( + kAuthorizationRequestHeaderFormat, access_token_.c_str()); + request->headers.SetHeader(net::HttpRequestHeaders::kAuthorization, + auth_header); + signed_in_status = variations::SignedIn::kYes; + } - variations::SignedIn signed_in_status = - is_authorized ? variations::SignedIn::kYes : variations::SignedIn::kNo; // Add X-Client-Data header with experiment IDs from field trials. - variations::AppendVariationHeaders(url_, variations::InIncognito::kNo, - signed_in_status, &headers); - return headers; + variations::AppendVariationsHeader(url_, variations::InIncognito::kNo, + signed_in_status, request); } void NetworkFetch::PopulateRequestBody(network::SimpleURLLoader* loader) { @@ -254,9 +264,16 @@ void NetworkFetch::OnSimpleLoaderComplete( response_body.assign(begin, end); } - base::TimeDelta duration = tick_clock_->NowTicks() - start_ticks_; + base::TimeDelta entire_send_duration = + tick_clock_->NowTicks() - entire_send_start_ticks_; UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.Duration", - duration); + entire_send_duration); + + base::TimeDelta loader_only_duration = + tick_clock_->NowTicks() - loader_only_start_ticks_; + // This histogram purposefully matches name and bucket size used in + // RemoteSuggestionsFetcherImpl. + UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", loader_only_duration); base::UmaHistogramSparse("ContentSuggestions.Feed.Network.RequestStatusCode", status_code); diff --git a/chromium/components/feed/core/feed_networking_host_unittest.cc b/chromium/components/feed/core/feed_networking_host_unittest.cc index 41aa465a46d..a2267b7ab3f 100644 --- a/chromium/components/feed/core/feed_networking_host_unittest.cc +++ b/chromium/components/feed/core/feed_networking_host_unittest.cc @@ -6,6 +6,7 @@ #include <utility> +#include "base/bind.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_split.h" #include "base/test/bind_test_util.h" diff --git a/chromium/components/feed/core/feed_scheduler_host.cc b/chromium/components/feed/core/feed_scheduler_host.cc index f7a488efa63..52e4d45d584 100644 --- a/chromium/components/feed/core/feed_scheduler_host.cc +++ b/chromium/components/feed/core/feed_scheduler_host.cc @@ -31,6 +31,33 @@ namespace { using TriggerType = FeedSchedulerHost::TriggerType; using UserClass = UserClassifier::UserClass; +// Enum for the relation between boolean fields the Feed and host both track. +// Reported through UMA and must match the corresponding definition in +// enums.xml +enum class FeedHostMismatch { + kNeitherAreSet = 0, + kFeedIsSetOnly = 1, + kHostIsSetOnly = 2, + kBothAreSet = 3, + kMaxValue = kBothAreSet, +}; + +// Copies boolean args into temps to avoid evaluating them multiple times. +#define UMA_HISTOGRAM_MISMATCH(name, feed_is_set, host_is_set) \ + do { \ + bool copied_feed_is_set = feed_is_set; \ + bool copied_host_is_set = host_is_set; \ + FeedHostMismatch status = FeedHostMismatch::kNeitherAreSet; \ + if (copied_feed_is_set && copied_host_is_set) { \ + status = FeedHostMismatch::kBothAreSet; \ + } else if (copied_feed_is_set) { \ + status = FeedHostMismatch::kFeedIsSetOnly; \ + } else if (copied_host_is_set) { \ + status = FeedHostMismatch::kHostIsSetOnly; \ + } \ + UMA_HISTOGRAM_ENUMERATION(name, status); \ + } while (false); + struct ParamPair { std::string name; double default_value; @@ -139,6 +166,66 @@ void ReportAgeWithSuffix(const std::string& qualified_trigger, /*bucket_count=*/50); } +void ReportReasonForNotRefreshingByBehavior( + NativeRequestBehavior behavior, + FeedSchedulerHost::ShouldRefreshResult status) { + DCHECK_NE(status, FeedSchedulerHost::kShouldRefresh); + switch (behavior) { + case kNoRequestWithWait: + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." + "NoRequestWithWait", + status); + break; + case kNoRequestWithContent: + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." + "NoRequestWithContent", + status); + break; + case kNoRequestWithTimeout: + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." + "NoRequestWithTimeout", + status); + break; + case kUnknown: + case kRequestWithWait: + case kRequestWithContent: + case kRequestWithTimeout: + NOTREACHED(); + break; + } +} + +void ReportReasonForNotRefreshingByTrigger( + FeedSchedulerHost::TriggerType trigger_type, + FeedSchedulerHost::ShouldRefreshResult status) { + DCHECK_NE(status, FeedSchedulerHost::kShouldRefresh); + switch (trigger_type) { + case FeedSchedulerHost::TriggerType::kNtpShown: + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." + "RequestByNtpShown", + status); + break; + case FeedSchedulerHost::TriggerType::kForegrounded: + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." + "RequestByForegrounded", + status); + break; + case FeedSchedulerHost::TriggerType::kFixedTimer: + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." + "RequestByFixedTimer", + status); + break; + } +} + +const int kHttpStatusOk = 200; + } // namespace FeedSchedulerHost::FeedSchedulerHost(PrefService* profile_prefs, @@ -210,22 +297,60 @@ NativeRequestBehavior FeedSchedulerHost::ShouldSessionRequestData( bool has_content, base::Time content_creation_date_time, bool has_outstanding_request) { - // The scheduler may not always know of outstanding requests, but the Feed - // should know about them all, and the scheduler should be notified upon - // completion of all requests. We should never encounter a scenario where only - // the scheduler thinks there is an outstanding request. - - // TODO(skym): Update this to use kTimeoutDurationSeconds. - // DCHECK(has_outstanding_request || !tracking_oustanding_request_); + // Both the Feed and the scheduler track if there are outstanding requests. + // It's possible that this data gets out of sync. We treat the Feed as + // authoritative and we change our values to match. + UMA_HISTOGRAM_MISMATCH("ContentSuggestions.Feed.Scheduler.OutstandingRequest", + has_outstanding_request, + !outstanding_request_until_.is_null()); + if (has_outstanding_request == outstanding_request_until_.is_null()) { + if (has_outstanding_request) { + outstanding_request_until_ = + clock_->Now() + + base::TimeDelta::FromSeconds(kTimeoutDurationSeconds.Get()); + } else { + outstanding_request_until_ = base::Time(); + } + } - if (outstanding_request_until_.is_null() && has_outstanding_request) { - outstanding_request_until_ = - clock_->Now() + - base::TimeDelta::FromSeconds(kTimeoutDurationSeconds.Get()); + // It seems to be possible for the scheduler's tracking of having content to + // get out of sync with the Feed. Root cause is currently unknown, but similar + // to outstanding request handling, we can repair with the information we + // have. + bool scheduler_thinks_has_content = + !profile_prefs_->FindPreference(prefs::kLastFetchAttemptTime) + ->IsDefaultValue(); + UMA_HISTOGRAM_MISMATCH("ContentSuggestions.Feed.Scheduler.HasContent", + has_content, scheduler_thinks_has_content); + if (has_content != scheduler_thinks_has_content) { + if (has_content) { + profile_prefs_->SetTime(prefs::kLastFetchAttemptTime, + content_creation_date_time); + } else { + profile_prefs_->ClearPref(prefs::kLastFetchAttemptTime); + } + } else if (has_content) { // && scheduler_thinks_has_content + // Split into two histograms so the difference is always positive. + base::Time last_attempt = + profile_prefs_->GetTime(prefs::kLastFetchAttemptTime); + if (content_creation_date_time > last_attempt) { + base::TimeDelta difference = (content_creation_date_time - last_attempt); + UMA_HISTOGRAM_CUSTOM_TIMES( + "ContentSuggestions.Feed.Scheduler.ContentAgeDifference.FeedIsOlder", + difference, base::TimeDelta::FromMilliseconds(1), + base::TimeDelta::FromDays(7), 100); + } else { + base::TimeDelta difference = (last_attempt - content_creation_date_time); + UMA_HISTOGRAM_CUSTOM_TIMES( + "ContentSuggestions.Feed.Scheduler.ContentAgeDifference.HostIsOlder", + difference, base::TimeDelta::FromMilliseconds(1), + base::TimeDelta::FromDays(7), 100); + } } NativeRequestBehavior behavior; - if (ShouldRefresh(TriggerType::kNtpShown)) { + ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kNtpShown); + if (kShouldRefresh == refresh_status) { if (!has_content) { behavior = kRequestWithWait; } else if (IsContentStale(content_creation_date_time)) { @@ -248,6 +373,7 @@ NativeRequestBehavior FeedSchedulerHost::ShouldSessionRequestData( } else { behavior = kNoRequestWithContent; } + ReportReasonForNotRefreshingByBehavior(behavior, refresh_status); } OnSuggestionsShown(); @@ -262,6 +388,7 @@ void FeedSchedulerHost::OnReceiveNewContent( base::Time content_creation_date_time) { profile_prefs_->SetTime(prefs::kLastFetchAttemptTime, content_creation_date_time); + last_fetch_status_ = kHttpStatusOk; TryRun(std::move(fixed_timer_completion_)); ScheduleFixedTimerWakeUp(GetTriggerThreshold(TriggerType::kFixedTimer)); outstanding_request_until_ = base::Time(); @@ -273,6 +400,7 @@ void FeedSchedulerHost::OnReceiveNewContent( void FeedSchedulerHost::OnRequestError(int network_response_code) { profile_prefs_->SetTime(prefs::kLastFetchAttemptTime, clock_->Now()); + last_fetch_status_ = network_response_code; TryRun(std::move(fixed_timer_completion_)); outstanding_request_until_ = base::Time(); time_until_first_shown_trigger_reported_ = false; @@ -282,8 +410,13 @@ void FeedSchedulerHost::OnRequestError(int network_response_code) { void FeedSchedulerHost::OnForegrounded() { DCHECK(refresh_callback_); - if (ShouldRefresh(TriggerType::kForegrounded)) { + ShouldRefreshResult refresh_status = + ShouldRefresh(TriggerType::kForegrounded); + if (kShouldRefresh == refresh_status) { refresh_callback_.Run(); + } else { + ReportReasonForNotRefreshingByTrigger(TriggerType::kForegrounded, + refresh_status); } } @@ -298,7 +431,8 @@ void FeedSchedulerHost::OnFixedTimer(base::OnceClosure on_completion) { CancelFixedTimerWakeUp(); } - if (ShouldRefresh(TriggerType::kFixedTimer)) { + ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kFixedTimer); + if (kShouldRefresh == refresh_status) { // There shouldn't typically be anything in |fixed_timer_completion_| right // now, but if there was, run it before we replace it. TryRun(std::move(fixed_timer_completion_)); @@ -306,6 +440,8 @@ void FeedSchedulerHost::OnFixedTimer(base::OnceClosure on_completion) { fixed_timer_completion_ = std::move(on_completion); refresh_callback_.Run(); } else { + ReportReasonForNotRefreshingByTrigger(TriggerType::kFixedTimer, + refresh_status); // The task driving this doesn't need to stay around, since no work is being // done on its behalf. TryRun(std::move(on_completion)); @@ -343,49 +479,68 @@ bool FeedSchedulerHost::OnArticlesCleared(bool suppress_refreshes) { suppress_refreshes_until_ = clock_->Now() + base::TimeDelta::FromMinutes(kSuppressRefreshDurationMinutes.Get()); - } else if (ShouldRefresh(TriggerType::kNtpShown)) { + } + + ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kNtpShown); + if (kShouldRefresh == refresh_status) { // Instead of using |refresh_callback_|, instead return our desire to // refresh back up to our caller. This allows more information to be given // all at once to the Feed which allows it to act more intelligently. return true; + } else { + ReportReasonForNotRefreshingByTrigger(TriggerType::kNtpShown, + refresh_status); } return false; } +UserClassifier* FeedSchedulerHost::GetUserClassifierForDebugging() { + return &user_classifier_; +} + +base::Time FeedSchedulerHost::GetSuppressRefreshesUntilForDebugging() const { + return suppress_refreshes_until_; +} + +int FeedSchedulerHost::GetLastFetchStatusForDebugging() const { + return last_fetch_status_; +} + void FeedSchedulerHost::OnEulaAccepted() { OnForegrounded(); } -bool FeedSchedulerHost::ShouldRefresh(TriggerType trigger) { +FeedSchedulerHost::ShouldRefreshResult FeedSchedulerHost::ShouldRefresh( + TriggerType trigger) { if (clock_->Now() < outstanding_request_until_) { DVLOG(2) << "Outstanding request stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshOutstandingRequest; } if (base::ContainsKey(disabled_triggers_, trigger)) { DVLOG(2) << "Disabled trigger stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshTriggerDisabled; } if (net::NetworkChangeNotifier::IsOffline()) { DVLOG(2) << "Network is offline stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshNetworkOffline; } if (eula_accepted_notifier_ && !eula_accepted_notifier_->IsEulaAccepted()) { DVLOG(2) << "EULA not being accepted stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshEulaNotAccepted; } if (!profile_prefs_->GetBoolean(prefs::kArticlesListVisible)) { DVLOG(2) << "Articles being hidden stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshArticlesHidden; } base::TimeDelta attempt_age = @@ -407,13 +562,13 @@ bool FeedSchedulerHost::ShouldRefresh(TriggerType trigger) { if (clock_->Now() < suppress_refreshes_until_) { DVLOG(2) << "Refresh suppression until " << suppress_refreshes_until_ << " stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshRefreshSuppressed; } if (attempt_age < GetTriggerThreshold(trigger)) { DVLOG(2) << "Last attempt age of " << attempt_age << " stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshNotStale; } auto throttlerIter = throttlers_.find(user_class); @@ -421,7 +576,7 @@ bool FeedSchedulerHost::ShouldRefresh(TriggerType trigger) { !throttlerIter->second->RequestQuota()) { DVLOG(2) << "Throttler stopped refresh from trigger " << static_cast<int>(trigger); - return false; + return kDontRefreshRefreshThrottled; } switch (trigger) { @@ -443,7 +598,7 @@ bool FeedSchedulerHost::ShouldRefresh(TriggerType trigger) { clock_->Now() + base::TimeDelta::FromSeconds(kTimeoutDurationSeconds.Get()); - return true; + return kShouldRefresh; } bool FeedSchedulerHost::IsContentStale(base::Time content_creation_date_time) { diff --git a/chromium/components/feed/core/feed_scheduler_host.h b/chromium/components/feed/core/feed_scheduler_host.h index 31469361e43..d2455dec39d 100644 --- a/chromium/components/feed/core/feed_scheduler_host.h +++ b/chromium/components/feed/core/feed_scheduler_host.h @@ -60,6 +60,24 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { kMaxValue = kFixedTimer }; + // Enum for the status of the refresh, reported through UMA. + // If any new values are added, update the corresponding definition in + // enums.xml. + // These values are persisted to logs. Entries should not be renumbered and + // numeric values should never be reused. + enum ShouldRefreshResult { + kShouldRefresh = 0, + kDontRefreshOutstandingRequest = 1, + kDontRefreshTriggerDisabled = 2, + kDontRefreshNetworkOffline = 3, + kDontRefreshEulaNotAccepted = 4, + kDontRefreshArticlesHidden = 5, + kDontRefreshRefreshSuppressed = 6, + kDontRefreshNotStale = 7, + kDontRefreshRefreshThrottled = 8, + kMaxValue = kDontRefreshRefreshThrottled, + }; + FeedSchedulerHost(PrefService* profile_prefs, PrefService* local_state, base::Clock* clock); @@ -116,6 +134,15 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { // the return value, and if true, the caller should start a refresh. bool OnArticlesCleared(bool suppress_refreshes); + // Surface user_classifier_ for internals debugging page. + UserClassifier* GetUserClassifierForDebugging(); + + // Surface suppress_refreshes_until_ for internals debugging page. + base::Time GetSuppressRefreshesUntilForDebugging() const; + + // Surface last_fetch_status_ for internals debugging page. + int GetLastFetchStatusForDebugging() const; + private: FRIEND_TEST_ALL_PREFIXES(FeedSchedulerHostTest, GetTriggerThreshold); @@ -126,7 +153,7 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { // If this method is called and returns true we presume the refresh will // happen, therefore we report metrics respectively and update // |tracking_oustanding_request_|. - bool ShouldRefresh(TriggerType trigger); + ShouldRefreshResult ShouldRefresh(TriggerType trigger); // Decides if content whose age is the difference between now and // |content_creation_date_time| is old enough to be considered stale. @@ -198,6 +225,9 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { base::flat_map<UserClassifier::UserClass, std::unique_ptr<RefreshThrottler>> throttlers_; + // Status of the last fetch for debugging. + int last_fetch_status_ = 0; + DISALLOW_COPY_AND_ASSIGN(FeedSchedulerHost); }; diff --git a/chromium/components/feed/core/feed_scheduler_host_unittest.cc b/chromium/components/feed/core/feed_scheduler_host_unittest.cc index d7ba47085c6..b7bfaca0f3e 100644 --- a/chromium/components/feed/core/feed_scheduler_host_unittest.cc +++ b/chromium/components/feed/core/feed_scheduler_host_unittest.cc @@ -918,6 +918,28 @@ TEST_F(FeedSchedulerHostTest, IncorporatesExternalOustandingRequest) { // prevent the OnForegrounded() from requesting a refresh. scheduler()->OnForegrounded(); EXPECT_EQ(0, refresh_call_count()); + + EXPECT_EQ(kRequestWithWait, + scheduler()->ShouldSessionRequestData( + /*has_content*/ false, /*content_creation_date_time*/ Time(), + /*has_outstanding_request*/ false)); +} + +TEST_F(FeedSchedulerHostTest, IncorporatesExternalHasContent) { + Time now = test_clock()->Now(); + EXPECT_EQ(Time(), profile_prefs()->GetTime(prefs::kLastFetchAttemptTime)); + + EXPECT_EQ(kNoRequestWithContent, + scheduler()->ShouldSessionRequestData( + /*has_content*/ true, now, /*has_outstanding_request*/ false)); + EXPECT_EQ(now, profile_prefs()->GetTime(prefs::kLastFetchAttemptTime)); + + // Use has_outstanding_request of true to keep the scheduler from actually + // triggering the refresh. We want to track the change to its internal state. + EXPECT_EQ(kNoRequestWithWait, scheduler()->ShouldSessionRequestData( + /*has_content*/ false, base::Time(), + /*has_outstanding_request*/ true)); + EXPECT_EQ(Time(), profile_prefs()->GetTime(prefs::kLastFetchAttemptTime)); } TEST_F(FeedSchedulerHostTest, TimeUntilFirstMetrics) { @@ -933,7 +955,7 @@ TEST_F(FeedSchedulerHostTest, TimeUntilFirstMetrics) { EXPECT_EQ(0U, histogram_tester.GetAllSamples(forgroundedHistogram).size()); scheduler()->ShouldSessionRequestData( - /*has_content*/ false, now, /*has_outstanding_request*/ false); + /*has_content*/ true, now, /*has_outstanding_request*/ false); EXPECT_EQ(1, histogram_tester.GetBucketCount(ntpOpenedHistogram, 0)); EXPECT_EQ(0U, histogram_tester.GetAllSamples(forgroundedHistogram).size()); @@ -942,7 +964,7 @@ TEST_F(FeedSchedulerHostTest, TimeUntilFirstMetrics) { EXPECT_EQ(1, histogram_tester.GetBucketCount(forgroundedHistogram, 0)); scheduler()->ShouldSessionRequestData( - /*has_content*/ false, now, /*has_outstanding_request*/ false); + /*has_content*/ true, now, /*has_outstanding_request*/ false); scheduler()->OnForegrounded(); EXPECT_EQ(1, histogram_tester.GetBucketCount(ntpOpenedHistogram, 0)); EXPECT_EQ(1, histogram_tester.GetBucketCount(forgroundedHistogram, 0)); @@ -952,7 +974,7 @@ TEST_F(FeedSchedulerHostTest, TimeUntilFirstMetrics) { scheduler()->OnRequestError(0); scheduler()->ShouldSessionRequestData( - /*has_content*/ false, now, /*has_outstanding_request*/ false); + /*has_content*/ true, now, /*has_outstanding_request*/ false); scheduler()->OnForegrounded(); EXPECT_EQ(2, histogram_tester.GetBucketCount(ntpOpenedHistogram, 0)); EXPECT_EQ(2, histogram_tester.GetBucketCount(forgroundedHistogram, 0)); diff --git a/chromium/components/feed/core/proto/BUILD.gn b/chromium/components/feed/core/proto/BUILD.gn index 2dfff8a2e5e..3a0b51b9a76 100644 --- a/chromium/components/feed/core/proto/BUILD.gn +++ b/chromium/components/feed/core/proto/BUILD.gn @@ -6,7 +6,6 @@ import("//third_party/protobuf/proto_library.gni") proto_library("proto") { sources = [ - "cached_image.proto", "content_storage.proto", "journal_storage.proto", ] diff --git a/chromium/components/feed/core/proto/cached_image.proto b/chromium/components/feed/core/proto/cached_image.proto deleted file mode 100644 index 960143697eb..00000000000 --- a/chromium/components/feed/core/proto/cached_image.proto +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -syntax = "proto2"; - -option optimize_for = LITE_RUNTIME; - -package feed; - -message CachedImageProto { - // The URL of the original source, ex. https://www.chromium.org/image.png. - optional string url = 1; - - // Raw image data fetched from network. - optional bytes data = 2; - - // Last used time (in microseconds since the origin (or "zero") point.). - optional int64 last_used_time = 3; -} diff --git a/chromium/components/feed/core/user_classifier_unittest.cc b/chromium/components/feed/core/user_classifier_unittest.cc index 8a8eb0302fe..ddb6895247b 100644 --- a/chromium/components/feed/core/user_classifier_unittest.cc +++ b/chromium/components/feed/core/user_classifier_unittest.cc @@ -296,7 +296,7 @@ TEST_P(FeedUserClassifierEventTest, Eq(rate_after_a_year)); } -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( , // An empty prefix for the parametrized tests names (no need to // distinguish the only instance we make here). FeedUserClassifierEventTest, |