diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2019-02-13 15:05:36 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2019-02-14 10:33:47 +0000 |
commit | e684a3455bcc29a6e3e66a004e352dea4e1141e7 (patch) | |
tree | d55b4003bde34d7d05f558f02cfd82b2a66a7aac /chromium/components/offline_pages | |
parent | 2b94bfe47ccb6c08047959d1c26e392919550e86 (diff) | |
download | qtwebengine-chromium-e684a3455bcc29a6e3e66a004e352dea4e1141e7.tar.gz |
BASELINE: Update Chromium to 72.0.3626.110 and Ninja to 1.9.0
Change-Id: Ic57220b00ecc929a893c91f5cc552f5d3e99e922
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/offline_pages')
167 files changed, 2813 insertions, 1339 deletions
diff --git a/chromium/components/offline_pages/content/background_loader/background_loader_contents.cc b/chromium/components/offline_pages/content/background_loader/background_loader_contents.cc index ed7a6be5651..a8636dcd317 100644 --- a/chromium/components/offline_pages/content/background_loader/background_loader_contents.cc +++ b/chromium/components/offline_pages/content/background_loader/background_loader_contents.cc @@ -148,6 +148,10 @@ void BackgroundLoaderContents::AdjustPreviewsStateForNavigation( } } +bool BackgroundLoaderContents::ShouldAllowLazyLoad() { + return false; +} + BackgroundLoaderContents::BackgroundLoaderContents() : browser_context_(nullptr) { web_contents_.reset(); diff --git a/chromium/components/offline_pages/content/background_loader/background_loader_contents.h b/chromium/components/offline_pages/content/background_loader/background_loader_contents.h index 4b55d90239d..7c64f085b56 100644 --- a/chromium/components/offline_pages/content/background_loader/background_loader_contents.h +++ b/chromium/components/offline_pages/content/background_loader/background_loader_contents.h @@ -88,10 +88,10 @@ class BackgroundLoaderContents : public content::WebContentsDelegate { bool CheckMediaAccessPermission(content::RenderFrameHost* render_frame_host, const GURL& security_origin, content::MediaStreamType type) override; - void AdjustPreviewsStateForNavigation( content::WebContents* web_contents, content::PreviewsState* previews_state) override; + bool ShouldAllowLazyLoad() override; private: friend class BackgroundLoaderContentsTest; diff --git a/chromium/components/offline_pages/core/BUILD.gn b/chromium/components/offline_pages/core/BUILD.gn index 4da54a9db93..1c9cf1d0ed6 100644 --- a/chromium/components/offline_pages/core/BUILD.gn +++ b/chromium/components/offline_pages/core/BUILD.gn @@ -12,6 +12,8 @@ static_library("core") { "archive_manager.h", "archive_validator.cc", "archive_validator.h", + "auto_fetch.cc", + "auto_fetch.h", "background_snapshot_controller.cc", "background_snapshot_controller.h", "client_id.cc", @@ -58,6 +60,8 @@ static_library("core") { "model/store_thumbnail_task.h", "model/update_file_path_task.cc", "model/update_file_path_task.h", + "offline_clock.cc", + "offline_clock.h", "offline_event_logger.cc", "offline_event_logger.h", "offline_page_archiver.cc", @@ -119,6 +123,8 @@ static_library("test_support") { "stub_offline_page_model.h", "stub_system_download_manager.cc", "stub_system_download_manager.h", + "test_scoped_offline_clock.cc", + "test_scoped_offline_clock.h", ] deps = [ @@ -152,6 +158,7 @@ source_set("unit_tests") { sources = [ "archive_manager_unittest.cc", "archive_validator_unittest.cc", + "auto_fetch_unittest.cc", "background_snapshot_controller_unittest.cc", "client_policy_controller_unittest.cc", "model/add_page_task_unittest.cc", diff --git a/chromium/components/offline_pages/core/archive_manager.cc b/chromium/components/offline_pages/core/archive_manager.cc index f24e1690be4..f139f6cfd84 100644 --- a/chromium/components/offline_pages/core/archive_manager.cc +++ b/chromium/components/offline_pages/core/archive_manager.cc @@ -14,7 +14,7 @@ #include "base/metrics/histogram_macros.h" #include "base/sequenced_task_runner.h" #include "base/strings/utf_string_conversions.h" -#include "base/sys_info.h" +#include "base/system/sys_info.h" #include "base/threading/thread_task_runner_handle.h" #include "build/build_config.h" diff --git a/chromium/components/offline_pages/core/archive_manager_unittest.cc b/chromium/components/offline_pages/core/archive_manager_unittest.cc index 5428468ee8b..94346d9805c 100644 --- a/chromium/components/offline_pages/core/archive_manager_unittest.cc +++ b/chromium/components/offline_pages/core/archive_manager_unittest.cc @@ -14,7 +14,7 @@ #include "base/files/file_path.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" -#include "base/sys_info.h" +#include "base/system/sys_info.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/test_simple_task_runner.h" #include "base/threading/thread_task_runner_handle.h" diff --git a/chromium/components/offline_pages/core/auto_fetch.cc b/chromium/components/offline_pages/core/auto_fetch.cc new file mode 100644 index 00000000000..95b55698500 --- /dev/null +++ b/chromium/components/offline_pages/core/auto_fetch.cc @@ -0,0 +1,37 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/auto_fetch.h" + +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_piece.h" +#include "components/offline_pages/core/client_namespace_constants.h" + +namespace offline_pages { +namespace auto_fetch { +ClientIdMetadata::ClientIdMetadata() = default; +ClientIdMetadata::ClientIdMetadata(const ClientIdMetadata&) = default; + +ClientId MakeClientId(const ClientIdMetadata& metadata) { + // Here, the 'A' prefix is used so that future versions can easily change the + // format if necessary. + return ClientId(kAutoAsyncNamespace, + base::StrCat({"A", std::to_string(metadata.android_tab_id)})); +} + +base::Optional<ClientIdMetadata> ExtractMetadata(const ClientId& id) { + if (id.name_space != kAutoAsyncNamespace) + return base::nullopt; + if (id.id.empty() || id.id[0] != 'A') + return base::nullopt; + ClientIdMetadata metadata; + if (!base::StringToInt(base::StringPiece(id.id).substr(1), + &metadata.android_tab_id)) + return base::nullopt; + return metadata; +} + +} // namespace auto_fetch +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/auto_fetch.h b/chromium/components/offline_pages/core/auto_fetch.h new file mode 100644 index 00000000000..25ecee5a540 --- /dev/null +++ b/chromium/components/offline_pages/core/auto_fetch.h @@ -0,0 +1,34 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_AUTO_FETCH_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_AUTO_FETCH_H_ + +#include "base/optional.h" +#include "components/offline_pages/core/client_id.h" + +// Most auto-fetch code is in browser/offline_pages. This file contains code +// that needs to be accessed within components/offline_pages. + +namespace offline_pages { +namespace auto_fetch { + +// This metadata is stored in the |ClientId|'s |id| field. +struct ClientIdMetadata { + ClientIdMetadata(); + ClientIdMetadata(const ClientIdMetadata&); + explicit ClientIdMetadata(int android_tab_id) + : android_tab_id(android_tab_id) {} + // ID of the Android tab that initiated the request. + int android_tab_id; +}; + +ClientId MakeClientId(const ClientIdMetadata& metadata); +// Extract metadata from a |ClientId| that was created with |MakeClientId|. +base::Optional<ClientIdMetadata> ExtractMetadata(const ClientId& id); + +} // namespace auto_fetch +} // namespace offline_pages + +#endif // COMPONENTS_OFFLINE_PAGES_CORE_AUTO_FETCH_H_ diff --git a/chromium/components/offline_pages/core/auto_fetch_unittest.cc b/chromium/components/offline_pages/core/auto_fetch_unittest.cc new file mode 100644 index 00000000000..e677426c88d --- /dev/null +++ b/chromium/components/offline_pages/core/auto_fetch_unittest.cc @@ -0,0 +1,36 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/auto_fetch.h" + +#include "components/offline_pages/core/client_namespace_constants.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace offline_pages { +namespace auto_fetch { +namespace { + +TEST(AutoFetch, MakeClientId) { + EXPECT_EQ(ClientId(kAutoAsyncNamespace, "A123"), + auto_fetch::MakeClientId(auto_fetch::ClientIdMetadata(123))); +} + +TEST(AutoFetch, ExtractMetadataSuccess) { + base::Optional<auto_fetch::ClientIdMetadata> metadata = + auto_fetch::ExtractMetadata( + auto_fetch::MakeClientId(auto_fetch::ClientIdMetadata(123))); + ASSERT_TRUE(metadata); + EXPECT_EQ(123, metadata.value().android_tab_id); +} + +TEST(AutoFetch, ExtractMetadataFail) { + EXPECT_FALSE( + auto_fetch::ExtractMetadata(ClientId(kAutoAsyncNamespace, "123"))); + EXPECT_FALSE(auto_fetch::ExtractMetadata(ClientId(kAutoAsyncNamespace, ""))); + EXPECT_FALSE(auto_fetch::ExtractMetadata(ClientId("other", "A123"))); +} + +} // namespace +} // namespace auto_fetch +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/background/BUILD.gn b/chromium/components/offline_pages/core/background/BUILD.gn index 102bafe21cd..817f94e06e1 100644 --- a/chromium/components/offline_pages/core/background/BUILD.gn +++ b/chromium/components/offline_pages/core/background/BUILD.gn @@ -28,6 +28,8 @@ static_library("background_offliner") { "mark_attempt_aborted_task.h", "mark_attempt_completed_task.cc", "mark_attempt_completed_task.h", + "mark_attempt_deferred_task.cc", + "mark_attempt_deferred_task.h", "mark_attempt_started_task.cc", "mark_attempt_started_task.h", "offliner.h", @@ -86,6 +88,7 @@ static_library("test_support") { "scheduler_stub.h", "test_request_queue_store.cc", "test_request_queue_store.h", + "test_util.cc", ] deps = [ diff --git a/chromium/components/offline_pages/core/background/add_request_task_unittest.cc b/chromium/components/offline_pages/core/background/add_request_task_unittest.cc index a2dc2efa0c7..2892dde7cc4 100644 --- a/chromium/components/offline_pages/core/background/add_request_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/add_request_task_unittest.cc @@ -9,9 +9,11 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -86,7 +88,7 @@ void AddRequestTaskTest::InitializeStoreDone(bool success) { TEST_F(AddRequestTaskTest, AddSingleRequest) { InitializeStore(&store_); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); AddRequestTask task(&store_, request_1, @@ -110,7 +112,7 @@ TEST_F(AddRequestTaskTest, AddSingleRequest) { TEST_F(AddRequestTaskTest, AddMultipleRequests) { InitializeStore(&store_); - base::Time creation_time_1 = base::Time::Now(); + base::Time creation_time_1 = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time_1, true); AddRequestTask task(&store_, request_1, @@ -122,7 +124,7 @@ TEST_F(AddRequestTaskTest, AddMultipleRequests) { EXPECT_EQ(ItemActionStatus::SUCCESS, last_status()); ClearResults(); - base::Time creation_time_2 = base::Time::Now(); + base::Time creation_time_2 = OfflineClock()->Now(); SavePageRequest request_2(kRequestId2, kUrl2, kClientId2, creation_time_2, true); AddRequestTask task_2(&store_, request_2, @@ -149,7 +151,7 @@ TEST_F(AddRequestTaskTest, AddMultipleRequests) { TEST_F(AddRequestTaskTest, AddDuplicateRequest) { InitializeStore(&store_); - base::Time creation_time_1 = base::Time::Now(); + base::Time creation_time_1 = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time_1, true); AddRequestTask task(&store_, request_1, @@ -161,7 +163,7 @@ TEST_F(AddRequestTaskTest, AddDuplicateRequest) { EXPECT_EQ(ItemActionStatus::SUCCESS, last_status()); ClearResults(); - base::Time creation_time_2 = base::Time::Now(); + base::Time creation_time_2 = OfflineClock()->Now(); // This was has the same request ID. SavePageRequest request_2(kRequestId1, kUrl2, kClientId2, creation_time_2, true); diff --git a/chromium/components/offline_pages/core/background/change_requests_state_task_unittest.cc b/chromium/components/offline_pages/core/background/change_requests_state_task_unittest.cc index c81d93a25b4..cb68eb5a7b2 100644 --- a/chromium/components/offline_pages/core/background/change_requests_state_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/change_requests_state_task_unittest.cc @@ -9,9 +9,11 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -41,7 +43,7 @@ class ChangeRequestsStateTaskTest : public RequestQueueTaskTestBase { }; void ChangeRequestsStateTaskTest::AddItemsToStore() { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); store_.AddRequest(request_1, diff --git a/chromium/components/offline_pages/core/background/cleanup_task_unittest.cc b/chromium/components/offline_pages/core/background/cleanup_task_unittest.cc index 3c6b494b2d9..c6576b1dd4b 100644 --- a/chromium/components/offline_pages/core/background/cleanup_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/cleanup_task_unittest.cc @@ -10,6 +10,7 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/offliner_policy.h" #include "components/offline_pages/core/background/request_coordinator.h" #include "components/offline_pages/core/background/request_coordinator_event_logger.h" @@ -18,6 +19,7 @@ #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/save_page_request.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -159,7 +161,7 @@ void CleanupTaskTest::MakeFactoryAndTask() { } TEST_F(CleanupTaskTest, CleanupExpiredRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); base::Time expired_time = creation_time - base::TimeDelta::FromSeconds( policy()->GetRequestExpirationTimeInSeconds() + 10); @@ -183,7 +185,7 @@ TEST_F(CleanupTaskTest, CleanupExpiredRequest) { } TEST_F(CleanupTaskTest, CleanupStartCountExceededRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // Request2 will have an exceeded start count. SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); @@ -205,7 +207,7 @@ TEST_F(CleanupTaskTest, CleanupStartCountExceededRequest) { } TEST_F(CleanupTaskTest, CleanupCompletionCountExceededRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // Request2 will have an exceeded completion count. SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); @@ -227,7 +229,7 @@ TEST_F(CleanupTaskTest, CleanupCompletionCountExceededRequest) { } TEST_F(CleanupTaskTest, IgnoreRequestInProgress) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // Both requests will have an exceeded completion count. // The first request will be marked as started. SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, diff --git a/chromium/components/offline_pages/core/background/get_requests_task_unittest.cc b/chromium/components/offline_pages/core/background/get_requests_task_unittest.cc index 0f78a3d20ba..83f1d53a2d5 100644 --- a/chromium/components/offline_pages/core/background/get_requests_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/get_requests_task_unittest.cc @@ -9,9 +9,11 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -50,13 +52,13 @@ class GetRequestsTaskTest : public RequestQueueTaskTestBase { }; void GetRequestsTaskTest::AddItemsToStore(RequestQueueStore* store) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); store->AddRequest(request_1, base::BindOnce(&GetRequestsTaskTest::AddRequestDone, base::Unretained(this))); - creation_time = base::Time::Now(); + creation_time = OfflineClock()->Now(); SavePageRequest request_2(kRequestId2, kUrl2, kClientId2, creation_time, true); store->AddRequest(request_2, diff --git a/chromium/components/offline_pages/core/background/mark_attempt_aborted_task_unittest.cc b/chromium/components/offline_pages/core/background/mark_attempt_aborted_task_unittest.cc index 37db9a4a4d5..1688862b5f9 100644 --- a/chromium/components/offline_pages/core/background/mark_attempt_aborted_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/mark_attempt_aborted_task_unittest.cc @@ -9,11 +9,13 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/change_requests_state_task.h" #include "components/offline_pages/core/background/mark_attempt_started_task.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -44,7 +46,7 @@ class MarkAttemptAbortedTaskTest : public RequestQueueTaskTestBase { }; void MarkAttemptAbortedTaskTest::AddItemToStore(RequestQueueStore* store) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); store->AddRequest(request_1, diff --git a/chromium/components/offline_pages/core/background/mark_attempt_completed_task_unittest.cc b/chromium/components/offline_pages/core/background/mark_attempt_completed_task_unittest.cc index e4585540eb0..a75923e436e 100644 --- a/chromium/components/offline_pages/core/background/mark_attempt_completed_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/mark_attempt_completed_task_unittest.cc @@ -8,9 +8,11 @@ #include <utility> #include "base/bind.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -37,10 +39,10 @@ class MarkAttemptCompletedTaskTest : public RequestQueueTaskTestBase { }; void MarkAttemptCompletedTaskTest::AddStartedItemToStore() { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); - request_1.MarkAttemptStarted(base::Time::Now()); + request_1.MarkAttemptStarted(OfflineClock()->Now()); store_.AddRequest( request_1, base::BindOnce(&MarkAttemptCompletedTaskTest::AddRequestDone, base::Unretained(this))); diff --git a/chromium/components/offline_pages/core/background/mark_attempt_deferred_task.cc b/chromium/components/offline_pages/core/background/mark_attempt_deferred_task.cc new file mode 100644 index 00000000000..ca2786e46e4 --- /dev/null +++ b/chromium/components/offline_pages/core/background/mark_attempt_deferred_task.cc @@ -0,0 +1,40 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/background/mark_attempt_deferred_task.h" + +#include <utility> + +#include "base/bind.h" +#include "base/time/clock.h" +#include "base/time/time.h" +#include "components/offline_pages/core/offline_clock.h" + +namespace offline_pages { + +MarkAttemptDeferredTask::MarkAttemptDeferredTask( + RequestQueueStore* store, + int64_t request_id, + RequestQueueStore::UpdateCallback callback) + : UpdateRequestTask(store, request_id, std::move(callback)) {} + +MarkAttemptDeferredTask::~MarkAttemptDeferredTask() {} + +void MarkAttemptDeferredTask::UpdateRequestImpl( + UpdateRequestsResult read_result) { + if (!ValidateReadResult(read_result)) { + CompleteWithResult(std::move(read_result)); + return; + } + + // It is perfectly fine to reuse the read_result.updated_items collection, as + // it is owned by this callback and will be destroyed when out of scope. + read_result.updated_items[0].MarkAttemptDeferred(OfflineClock()->Now()); + store()->UpdateRequests( + read_result.updated_items, + base::BindOnce(&MarkAttemptDeferredTask::CompleteWithResult, + GetWeakPtr())); +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/background/mark_attempt_deferred_task.h b/chromium/components/offline_pages/core/background/mark_attempt_deferred_task.h new file mode 100644 index 00000000000..35de79e35ae --- /dev/null +++ b/chromium/components/offline_pages/core/background/mark_attempt_deferred_task.h @@ -0,0 +1,34 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_BACKGROUND_MARK_ATTEMPT_DEFERRED_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_BACKGROUND_MARK_ATTEMPT_DEFERRED_TASK_H_ + +#include <stdint.h> +#include <memory> + +#include "components/offline_items_collection/core/fail_state.h" +#include "components/offline_pages/core/background/request_queue_results.h" +#include "components/offline_pages/core/background/update_request_task.h" +#include "components/offline_pages/task/task.h" + +namespace offline_pages { + +class RequestQueueStore; + +class MarkAttemptDeferredTask : public UpdateRequestTask { + public: + MarkAttemptDeferredTask(RequestQueueStore* store, + int64_t request_id, + RequestQueueStore::UpdateCallback callback); + ~MarkAttemptDeferredTask() override; + + protected: + // UpdateRequestTask implementation: + void UpdateRequestImpl(UpdateRequestsResult result) override; +}; + +} // namespace offline_pages + +#endif // COMPONENTS_OFFLINE_PAGES_CORE_BACKGROUND_MARK_ATTEMPT_DEFERRED_TASK_H_ diff --git a/chromium/components/offline_pages/core/background/mark_attempt_started_task.cc b/chromium/components/offline_pages/core/background/mark_attempt_started_task.cc index 50673485232..3d702fe5f54 100644 --- a/chromium/components/offline_pages/core/background/mark_attempt_started_task.cc +++ b/chromium/components/offline_pages/core/background/mark_attempt_started_task.cc @@ -7,7 +7,9 @@ #include <utility> #include "base/bind.h" +#include "base/time/clock.h" #include "base/time/time.h" +#include "components/offline_pages/core/offline_clock.h" namespace offline_pages { @@ -28,7 +30,7 @@ void MarkAttemptStartedTask::UpdateRequestImpl( // It is perfectly fine to reuse the read_result.updated_items collection, as // it is owned by this callback and will be destroyed when out of scope. - read_result.updated_items[0].MarkAttemptStarted(base::Time::Now()); + read_result.updated_items[0].MarkAttemptStarted(OfflineClock()->Now()); store()->UpdateRequests( read_result.updated_items, base::BindOnce(&MarkAttemptStartedTask::CompleteWithResult, diff --git a/chromium/components/offline_pages/core/background/mark_attempt_started_task_unittest.cc b/chromium/components/offline_pages/core/background/mark_attempt_started_task_unittest.cc index 92fab85ba69..12b3ab767b3 100644 --- a/chromium/components/offline_pages/core/background/mark_attempt_started_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/mark_attempt_started_task_unittest.cc @@ -9,9 +9,11 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -38,7 +40,7 @@ class MarkAttemptStartedTaskTest : public RequestQueueTaskTestBase { }; void MarkAttemptStartedTaskTest::AddItemToStore() { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); store_.AddRequest(request_1, @@ -83,7 +85,7 @@ TEST_F(MarkAttemptStartedTaskTest, MarkAttemptStartedWhenExists) { base::Unretained(this))); // Current time for verification. - base::Time before_time = base::Time::Now(); + base::Time before_time = OfflineClock()->Now(); task.Run(); PumpLoop(); ASSERT_TRUE(last_result()); @@ -94,7 +96,7 @@ TEST_F(MarkAttemptStartedTaskTest, MarkAttemptStartedWhenExists) { EXPECT_EQ(1UL, last_result()->updated_items.size()); EXPECT_LE(before_time, last_result()->updated_items.at(0).last_attempt_time()); - EXPECT_GE(base::Time::Now(), + EXPECT_GE(OfflineClock()->Now(), last_result()->updated_items.at(0).last_attempt_time()); EXPECT_EQ(1, last_result()->updated_items.at(0).started_attempt_count()); EXPECT_EQ(SavePageRequest::RequestState::OFFLINING, diff --git a/chromium/components/offline_pages/core/background/offliner.h b/chromium/components/offline_pages/core/background/offliner.h index d7121633bb8..87e8d1c87f4 100644 --- a/chromium/components/offline_pages/core/background/offliner.h +++ b/chromium/components/offline_pages/core/background/offliner.h @@ -21,7 +21,7 @@ class Offliner { // WARNING: You must update histograms.xml to match any changes made to // this enum (ie, OfflinePagesBackgroundOfflinerRequestStatus histogram enum). // Also update related switch code in RequestCoordinatorEventLogger. - enum RequestStatus { + enum class RequestStatus { // No status determined/reported yet. Interim status, not sent in callback. UNKNOWN = 0, // Page loaded but not (yet) saved. Interim status, not sent in callback. @@ -75,8 +75,10 @@ class Offliner { LOADING_FAILED_NET_ERROR = 19, // Loader failed to load page due to HTTP error. LOADING_FAILED_HTTP_ERROR = 20, - // NOTE: insert new values above this line and update histogram enum too. - STATUS_COUNT + // Loading was deferred because the active tab URL matches. + LOADING_DEFERRED = 21, + + kMaxValue = LOADING_DEFERRED, }; // Reports the load progress of a request. diff --git a/chromium/components/offline_pages/core/background/offliner_policy_utils.cc b/chromium/components/offline_pages/core/background/offliner_policy_utils.cc index a9383f2423e..2ba3e608e96 100644 --- a/chromium/components/offline_pages/core/background/offliner_policy_utils.cc +++ b/chromium/components/offline_pages/core/background/offliner_policy_utils.cc @@ -4,8 +4,10 @@ #include "components/offline_pages/core/background/offliner_policy_utils.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/offliner_policy.h" #include "components/offline_pages/core/background/save_page_request.h" +#include "components/offline_pages/core/offline_clock.h" namespace offline_pages { @@ -17,7 +19,7 @@ OfflinerPolicyUtils::CheckRequestExpirationStatus( DCHECK(request); DCHECK(policy); - if (base::Time::Now() - request->creation_time() >= + if (OfflineClock()->Now() - request->creation_time() >= base::TimeDelta::FromSeconds( policy->GetRequestExpirationTimeInSeconds())) { return RequestExpirationStatus::EXPIRED; diff --git a/chromium/components/offline_pages/core/background/pick_request_task.cc b/chromium/components/offline_pages/core/background/pick_request_task.cc index 37016464469..02f0ddeca3a 100644 --- a/chromium/components/offline_pages/core/background/pick_request_task.cc +++ b/chromium/components/offline_pages/core/background/pick_request_task.cc @@ -4,10 +4,14 @@ #include "components/offline_pages/core/background/pick_request_task.h" +#include <memory> #include <unordered_set> +#include <utility> +#include <vector> #include "base/bind.h" #include "base/logging.h" +#include "base/time/clock.h" #include "base/time/time.h" #include "components/offline_pages/core/background/device_conditions.h" #include "components/offline_pages/core/background/offliner_policy.h" @@ -16,6 +20,8 @@ #include "components/offline_pages/core/background/request_notifier.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/save_page_request.h" +#include "components/offline_pages/core/client_policy_controller.h" +#include "components/offline_pages/core/offline_clock.h" namespace { template <typename T> @@ -31,9 +37,13 @@ bool kNonUserRequestsFound = true; namespace offline_pages { +const base::TimeDelta PickRequestTask::kDeferInterval = + base::TimeDelta::FromMinutes(1); + PickRequestTask::PickRequestTask( RequestQueueStore* store, OfflinerPolicy* policy, + ClientPolicyController* policy_controller, RequestPickedCallback picked_callback, RequestNotPickedCallback not_picked_callback, RequestCountCallback request_count_callback, @@ -42,6 +52,7 @@ PickRequestTask::PickRequestTask( base::circular_deque<int64_t>* prioritized_requests) : store_(store), policy_(policy), + policy_controller_(policy_controller), picked_callback_(std::move(picked_callback)), not_picked_callback_(std::move(not_picked_callback)), request_count_callback_(std::move(request_count_callback)), @@ -69,7 +80,7 @@ void PickRequestTask::Choose( if (requests.empty()) { std::move(request_count_callback_).Run(requests.size(), 0); std::move(not_picked_callback_) - .Run(!kNonUserRequestsFound, !kCleanupNeeded); + .Run(!kNonUserRequestsFound, !kCleanupNeeded, base::Time()); TaskComplete(); return; } @@ -95,7 +106,9 @@ void PickRequestTask::Choose( size_t total_request_count = requests.size(); // Request ids which are available for picking. std::unordered_set<int64_t> available_request_ids; - + // If there was a deferred task, this records the earliest time a task will + // become available. + base::Time defer_available_time; // Iterate through the requests, filter out unavailable requests and get other // information (if cleanup is needed and number of non-user-requested // requests). @@ -123,6 +136,15 @@ void PickRequestTask::Choose( available_requests->push_back(*request); if (!RequestConditionsSatisfied(*request)) continue; + if (policy_controller_->GetPolicy(request->client_id().name_space) + .defer_background_fetch_while_page_is_active) { + if (!request->last_attempt_time().is_null() && + OfflineClock()->Now() - request->last_attempt_time() < + kDeferInterval) { + defer_available_time = request->last_attempt_time() + kDeferInterval; + continue; + } + } available_request_ids.insert(request->request_id()); } // Report the request queue counts. @@ -169,7 +191,8 @@ void PickRequestTask::Choose( .Run(*picked_request, std::move(available_requests), cleanup_needed); } else { std::move(not_picked_callback_) - .Run(non_user_requested_tasks_remaining, cleanup_needed); + .Run(non_user_requested_tasks_remaining, cleanup_needed, + defer_available_time); } TaskComplete(); diff --git a/chromium/components/offline_pages/core/background/pick_request_task.h b/chromium/components/offline_pages/core/background/pick_request_task.h index 7a4aa84d313..9bc9c9dc4b5 100644 --- a/chromium/components/offline_pages/core/background/pick_request_task.h +++ b/chromium/components/offline_pages/core/background/pick_request_task.h @@ -5,7 +5,9 @@ #ifndef COMPONENTS_OFFLINE_PAGES_CORE_BACKGROUND_PICK_REQUEST_TASK_H_ #define COMPONENTS_OFFLINE_PAGES_CORE_BACKGROUND_PICK_REQUEST_TASK_H_ +#include <memory> #include <set> +#include <vector> #include "base/containers/circular_deque.h" #include "base/memory/weak_ptr.h" @@ -16,6 +18,7 @@ namespace offline_pages { +class ClientPolicyController; class OfflinerPolicy; class PickRequestTask; class RequestQueueStore; @@ -26,6 +29,8 @@ typedef bool (PickRequestTask::*RequestCompareFunction)( class PickRequestTask : public Task { public: + static const base::TimeDelta kDeferInterval; + // Callback to report when a request was available. typedef base::OnceCallback<void( const SavePageRequest& request, @@ -34,7 +39,9 @@ class PickRequestTask : public Task { RequestPickedCallback; // Callback to report when no request was available. - typedef base::OnceCallback<void(bool non_user_requests, bool cleanup_needed)> + typedef base::OnceCallback<void(bool non_user_requests, + bool cleanup_needed, + base::Time available_time)> RequestNotPickedCallback; // Callback to report available total and available queued request counts. @@ -42,6 +49,7 @@ class PickRequestTask : public Task { PickRequestTask(RequestQueueStore* store, OfflinerPolicy* policy, + ClientPolicyController* policy_controller, RequestPickedCallback picked_callback, RequestNotPickedCallback not_picked_callback, RequestCountCallback request_count_callback, @@ -93,6 +101,7 @@ class PickRequestTask : public Task { // Member variables, all pointers are not owned here. RequestQueueStore* store_; OfflinerPolicy* policy_; + ClientPolicyController* policy_controller_; RequestPickedCallback picked_callback_; RequestNotPickedCallback not_picked_callback_; RequestCountCallback request_count_callback_; diff --git a/chromium/components/offline_pages/core/background/pick_request_task_unittest.cc b/chromium/components/offline_pages/core/background/pick_request_task_unittest.cc index cb45cee9de8..945498b74a8 100644 --- a/chromium/components/offline_pages/core/background/pick_request_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/pick_request_task_unittest.cc @@ -6,11 +6,11 @@ #include <memory> #include <set> +#include <vector> #include "base/bind.h" #include "base/containers/circular_deque.h" -#include "base/test/test_mock_time_task_runner.h" -#include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "base/time/time.h" #include "components/offline_pages/core/background/device_conditions.h" #include "components/offline_pages/core/background/offliner_policy.h" @@ -21,6 +21,8 @@ #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/save_page_request.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/client_policy_controller.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -103,7 +105,8 @@ class PickRequestTaskTest : public RequestQueueTaskTestBase { bool cleanup_needed); void RequestNotPicked(const bool non_user_requested_tasks_remaining, - bool cleanup_needed); + bool cleanup_needed, + base::Time available_time); void RequestCountCallback(size_t total_count, size_t available_count); @@ -120,10 +123,10 @@ class PickRequestTaskTest : public RequestQueueTaskTestBase { void TaskCompletionCallback(Task* completed_task); protected: - std::unique_ptr<RequestNotifierStub> notifier_; std::unique_ptr<SavePageRequest> last_picked_; std::unique_ptr<OfflinerPolicy> policy_; + ClientPolicyController policy_controller_; RequestCoordinatorEventLogger event_logger_; std::set<int64_t> disabled_requests_; base::circular_deque<int64_t> prioritized_requests_; @@ -167,7 +170,8 @@ void PickRequestTaskTest::RequestPicked( void PickRequestTaskTest::RequestNotPicked( const bool non_user_requested_tasks_remaining, - const bool cleanup_needed) { + const bool cleanup_needed, + base::Time available_time) { request_queue_not_picked_called_ = true; } @@ -197,7 +201,7 @@ void PickRequestTaskTest::QueueRequests(const SavePageRequest& request1, void PickRequestTaskTest::MakePickRequestTask() { DeviceConditions conditions; task_.reset(new PickRequestTask( - &store_, policy_.get(), + &store_, policy_.get(), &policy_controller_, base::BindOnce(&PickRequestTaskTest::RequestPicked, base::Unretained(this)), base::BindOnce(&PickRequestTaskTest::RequestNotPicked, @@ -206,7 +210,6 @@ void PickRequestTaskTest::MakePickRequestTask() { base::Unretained(this)), conditions, disabled_requests_, &prioritized_requests_)); task_->SetTaskCompletionCallbackForTesting( - task_runner_.get(), base::BindRepeating(&PickRequestTaskTest::TaskCompletionCallback, base::Unretained(this))); } @@ -233,7 +236,7 @@ TEST_F(PickRequestTaskTest, ChooseRequestWithHigherRetryCount) { kMaxCompletedTries + 1, kBackgroundProcessingTimeBudgetSeconds)); MakePickRequestTask(); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time, @@ -254,8 +257,8 @@ TEST_F(PickRequestTaskTest, ChooseRequestWithHigherRetryCount) { TEST_F(PickRequestTaskTest, ChooseRequestWithSameRetryCountButEarlier) { base::Time creation_time1 = - base::Time::Now() - base::TimeDelta::FromSeconds(10); - base::Time creation_time2 = base::Time::Now(); + OfflineClock()->Now() - base::TimeDelta::FromSeconds(10); + base::Time creation_time2 = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time1, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time2, @@ -279,8 +282,8 @@ TEST_F(PickRequestTaskTest, ChooseEarlierRequest) { MakePickRequestTask(); base::Time creation_time1 = - base::Time::Now() - base::TimeDelta::FromSeconds(10); - base::Time creation_time2 = base::Time::Now(); + OfflineClock()->Now() - base::TimeDelta::FromSeconds(10); + base::Time creation_time2 = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time1, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time2, @@ -304,7 +307,7 @@ TEST_F(PickRequestTaskTest, ChooseSameTimeRequestWithHigherRetryCount) { kMaxCompletedTries + 1, kBackgroundProcessingTimeBudgetSeconds)); MakePickRequestTask(); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time, @@ -328,7 +331,7 @@ TEST_F(PickRequestTaskTest, ChooseRequestWithLowerRetryCount) { kMaxCompletedTries + 1, kBackgroundProcessingTimeBudgetSeconds)); MakePickRequestTask(); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time, @@ -353,8 +356,8 @@ TEST_F(PickRequestTaskTest, ChooseLaterRequest) { MakePickRequestTask(); base::Time creation_time1 = - base::Time::Now() - base::TimeDelta::FromSeconds(10); - base::Time creation_time2 = base::Time::Now(); + OfflineClock()->Now() - base::TimeDelta::FromSeconds(10); + base::Time creation_time2 = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time1, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time2, @@ -371,7 +374,7 @@ TEST_F(PickRequestTaskTest, ChooseLaterRequest) { } TEST_F(PickRequestTaskTest, ChooseNonExpiredRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); base::Time expired_time = creation_time - base::TimeDelta::FromSeconds( policy_->GetRequestExpirationTimeInSeconds() + 60); @@ -395,8 +398,8 @@ TEST_F(PickRequestTaskTest, ChooseNonExpiredRequest) { TEST_F(PickRequestTaskTest, ChooseRequestThatHasNotExceededStartLimit) { base::Time creation_time1 = - base::Time::Now() - base::TimeDelta::FromSeconds(1); - base::Time creation_time2 = base::Time::Now(); + OfflineClock()->Now() - base::TimeDelta::FromSeconds(1); + base::Time creation_time2 = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time1, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time2, @@ -421,8 +424,8 @@ TEST_F(PickRequestTaskTest, ChooseRequestThatHasNotExceededStartLimit) { TEST_F(PickRequestTaskTest, ChooseRequestThatHasNotExceededCompletionLimit) { base::Time creation_time1 = - base::Time::Now() - base::TimeDelta::FromSeconds(1); - base::Time creation_time2 = base::Time::Now(); + OfflineClock()->Now() - base::TimeDelta::FromSeconds(1); + base::Time creation_time2 = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time1, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time2, @@ -453,7 +456,7 @@ TEST_F(PickRequestTaskTest, ChooseRequestThatIsNotDisabled) { disabled_requests_.insert(kRequestId2); MakePickRequestTask(); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time, @@ -482,7 +485,7 @@ TEST_F(PickRequestTaskTest, ChoosePrioritizedRequests) { prioritized_requests_.push_back(kRequestId2); MakePickRequestTask(); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); SavePageRequest request2(kRequestId2, kUrl2, kClientId2, creation_time, @@ -521,7 +524,7 @@ TEST_F(PickRequestTaskTest, ChooseFromTwoPrioritizedRequests) { // Making request 1 more attractive to be picked not considering the // prioritizing issues with older creation time, fewer attempt count and it's // earlier in the request queue. - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); base::Time older_creation_time = creation_time - base::TimeDelta::FromMinutes(10); SavePageRequest request1(kRequestId1, kUrl1, kClientId1, older_creation_time, diff --git a/chromium/components/offline_pages/core/background/reconcile_task_unittest.cc b/chromium/components/offline_pages/core/background/reconcile_task_unittest.cc index e92b7c3d858..240eebeaf3b 100644 --- a/chromium/components/offline_pages/core/background/reconcile_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/reconcile_task_unittest.cc @@ -10,11 +10,13 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_coordinator.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/save_page_request.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -116,7 +118,7 @@ void ReconcileTaskTest::MakeTask() { } TEST_F(ReconcileTaskTest, Reconcile) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // Request2 will be expired, request1 will be current. SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); @@ -151,7 +153,7 @@ TEST_F(ReconcileTaskTest, Reconcile) { } TEST_F(ReconcileTaskTest, NothingToReconcile) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // Request2 will be expired, request1 will be current. SavePageRequest request1(kRequestId1, kUrl1, kClientId1, creation_time, kUserRequested); diff --git a/chromium/components/offline_pages/core/background/remove_requests_task.cc b/chromium/components/offline_pages/core/background/remove_requests_task.cc index b261bba7c65..0135c844d86 100644 --- a/chromium/components/offline_pages/core/background/remove_requests_task.cc +++ b/chromium/components/offline_pages/core/background/remove_requests_task.cc @@ -37,7 +37,7 @@ void RemoveRequestsTask::RemoveRequests() { void RemoveRequestsTask::CompleteEarly(ItemActionStatus status) { UpdateRequestsResult result(store_->state()); for (int64_t request_id : request_ids_) - result.item_statuses.push_back(std::make_pair(request_id, status)); + result.item_statuses.emplace_back(request_id, status); CompleteWithResult(std::move(result)); } diff --git a/chromium/components/offline_pages/core/background/remove_requests_task_unittest.cc b/chromium/components/offline_pages/core/background/remove_requests_task_unittest.cc index b93eae05537..be0af62ecc8 100644 --- a/chromium/components/offline_pages/core/background/remove_requests_task_unittest.cc +++ b/chromium/components/offline_pages/core/background/remove_requests_task_unittest.cc @@ -9,9 +9,11 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/request_queue_task_test_base.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -47,7 +49,7 @@ void RemoveRequestsTaskTest::PumpLoop() { } void RemoveRequestsTaskTest::AddRequestsToStore() { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request_1(kRequestId1, kUrl1, kClientId1, creation_time, true); store_.AddRequest(request_1, diff --git a/chromium/components/offline_pages/core/background/request_coordinator.cc b/chromium/components/offline_pages/core/background/request_coordinator.cc index 7470295a1ce..086ae2d280a 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator.cc +++ b/chromium/components/offline_pages/core/background/request_coordinator.cc @@ -10,15 +10,18 @@ #include "base/bind.h" #include "base/callback.h" #include "base/logging.h" +#include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" #include "base/rand_util.h" #include "base/stl_util.h" -#include "base/sys_info.h" +#include "base/system/sys_info.h" +#include "base/time/clock.h" #include "base/time/time.h" #include "components/offline_pages/core/background/offliner.h" #include "components/offline_pages/core/background/offliner_policy.h" #include "components/offline_pages/core/background/save_page_request.h" #include "components/offline_pages/core/client_policy_controller.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_page_feature.h" #include "components/offline_pages/core/offline_page_item.h" #include "components/offline_pages/core/offline_page_model.h" @@ -31,8 +34,8 @@ namespace offline_pages { namespace { const bool kUserRequest = true; const bool kStartOfProcessing = true; -const int kMinDurationSeconds = 1; -const int kMaxDurationSeconds = 7 * 24 * 60 * 60; // 7 days +constexpr base::TimeDelta kMinDuration = base::TimeDelta::FromSeconds(1); +constexpr base::TimeDelta kMaxDuration = base::TimeDelta::FromDays(7); const int kDurationBuckets = 50; const int kDisabledTaskRecheckSeconds = 5; @@ -44,7 +47,8 @@ std::string AddHistogramSuffix(const ClientId& client_id, return histogram_name; } std::string adjusted_histogram_name(histogram_name); - adjusted_histogram_name += "." + client_id.name_space; + adjusted_histogram_name += "."; + adjusted_histogram_name += client_id.name_space; return adjusted_histogram_name; } @@ -53,28 +57,19 @@ std::string AddHistogramSuffix(const ClientId& client_id, void RecordOfflinerResultUMA(const ClientId& client_id, const base::Time& request_creation_time, Offliner::RequestStatus request_status) { - // The histogram below is an expansion of the UMA_HISTOGRAM_ENUMERATION - // macro adapted to allow for a dynamically suffixed histogram name. - // Note: The factory creates and owns the histogram. - base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( + base::UmaHistogramEnumeration( AddHistogramSuffix(client_id, "OfflinePages.Background.OfflinerRequestStatus"), - 1, static_cast<int>(Offliner::RequestStatus::STATUS_COUNT), - static_cast<int>(Offliner::RequestStatus::STATUS_COUNT) + 1, - base::HistogramBase::kUmaTargetedHistogramFlag); - histogram->Add(static_cast<int>(request_status)); + request_status); // For successful requests also record time from request to save. if (request_status == Offliner::RequestStatus::SAVED || request_status == Offliner::RequestStatus::SAVED_ON_LAST_RETRY) { - // Using regular histogram (with dynamic suffix) rather than time-oriented - // one to record samples in seconds rather than milliseconds. - base::HistogramBase* histogram = base::Histogram::FactoryGet( + base::TimeDelta duration = OfflineClock()->Now() - request_creation_time; + base::UmaHistogramCustomCounts( AddHistogramSuffix(client_id, "OfflinePages.Background.TimeToSaved"), - kMinDurationSeconds, kMaxDurationSeconds, kDurationBuckets, - base::HistogramBase::kUmaTargetedHistogramFlag); - base::TimeDelta duration = base::Time::Now() - request_creation_time; - histogram->Add(duration.InSeconds()); + duration.InSeconds(), kMinDuration.InSeconds(), + kMaxDuration.InSeconds(), kDurationBuckets); } } @@ -84,19 +79,10 @@ void RecordOfflinerResultUMA(const ClientId& client_id, void RecordSavePageResultUMA( const ClientId& client_id, RequestNotifier::BackgroundSavePageResult request_status) { - // The histogram below is an expansion of the UMA_HISTOGRAM_ENUMERATION - // macro adapted to allow for a dynamically suffixed histogram name. - // Note: The factory creates and owns the histogram. - base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( + base::UmaHistogramEnumeration( AddHistogramSuffix(client_id, "OfflinePages.Background.FinalSavePageResult"), - 1, - static_cast<int>(RequestNotifier::BackgroundSavePageResult::STATUS_COUNT), - static_cast<int>( - RequestNotifier::BackgroundSavePageResult::STATUS_COUNT) + - 1, - base::HistogramBase::kUmaTargetedHistogramFlag); - histogram->Add(static_cast<int>(request_status)); + request_status); } // Records whether the request comes from CCT or not @@ -114,28 +100,22 @@ void RecordStartTimeUMA(const SavePageRequest& request) { histogram_name += ".Svelte"; } - // The histogram below is an expansion of the UMA_HISTOGRAM_CUSTOM_TIMES - // macro adapted to allow for a dynamically suffixed histogram name. - // Note: The factory creates and owns the histogram. - base::HistogramBase* histogram = base::Histogram::FactoryTimeGet( - AddHistogramSuffix(request.client_id(), histogram_name.c_str()), - base::TimeDelta::FromMilliseconds(100), base::TimeDelta::FromDays(7), 50, - base::HistogramBase::kUmaTargetedHistogramFlag); - base::TimeDelta duration = base::Time::Now() - request.creation_time(); - histogram->AddTime(duration); + base::TimeDelta duration = OfflineClock()->Now() - request.creation_time(); + base::UmaHistogramCustomTimes( + AddHistogramSuffix(request.client_id(), histogram_name.c_str()), duration, + base::TimeDelta::FromMilliseconds(100), base::TimeDelta::FromDays(7), 50); } void RecordCancelTimeUMA(const SavePageRequest& canceled_request) { // Using regular histogram (with dynamic suffix) rather than time-oriented // one to record samples in seconds rather than milliseconds. - base::HistogramBase* histogram = base::Histogram::FactoryGet( + base::TimeDelta duration = + OfflineClock()->Now() - canceled_request.creation_time(); + base::UmaHistogramCustomCounts( AddHistogramSuffix(canceled_request.client_id(), "OfflinePages.Background.TimeToCanceled"), - kMinDurationSeconds, kMaxDurationSeconds, kDurationBuckets, - base::HistogramBase::kUmaTargetedHistogramFlag); - base::TimeDelta duration = - base::Time::Now() - canceled_request.creation_time(); - histogram->Add(duration.InSeconds()); + duration.InSeconds(), kMinDuration.InSeconds(), kMaxDuration.InSeconds(), + kDurationBuckets); } // Records the number of started attempts for completed requests (whether @@ -214,14 +194,7 @@ RequestCoordinator::SavePageLaterParams::SavePageLaterParams() availability(RequestAvailability::ENABLED_FOR_OFFLINER) {} RequestCoordinator::SavePageLaterParams::SavePageLaterParams( - const SavePageLaterParams& other) { - url = other.url; - client_id = other.client_id; - user_requested = other.user_requested; - availability = other.availability; - original_url = other.original_url; - request_origin = other.request_origin; -} + const SavePageLaterParams& other) = default; RequestCoordinator::SavePageLaterParams::~SavePageLaterParams() = default; @@ -231,7 +204,8 @@ RequestCoordinator::RequestCoordinator( std::unique_ptr<RequestQueue> queue, std::unique_ptr<Scheduler> scheduler, network::NetworkQualityTracker* network_quality_tracker, - std::unique_ptr<OfflinePagesUkmReporter> ukm_reporter) + std::unique_ptr<OfflinePagesUkmReporter> ukm_reporter, + std::unique_ptr<ActiveTabInfo> active_tab_info) : is_low_end_device_(base::SysInfo::IsLowEndDevice()), state_(RequestCoordinatorState::IDLE), processing_state_(ProcessingWindowState::STOPPED), @@ -249,6 +223,7 @@ RequestCoordinator::RequestCoordinator( scheduler_callback_(base::DoNothing()), internal_start_processing_callback_(base::DoNothing()), pending_state_updater_(this), + active_tab_info_(std::move(active_tab_info)), weak_ptr_factory_(this) { DCHECK(policy_ != nullptr); DCHECK(network_quality_tracker_); @@ -286,7 +261,7 @@ int64_t RequestCoordinator::SavePageLater( // Build a SavePageRequest. offline_pages::SavePageRequest request( id, save_page_later_params.url, save_page_later_params.client_id, - base::Time::Now(), save_page_later_params.user_requested); + OfflineClock()->Now(), save_page_later_params.user_requested); request.set_original_url(save_page_later_params.original_url); request.set_request_origin(save_page_later_params.request_origin); pending_state_updater_.SetPendingState(request); @@ -621,6 +596,14 @@ void RequestCoordinator::TryNextRequestCallback(int64_t offline_id) { TryNextRequest(!kStartOfProcessing); } +void RequestCoordinator::MarkDeferredAttemptCallback( + UpdateRequestsResult result) { + // This is called after the attempt has been marked as deferred in the + // database. StopProcessing() is called to resume request processing. + state_ = RequestCoordinatorState::IDLE; + StopProcessing(Offliner::RequestStatus::LOADING_DEFERRED); +} + void RequestCoordinator::ScheduleAsNeeded() { // Get all requests from queue (there is no filtering mechanism). queue_->GetRequests( @@ -637,7 +620,7 @@ void RequestCoordinator::StopProcessing(Offliner::RequestStatus stop_status) { void RequestCoordinator::HandleWatchdogTimeout() { Offliner::RequestStatus watchdog_status = - Offliner::REQUEST_COORDINATOR_TIMED_OUT; + Offliner::RequestStatus::REQUEST_COORDINATOR_TIMED_OUT; if (offliner_->HandleTimeout(active_request_id_)) return; StopOfflining(base::BindOnce(&RequestCoordinator::TryNextRequestCallback, @@ -680,7 +663,7 @@ bool RequestCoordinator::StartProcessingInternal( // Mark the time at which we started processing so we can check our time // budget. - operation_start_time_ = base::Time::Now(); + operation_start_time_ = OfflineClock()->Now(); TryNextRequest(kStartOfProcessing); @@ -794,7 +777,8 @@ void RequestCoordinator::TryNextRequest(bool is_start_of_processing) { // will return to us at the next opportunity to run background tasks. if (connection_type == net::NetworkChangeNotifier::ConnectionType::CONNECTION_NONE || - (base::Time::Now() - operation_start_time_) > processing_time_budget) { + (OfflineClock()->Now() - operation_start_time_) > + processing_time_budget) { state_ = RequestCoordinatorState::IDLE; // If we were doing immediate processing, try to start it again @@ -812,7 +796,7 @@ void RequestCoordinator::TryNextRequest(bool is_start_of_processing) { // Ask request queue to make a new PickRequestTask object, then put it on // the task queue. queue_->PickNextRequest( - policy_.get(), + policy_.get(), policy_controller_.get(), base::BindOnce(&RequestCoordinator::RequestPicked, weak_ptr_factory_.GetWeakPtr()), base::BindOnce(&RequestCoordinator::RequestNotPicked, @@ -848,7 +832,8 @@ void RequestCoordinator::RequestPicked( void RequestCoordinator::RequestNotPicked( bool non_user_requested_tasks_remaining, - bool cleanup_needed) { + bool cleanup_needed, + base::Time available_time) { DVLOG(2) << __func__; state_ = RequestCoordinatorState::IDLE; @@ -862,6 +847,11 @@ void RequestCoordinator::RequestNotPicked( } else if (non_user_requested_tasks_remaining) { // If we don't have any of those, check for non-user-requested tasks. scheduler_->Schedule(GetTriggerConditions(!kUserRequest)); + } else if (!available_time.is_null()) { + scheduler_->BackupSchedule( + GetTriggerConditions(kUserRequest), + (available_time - OfflineClock()->Now()).InSeconds() + + 1 /*Add an extra second to avoid rounding down.*/); } // Schedule a queue cleanup if needed. @@ -925,6 +915,16 @@ void RequestCoordinator::SendRequestToOffliner(const SavePageRequest& request) { if (request.started_attempt_count() == 0) { RecordStartTimeUMA(request); } + const OfflinePageClientPolicy& policy = + policy_controller_->GetPolicy(request.client_id().name_space); + if (policy.defer_background_fetch_while_page_is_active && + active_tab_info_->DoesActiveTabMatch(request.url())) { + queue_->MarkAttemptDeferred( + request.request_id(), + base::BindOnce(&RequestCoordinator::MarkDeferredAttemptCallback, + weak_ptr_factory_.GetWeakPtr())); + return; + } // Mark attempt started in the database and start offliner when completed. queue_->MarkAttemptStarted( @@ -942,7 +942,7 @@ void RequestCoordinator::StartOffliner(int64_t request_id, update_result.item_statuses.at(0).first != request_id || update_result.item_statuses.at(0).second != ItemActionStatus::SUCCESS) { state_ = RequestCoordinatorState::IDLE; - StopProcessing(Offliner::QUEUE_UPDATE_FAILED); + StopProcessing(Offliner::RequestStatus::QUEUE_UPDATE_FAILED); DVLOG(1) << "Failed to mark attempt started: " << request_id; UpdateRequestResult request_result = update_result.store_state != StoreState::LOADED @@ -982,7 +982,7 @@ void RequestCoordinator::StartOffliner(int64_t request_id, } else { state_ = RequestCoordinatorState::IDLE; DVLOG(0) << "Unable to start LoadAndSave"; - StopProcessing(Offliner::LOADING_NOT_ACCEPTED); + StopProcessing(Offliner::RequestStatus::LOADING_NOT_ACCEPTED); // We need to undo the MarkAttemptStarted that brought us to this // method since we didn't success in starting after all. @@ -1014,7 +1014,7 @@ void RequestCoordinator::OfflinerProgressCallback( const SavePageRequest& request, int64_t received_bytes) { DVLOG(2) << "offliner progress, received bytes: " << received_bytes; - DCHECK(received_bytes >= 0); + DCHECK_GE(received_bytes, 0); NotifyNetworkProgress(request, received_bytes); } @@ -1121,7 +1121,6 @@ void RequestCoordinator::MarkRequestCompleted(int64_t request_id) { // calls for a particular request_id. if (disabled_requests_.find(request_id) == disabled_requests_.end()) return; - // Clear from disabled list. disabled_requests_.erase(request_id); diff --git a/chromium/components/offline_pages/core/background/request_coordinator.h b/chromium/components/offline_pages/core/background/request_coordinator.h index 20c78b9bb6b..4ca42f1b946 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator.h +++ b/chromium/components/offline_pages/core/background/request_coordinator.h @@ -64,6 +64,14 @@ class RequestCoordinator : public KeyedService, int64_t received_bytes) = 0; }; + class ActiveTabInfo { + public: + virtual ~ActiveTabInfo() {} + // Returns true if the active tab's URL matches |url|. If Chrome is in the + // background, this should return false. + virtual bool DoesActiveTabMatch(const GURL& url) = 0; + }; + enum class RequestAvailability { ENABLED_FOR_OFFLINER, DISABLED_FOR_OFFLINER, @@ -121,7 +129,8 @@ class RequestCoordinator : public KeyedService, std::unique_ptr<RequestQueue> queue, std::unique_ptr<Scheduler> scheduler, network::NetworkQualityTracker* network_quality_tracker, - std::unique_ptr<OfflinePagesUkmReporter> ukm_reporter); + std::unique_ptr<OfflinePagesUkmReporter> ukm_reporter, + std::unique_ptr<ActiveTabInfo> active_tab_info); ~RequestCoordinator() override; @@ -313,6 +322,7 @@ class RequestCoordinator : public KeyedService, void ResetActiveRequestCallback(int64_t offline_id); void StartSchedulerCallback(int64_t offline_id); void TryNextRequestCallback(int64_t offline_id); + void MarkDeferredAttemptCallback(UpdateRequestsResult result); bool StartProcessingInternal( const ProcessingWindowState processing_state, @@ -348,7 +358,8 @@ class RequestCoordinator : public KeyedService, // The parameter is a signal for what (if any) conditions to schedule future // processing for. void RequestNotPicked(bool non_user_requested_tasks_remaining, - bool cleanup_needed); + bool cleanup_needed, + base::Time available_time); // Callback from request picker that receives the current available queued // request count as well as the total queued request count (which may be @@ -503,6 +514,8 @@ class RequestCoordinator : public KeyedService, base::circular_deque<int64_t> prioritized_requests_; // Updates a request's PendingState. PendingStateUpdater pending_state_updater_; + + std::unique_ptr<ActiveTabInfo> active_tab_info_; // Allows us to pass a weak pointer to callbacks. base::WeakPtrFactory<RequestCoordinator> weak_ptr_factory_; diff --git a/chromium/components/offline_pages/core/background/request_coordinator_event_logger.cc b/chromium/components/offline_pages/core/background/request_coordinator_event_logger.cc index 5f22fd8981a..88554d9981c 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator_event_logger.cc +++ b/chromium/components/offline_pages/core/background/request_coordinator_event_logger.cc @@ -11,52 +11,53 @@ namespace { static std::string OfflinerRequestStatusToString( Offliner::RequestStatus request_status) { switch (request_status) { - case Offliner::UNKNOWN: + case Offliner::RequestStatus::UNKNOWN: return "UNKNOWN"; - case Offliner::LOADED: + case Offliner::RequestStatus::LOADED: return "LOADED"; - case Offliner::SAVED: + case Offliner::RequestStatus::SAVED: return "SAVED"; - case Offliner::REQUEST_COORDINATOR_CANCELED: + case Offliner::RequestStatus::REQUEST_COORDINATOR_CANCELED: return "REQUEST_COORDINATOR_CANCELED"; - case Offliner::LOADING_CANCELED: + case Offliner::RequestStatus::LOADING_CANCELED: return "LOADING_CANCELED"; - case Offliner::LOADING_FAILED: + case Offliner::RequestStatus::LOADING_FAILED: return "LOADING_FAILED"; - case Offliner::SAVE_FAILED: + case Offliner::RequestStatus::SAVE_FAILED: return "SAVE_FAILED"; - case Offliner::FOREGROUND_CANCELED: + case Offliner::RequestStatus::FOREGROUND_CANCELED: return "FOREGROUND_CANCELED"; - case Offliner::REQUEST_COORDINATOR_TIMED_OUT: + case Offliner::RequestStatus::REQUEST_COORDINATOR_TIMED_OUT: return "REQUEST_COORDINATOR_TIMED_OUT"; - case Offliner::DEPRECATED_LOADING_NOT_STARTED: + case Offliner::RequestStatus::DEPRECATED_LOADING_NOT_STARTED: + NOTREACHED(); return "DEPRECATED_LOADING_NOT_STARTED"; - case Offliner::LOADING_FAILED_NO_RETRY: + case Offliner::RequestStatus::LOADING_FAILED_NO_RETRY: return "LOADING_FAILED_NO_RETRY"; - case Offliner::LOADING_FAILED_NO_NEXT: + case Offliner::RequestStatus::LOADING_FAILED_NO_NEXT: return "LOADING_FAILED_NO_NEXT"; - case Offliner::LOADING_NOT_ACCEPTED: + case Offliner::RequestStatus::LOADING_NOT_ACCEPTED: return "LOADING_NOT_ACCEPTED"; - case Offliner::QUEUE_UPDATE_FAILED: + case Offliner::RequestStatus::QUEUE_UPDATE_FAILED: return "QUEUE_UPDATE_FAILED"; - case Offliner::BACKGROUND_SCHEDULER_CANCELED: + case Offliner::RequestStatus::BACKGROUND_SCHEDULER_CANCELED: return "BACKGROUND_SCHEDULER_CANCELED"; - case Offliner::SAVED_ON_LAST_RETRY: + case Offliner::RequestStatus::SAVED_ON_LAST_RETRY: return "SAVED_ON_LAST_RETRY"; - case Offliner::BROWSER_KILLED: + case Offliner::RequestStatus::BROWSER_KILLED: return "BROWSER_KILLED"; - case Offliner::LOADING_FAILED_DOWNLOAD: + case Offliner::RequestStatus::LOADING_FAILED_DOWNLOAD: return "LOADING_FAILED_DOWNLOAD"; - case Offliner::DOWNLOAD_THROTTLED: + case Offliner::RequestStatus::DOWNLOAD_THROTTLED: return "DOWNLOAD_THROTTLED"; - case Offliner::LOADING_FAILED_NET_ERROR: + case Offliner::RequestStatus::LOADING_FAILED_NET_ERROR: return "LOADING_FAILED_NET_ERROR"; - case Offliner::LOADING_FAILED_HTTP_ERROR: + case Offliner::RequestStatus::LOADING_FAILED_HTTP_ERROR: return "LOADING_FAILED_HTTP_ERROR"; - default: - NOTREACHED(); - return std::to_string(static_cast<int>(request_status)); + case Offliner::RequestStatus::LOADING_DEFERRED: + return "LOADING_DEFERRED"; } + return "UNKNOWN"; } static std::string BackgroundSavePageResultToString( diff --git a/chromium/components/offline_pages/core/background/request_coordinator_event_logger_unittest.cc b/chromium/components/offline_pages/core/background/request_coordinator_event_logger_unittest.cc index 8aa7f6ca351..be3df7ab36b 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator_event_logger_unittest.cc +++ b/chromium/components/offline_pages/core/background/request_coordinator_event_logger_unittest.cc @@ -11,7 +11,7 @@ namespace offline_pages { namespace { const char kNamespace[] = "last_n"; -const Offliner::RequestStatus kOfflinerStatus = Offliner::SAVED; +const Offliner::RequestStatus kOfflinerStatus = Offliner::RequestStatus::SAVED; const RequestNotifier::BackgroundSavePageResult kDroppedResult = RequestNotifier::BackgroundSavePageResult::START_COUNT_EXCEEDED; const int64_t kId = 1234; diff --git a/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.cc b/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.cc index ddda5ed4352..82d545fe948 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.cc +++ b/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.cc @@ -4,6 +4,8 @@ #include "components/offline_pages/core/background/request_coordinator_stub_taco.h" +#include <utility> + #include "components/offline_pages/core/background/offliner_stub.h" #include "components/offline_pages/core/background/request_queue.h" #include "components/offline_pages/core/background/request_queue_store.h" @@ -16,6 +18,12 @@ namespace offline_pages { +class ActiveTabInfo : public RequestCoordinator::ActiveTabInfo { + public: + ~ActiveTabInfo() override {} + bool DoesActiveTabMatch(const GURL&) override { return false; } +}; + RequestCoordinatorStubTaco::RequestCoordinatorStubTaco() { policy_ = std::make_unique<OfflinerPolicy>(); queue_ = @@ -25,6 +33,7 @@ RequestCoordinatorStubTaco::RequestCoordinatorStubTaco() { network_quality_tracker_ = std::make_unique<network::TestNetworkQualityTracker>(); ukm_reporter_ = std::make_unique<OfflinePagesUkmReporterStub>(); + active_tab_info_ = std::make_unique<ActiveTabInfo>(); } RequestCoordinatorStubTaco::~RequestCoordinatorStubTaco() { @@ -73,15 +82,46 @@ void RequestCoordinatorStubTaco::SetOfflinePagesUkmReporter( ukm_reporter_ = std::move(ukm_reporter); } +void RequestCoordinatorStubTaco::SetRequestCoordinatorDelegate( + std::unique_ptr<RequestCoordinator::ActiveTabInfo> active_tab_info) { + active_tab_info_ = std::move(active_tab_info); +} + void RequestCoordinatorStubTaco::CreateRequestCoordinator() { - request_coordinator_ = std::make_unique<RequestCoordinator>( + CHECK(!request_coordinator_) + << "CreateRequestCoordinator can be called only once"; + owned_request_coordinator_ = std::make_unique<RequestCoordinator>( std::move(policy_), std::move(offliner_), std::move(queue_), std::move(scheduler_), network_quality_tracker_.get(), - std::move(ukm_reporter_)); + std::move(ukm_reporter_), std::move(active_tab_info_)); + request_coordinator_ = owned_request_coordinator_.get(); } RequestCoordinator* RequestCoordinatorStubTaco::request_coordinator() { CHECK(request_coordinator_); - return request_coordinator_.get(); + return request_coordinator_; } -} // namespace offline_page + +base::RepeatingCallback<std::unique_ptr<KeyedService>(content::BrowserContext*)> +RequestCoordinatorStubTaco::FactoryFunction() { + return base::BindRepeating( + &RequestCoordinatorStubTaco::InternalFactoryFunction, GetWeakPtr()); +} + +// static +std::unique_ptr<KeyedService> +RequestCoordinatorStubTaco::InternalFactoryFunction( + base::WeakPtr<RequestCoordinatorStubTaco> taco, + content::BrowserContext* context) { + if (!taco) + return nullptr; + // Call CreateRequestCoordinator if it hasn't already been called. + if (!taco->request_coordinator_) { + taco->CreateRequestCoordinator(); + } + // This function can only be used once. + CHECK(taco->owned_request_coordinator_); + return std::move(taco->owned_request_coordinator_); +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.h b/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.h index d99d16a010f..d1c07565f69 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.h +++ b/chromium/components/offline_pages/core/background/request_coordinator_stub_taco.h @@ -14,6 +14,9 @@ #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/scheduler.h" +namespace content { +class BrowserContext; +} namespace network { class NetworkQualityTracker; } @@ -44,6 +47,8 @@ class RequestCoordinatorStubTaco { std::unique_ptr<network::NetworkQualityTracker> network_quality_tracker); void SetOfflinePagesUkmReporter( std::unique_ptr<OfflinePagesUkmReporter> ukm_reporter); + void SetRequestCoordinatorDelegate( + std::unique_ptr<RequestCoordinator::ActiveTabInfo> delegate); // Creates and caches an instance of RequestCoordinator, using default or // overridden stub dependencies. @@ -55,7 +60,21 @@ class RequestCoordinatorStubTaco { RequestCoordinator* request_coordinator(); + // A factory function that can be used with + // RequestCoordinatorFactory::SetTestingFactoryAndUse. + base::RepeatingCallback< + std::unique_ptr<KeyedService>(content::BrowserContext*)> + FactoryFunction(); + private: + base::WeakPtr<RequestCoordinatorStubTaco> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + + static std::unique_ptr<KeyedService> InternalFactoryFunction( + base::WeakPtr<RequestCoordinatorStubTaco> taco, + content::BrowserContext*); + bool store_overridden_ = false; bool queue_overridden_ = false; @@ -65,8 +84,14 @@ class RequestCoordinatorStubTaco { std::unique_ptr<Scheduler> scheduler_; std::unique_ptr<network::NetworkQualityTracker> network_quality_tracker_; std::unique_ptr<OfflinePagesUkmReporter> ukm_reporter_; + std::unique_ptr<RequestCoordinator::ActiveTabInfo> active_tab_info_; + + // This is null if the request coordinator was given to the + // RequestCoordinatorFactory through the factory function. + std::unique_ptr<RequestCoordinator> owned_request_coordinator_; + RequestCoordinator* request_coordinator_ = nullptr; - std::unique_ptr<RequestCoordinator> request_coordinator_; + base::WeakPtrFactory<RequestCoordinatorStubTaco> weak_ptr_factory_{this}; }; } // namespace offline_pages #endif // COMPONENTS_OFFLINE_PAGES_CORE_BACKGROUND_REQUEST_COORDINATOR_STUB_TACO_H_ diff --git a/chromium/components/offline_pages/core/background/request_coordinator_unittest.cc b/chromium/components/offline_pages/core/background/request_coordinator_unittest.cc index 8c03fce86c4..28ea3e2280d 100644 --- a/chromium/components/offline_pages/core/background/request_coordinator_unittest.cc +++ b/chromium/components/offline_pages/core/background/request_coordinator_unittest.cc @@ -14,11 +14,12 @@ #include "base/location.h" #include "base/logging.h" #include "base/synchronization/waitable_event.h" -#include "base/sys_info.h" +#include "base/system/sys_info.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/scoped_feature_list.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "base/time/time.h" #include "components/offline_items_collection/core/pending_state.h" #include "components/offline_pages/core/background/device_conditions.h" @@ -31,6 +32,8 @@ #include "components/offline_pages/core/background/save_page_request.h" #include "components/offline_pages/core/background/scheduler.h" #include "components/offline_pages/core/background/scheduler_stub.h" +#include "components/offline_pages/core/client_namespace_constants.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_page_feature.h" #include "services/network/test/test_network_quality_tracker.h" #include "testing/gtest/include/gtest/gtest.h" @@ -124,6 +127,18 @@ class ObserverStub : public RequestCoordinator::Observer { PendingState pending_state_; }; +class ActiveTabInfoStub : public RequestCoordinator::ActiveTabInfo { + public: + ~ActiveTabInfoStub() override {} + bool DoesActiveTabMatch(const GURL&) override { + return does_active_tab_match_; + } + void set_does_active_tab_match(bool match) { does_active_tab_match_ = match; } + + private: + bool does_active_tab_match_ = false; +}; + } // namespace // This class is a friend of RequestCoordinator, and can't be in the anonymous @@ -147,6 +162,9 @@ class RequestCoordinatorTest : public testing::Test { RequestCoordinator* coordinator() const { return coordinator_taco_->request_coordinator(); } + SchedulerStub* scheduler_stub() const { + return reinterpret_cast<SchedulerStub*>(coordinator()->scheduler()); + } RequestQueue* queue() { return coordinator_taco_->request_coordinator()->queue_for_testing(); } @@ -258,7 +276,8 @@ class RequestCoordinatorTest : public testing::Test { else coordinator()->disabled_requests_.clear(); - coordinator()->RequestNotPicked(non_user_requested_tasks_remaining, false); + coordinator()->RequestNotPicked(non_user_requested_tasks_remaining, false, + base::Time()); } void SetDeviceConditionsForTest(DeviceConditions device_conditions) { @@ -348,6 +367,9 @@ class RequestCoordinatorTest : public testing::Test { bool add_request_callback_called() { return add_request_callback_called_; } + protected: + ActiveTabInfoStub* active_tab_info_ = nullptr; + private: GetRequestsResult last_get_requests_result_; MultipleItemStatuses last_remove_results_; @@ -400,6 +422,9 @@ void RequestCoordinatorTest::SetUp() { network_quality_tracker_ = test_network_quality_tracker.get(); coordinator_taco_->SetNetworkQualityProvider( std::move(test_network_quality_tracker)); + auto delegate = std::make_unique<ActiveTabInfoStub>(); + active_tab_info_ = delegate.get(); + coordinator_taco_->SetRequestCoordinatorDelegate(std::move(delegate)); coordinator_taco_->CreateRequestCoordinator(); @@ -447,7 +472,7 @@ void RequestCoordinatorTest::SetupForOfflinerDoneCallbackTest( offline_pages::SavePageRequest* request) { // Mark request as started and add it to the queue, // then wait for callback to finish. - request->MarkAttemptStarted(base::Time::Now()); + request->MarkAttemptStarted(OfflineClock()->Now()); queue()->AddRequest(*request, base::BindOnce(&RequestCoordinatorTest::AddRequestDone, base::Unretained(this))); @@ -462,7 +487,7 @@ void RequestCoordinatorTest::SetupForOfflinerDoneCallbackTest( // Mock that coordinator is in actively processing state starting now. SetProcessingStateForTest( RequestCoordinator::ProcessingWindowState::IMMEDIATE_WINDOW); - SetOperationStartTimeForTest(base::Time::Now()); + SetOperationStartTimeForTest(OfflineClock()->Now()); } void RequestCoordinatorTest::SendOfflinerDoneCallback( @@ -473,8 +498,8 @@ void RequestCoordinatorTest::SendOfflinerDoneCallback( } SavePageRequest RequestCoordinatorTest::AddRequest1() { - offline_pages::SavePageRequest request1(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + offline_pages::SavePageRequest request1( + kRequestId1, kUrl1, kClientId1, OfflineClock()->Now(), kUserRequested); queue()->AddRequest(request1, base::BindOnce(&RequestCoordinatorTest::AddRequestDone, base::Unretained(this))); @@ -482,8 +507,8 @@ SavePageRequest RequestCoordinatorTest::AddRequest1() { } SavePageRequest RequestCoordinatorTest::AddRequest2() { - offline_pages::SavePageRequest request2(kRequestId2, kUrl2, kClientId2, - base::Time::Now(), kUserRequested); + offline_pages::SavePageRequest request2( + kRequestId2, kUrl2, kClientId2, OfflineClock()->Now(), kUserRequested); queue()->AddRequest(request2, base::BindOnce(&RequestCoordinatorTest::AddRequestDone, base::Unretained(this))); @@ -629,13 +654,11 @@ TEST_F(RequestCoordinatorTest, SavePageLater) { EXPECT_EQ(kRequestOrigin, last_requests().at(0)->request_origin()); // Expect that the scheduler got notified. - SchedulerStub* scheduler_stub = - reinterpret_cast<SchedulerStub*>(coordinator()->scheduler()); - EXPECT_TRUE(scheduler_stub->schedule_called()); + EXPECT_TRUE(scheduler_stub()->schedule_called()); EXPECT_EQ(coordinator() ->GetTriggerConditions(last_requests()[0]->user_requested()) .minimum_battery_percentage, - scheduler_stub->trigger_conditions()->minimum_battery_percentage); + scheduler_stub()->trigger_conditions()->minimum_battery_percentage); // Check that the observer got the notification that a page is available EXPECT_TRUE(observer().added_called()); @@ -679,13 +702,11 @@ TEST_F(RequestCoordinatorTest, SavePageLaterFailed) { EXPECT_EQ(kClientId1, last_requests().at(0)->client_id()); // Expect that the scheduler got notified. - SchedulerStub* scheduler_stub = - reinterpret_cast<SchedulerStub*>(coordinator()->scheduler()); - EXPECT_TRUE(scheduler_stub->schedule_called()); + EXPECT_TRUE(scheduler_stub()->schedule_called()); EXPECT_EQ(coordinator() ->GetTriggerConditions(last_requests()[0]->user_requested()) .minimum_battery_percentage, - scheduler_stub->trigger_conditions()->minimum_battery_percentage); + scheduler_stub()->trigger_conditions()->minimum_battery_percentage); // Check that the observer got the notification that a page is available EXPECT_TRUE(observer().added_called()); @@ -694,7 +715,7 @@ TEST_F(RequestCoordinatorTest, SavePageLaterFailed) { TEST_F(RequestCoordinatorTest, OfflinerDoneRequestSucceeded) { // Add a request to the queue, wait for callbacks to finish. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); SetupForOfflinerDoneCallbackTest(&request); // Call the OfflinerDoneCallback to simulate the page being completed, wait @@ -725,7 +746,7 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneRequestSucceeded) { TEST_F(RequestCoordinatorTest, OfflinerDoneRequestSucceededButLostNetwork) { // Add a request to the queue and set offliner done callback for it. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); SetupForOfflinerDoneCallbackTest(&request); EnableOfflinerCallback(false); @@ -754,7 +775,7 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneRequestSucceededButLostNetwork) { TEST_F(RequestCoordinatorTest, OfflinerDoneRequestFailed) { // Add a request to the queue, wait for callbacks to finish. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); request.set_completed_attempt_count(kMaxCompletedTries - 1); SetupForOfflinerDoneCallbackTest(&request); // Stop processing before completing the second request on the queue. @@ -797,7 +818,7 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneRequestFailed) { TEST_F(RequestCoordinatorTest, OfflinerDoneRequestFailedNoRetryFailure) { // Add a request to the queue, wait for callbacks to finish. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); SetupForOfflinerDoneCallbackTest(&request); EnableOfflinerCallback(false); @@ -840,7 +861,7 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneRequestFailedNoRetryFailure) { TEST_F(RequestCoordinatorTest, OfflinerDoneRequestFailedNoNextFailure) { // Add a request to the queue, wait for callbacks to finish. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); SetupForOfflinerDoneCallbackTest(&request); EnableOfflinerCallback(false); @@ -872,7 +893,7 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneRequestFailedNoNextFailure) { TEST_F(RequestCoordinatorTest, OfflinerDoneForegroundCancel) { // Add a request to the queue, wait for callbacks to finish. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); SetupForOfflinerDoneCallbackTest(&request); // Call the OfflinerDoneCallback to simulate the request failed, wait @@ -893,17 +914,16 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneForegroundCancel) { EXPECT_EQ(0L, last_requests().at(0)->completed_attempt_count()); } -TEST_F(RequestCoordinatorTest, OfflinerDoneOffliningCancel) { - // Add a request to the queue, wait for callbacks to finish. - offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); - SetupForOfflinerDoneCallbackTest(&request); - - // Call the OfflinerDoneCallback to simulate the request failed, wait - // for callbacks. - SendOfflinerDoneCallback(request, Offliner::RequestStatus::LOADING_CANCELED); +TEST_F(RequestCoordinatorTest, RequestDeferred) { + // Test handling of requests that can be deferred due to + // defer_while_page_is_active. + active_tab_info_->set_does_active_tab_match(true); + RequestCoordinator::SavePageLaterParams params; + params.url = kUrl1; + // Auto-async uses defer_background_fetch_while_page_is_active. + params.client_id = ClientId(kAutoAsyncNamespace, "1"); + coordinator()->SavePageLater(params, base::DoNothing()); PumpLoop(); - EXPECT_TRUE(processing_callback_called()); // Verify the request is not removed from the queue, and wait for callbacks. queue()->GetRequests(base::BindOnce(&RequestCoordinatorTest::GetRequestsDone, @@ -911,11 +931,42 @@ TEST_F(RequestCoordinatorTest, OfflinerDoneOffliningCancel) { PumpLoop(); // Request still in the queue. - EXPECT_EQ(1UL, last_requests().size()); - // Verify offlining cancel not counted as an attempt after all. - const std::unique_ptr<SavePageRequest>& found_request = - last_requests().front(); - EXPECT_EQ(0L, found_request->completed_attempt_count()); + ASSERT_EQ(1UL, last_requests().size()); + EXPECT_EQ(1L, last_requests()[0]->started_attempt_count()); + EXPECT_EQ(1L, last_requests()[0]->completed_attempt_count()); + + // The scheduler is called. Simulate the scheduler calling us back. + // This time, the request was tried recently, and will not be retried again. + // Since there are no requests this time, backup_schedule is called with a + // delay that matches the deferral interval. + ASSERT_TRUE(scheduler_stub()->schedule_called()); + coordinator()->StartScheduledProcessing(device_conditions(), + processing_callback()); + + PumpLoop(); + EXPECT_TRUE(scheduler_stub()->backup_schedule_called()); + // Add plenty of tolerance to avoid flakes. + EXPECT_LT(PickRequestTask::kDeferInterval.InSeconds() - 10, + scheduler_stub()->schedule_delay()); +} + +TEST_F(RequestCoordinatorTest, RequestNotDeferred) { + // Test defer_while_page_is_active=true, but the DoesActiveTabMatch returns + // false. The page should be offlined. + active_tab_info_->set_does_active_tab_match(false); + RequestCoordinator::SavePageLaterParams params; + params.url = kUrl1; + // Auto-async uses defer_background_fetch_while_page_is_active. + params.client_id = ClientId(kAutoAsyncNamespace, "1"); + coordinator()->SavePageLater(params, base::DoNothing()); + PumpLoop(); + + queue()->GetRequests(base::BindOnce(&RequestCoordinatorTest::GetRequestsDone, + base::Unretained(this))); + PumpLoop(); + + // Request was completed. + ASSERT_EQ(0UL, last_requests().size()); } // If one item completes, and there are no more user requeted items left, @@ -933,10 +984,8 @@ TEST_F(RequestCoordinatorTest, RequestNotPickedDisabledItemsRemain) { // The scheduler should have been called to schedule the disabled task for // 5 minutes from now. - SchedulerStub* scheduler_stub = - reinterpret_cast<SchedulerStub*>(coordinator()->scheduler()); - EXPECT_TRUE(scheduler_stub->backup_schedule_called()); - EXPECT_TRUE(scheduler_stub->unschedule_called()); + EXPECT_TRUE(scheduler_stub()->backup_schedule_called()); + EXPECT_TRUE(scheduler_stub()->unschedule_called()); } // If one item completes, and there are no more user requeted items left, @@ -956,12 +1005,10 @@ TEST_F(RequestCoordinatorTest, RequestNotPickedNonUserRequestedItemsRemain) { // The scheduler should have been called to schedule the non-user requested // task. - SchedulerStub* scheduler_stub = - reinterpret_cast<SchedulerStub*>(coordinator()->scheduler()); - EXPECT_TRUE(scheduler_stub->schedule_called()); - EXPECT_TRUE(scheduler_stub->unschedule_called()); + EXPECT_TRUE(scheduler_stub()->schedule_called()); + EXPECT_TRUE(scheduler_stub()->unschedule_called()); const Scheduler::TriggerConditions* conditions = - scheduler_stub->trigger_conditions(); + scheduler_stub()->trigger_conditions(); EXPECT_EQ(conditions->require_power_connected, coordinator()->policy()->PowerRequired(!kUserRequested)); EXPECT_EQ( @@ -975,8 +1022,8 @@ TEST_F(RequestCoordinatorTest, SchedulerGetsLeastRestrictiveConditions) { // Put two requests on the queue - The first is user requested, and // the second is not user requested. AddRequest1(); - offline_pages::SavePageRequest request2(kRequestId2, kUrl2, kClientId2, - base::Time::Now(), !kUserRequested); + offline_pages::SavePageRequest request2( + kRequestId2, kUrl2, kClientId2, OfflineClock()->Now(), !kUserRequested); queue()->AddRequest(request2, base::BindOnce(&RequestCoordinatorTest::AddRequestDone, base::Unretained(this))); @@ -988,11 +1035,9 @@ TEST_F(RequestCoordinatorTest, SchedulerGetsLeastRestrictiveConditions) { // Expect that the scheduler got notified, and it is at user_requested // priority. - SchedulerStub* scheduler_stub = - reinterpret_cast<SchedulerStub*>(coordinator()->scheduler()); const Scheduler::TriggerConditions* conditions = - scheduler_stub->trigger_conditions(); - EXPECT_TRUE(scheduler_stub->schedule_called()); + scheduler_stub()->trigger_conditions(); + EXPECT_TRUE(scheduler_stub()->schedule_called()); EXPECT_EQ(conditions->require_power_connected, coordinator()->policy()->PowerRequired(kUserRequested)); EXPECT_EQ(conditions->minimum_battery_percentage, @@ -1015,7 +1060,8 @@ TEST_F(RequestCoordinatorTest, StartScheduledProcessingWithLoadingDisabled) { EXPECT_TRUE(processing_callback_called()); EXPECT_FALSE(state() == RequestCoordinatorState::PICKING); - EXPECT_EQ(Offliner::LOADING_NOT_ACCEPTED, last_offlining_status()); + EXPECT_EQ(Offliner::RequestStatus::LOADING_NOT_ACCEPTED, + last_offlining_status()); } // TODO(dougarnett): Add StartScheduledProcessing test for QUEUE_UPDATE_FAILED. @@ -1033,7 +1079,8 @@ TEST_F(RequestCoordinatorTest, EXPECT_TRUE(state() == RequestCoordinatorState::PICKING); // Now, quick, before it can do much (we haven't called PumpLoop), cancel it. - coordinator()->StopProcessing(Offliner::REQUEST_COORDINATOR_CANCELED); + coordinator()->StopProcessing( + Offliner::RequestStatus::REQUEST_COORDINATOR_CANCELED); // Let the async callbacks in the request coordinator run. PumpLoop(); @@ -1081,7 +1128,8 @@ TEST_F(RequestCoordinatorTest, EXPECT_FALSE(state() == RequestCoordinatorState::PICKING); // Now we cancel it while the background loader is busy. - coordinator()->StopProcessing(Offliner::REQUEST_COORDINATOR_CANCELED); + coordinator()->StopProcessing( + Offliner::RequestStatus::REQUEST_COORDINATOR_CANCELED); // Let the async callbacks in the cancel run. PumpLoop(); @@ -1191,7 +1239,7 @@ TEST_F(RequestCoordinatorTest, WatchdogTimeoutForScheduledProcessingNoLastSnapshot) { // Build a request to use with the pre-renderer, and put it on the queue. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); // Set request to allow one more completed attempt. int max_tries = coordinator()->policy()->GetMaxCompletedTries(); request.set_completed_attempt_count(max_tries - 1); @@ -1266,7 +1314,8 @@ TEST_F(RequestCoordinatorTest, TimeBudgetExceeded) { AddRequest1(); // The second request will have a larger completed attempt count. offline_pages::SavePageRequest request2(kRequestId1 + 1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), + kUserRequested); request2.set_completed_attempt_count(kAttemptCount); queue()->AddRequest(request2, base::BindOnce(&RequestCoordinatorTest::AddRequestDone, @@ -1538,7 +1587,7 @@ TEST_F(RequestCoordinatorTest, TEST_F(RequestCoordinatorTest, SnapshotOnLastTryForScheduledProcessing) { // Build a request to use with the pre-renderer, and put it on the queue. offline_pages::SavePageRequest request(kRequestId1, kUrl1, kClientId1, - base::Time::Now(), kUserRequested); + OfflineClock()->Now(), kUserRequested); // Set request to allow one more completed attempt. So that the next try would // be the last retry. int max_tries = coordinator()->policy()->GetMaxCompletedTries(); diff --git a/chromium/components/offline_pages/core/background/request_notifier.h b/chromium/components/offline_pages/core/background/request_notifier.h index 10643a64f41..1bf74689756 100644 --- a/chromium/components/offline_pages/core/background/request_notifier.h +++ b/chromium/components/offline_pages/core/background/request_notifier.h @@ -28,8 +28,7 @@ class RequestNotifier { START_COUNT_EXCEEDED = 7, USER_CANCELED = 8, DOWNLOAD_THROTTLED = 9, - // NOTE: insert new values above this line and update histogram enum too. - STATUS_COUNT + kMaxValue = DOWNLOAD_THROTTLED, }; virtual ~RequestNotifier() = default; diff --git a/chromium/components/offline_pages/core/background/request_queue.cc b/chromium/components/offline_pages/core/background/request_queue.cc index 421961453c1..d6644d78564 100644 --- a/chromium/components/offline_pages/core/background/request_queue.cc +++ b/chromium/components/offline_pages/core/background/request_queue.cc @@ -15,6 +15,7 @@ #include "components/offline_pages/core/background/initialize_store_task.h" #include "components/offline_pages/core/background/mark_attempt_aborted_task.h" #include "components/offline_pages/core/background/mark_attempt_completed_task.h" +#include "components/offline_pages/core/background/mark_attempt_deferred_task.h" #include "components/offline_pages/core/background/mark_attempt_started_task.h" #include "components/offline_pages/core/background/pick_request_task.h" #include "components/offline_pages/core/background/reconcile_task.h" @@ -120,8 +121,16 @@ void RequestQueue::MarkAttemptCompleted(int64_t request_id, task_queue_.AddTask(std::move(task)); } +void RequestQueue::MarkAttemptDeferred(int64_t request_id, + UpdateCallback callback) { + std::unique_ptr<Task> task(new MarkAttemptDeferredTask( + store_.get(), request_id, std::move(callback))); + task_queue_.AddTask(std::move(task)); +} + void RequestQueue::PickNextRequest( OfflinerPolicy* policy, + ClientPolicyController* policy_controller, PickRequestTask::RequestPickedCallback picked_callback, PickRequestTask::RequestNotPickedCallback not_picked_callback, PickRequestTask::RequestCountCallback request_count_callback, @@ -130,7 +139,7 @@ void RequestQueue::PickNextRequest( base::circular_deque<int64_t>* prioritized_requests) { // Using the PickerContext, create a picker task. std::unique_ptr<Task> task(new PickRequestTask( - store_.get(), policy, std::move(picked_callback), + store_.get(), policy, policy_controller, std::move(picked_callback), std::move(not_picked_callback), std::move(request_count_callback), std::move(conditions), disabled_requests, prioritized_requests)); diff --git a/chromium/components/offline_pages/core/background/request_queue.h b/chromium/components/offline_pages/core/background/request_queue.h index 5328067163b..bab9ff93283 100644 --- a/chromium/components/offline_pages/core/background/request_queue.h +++ b/chromium/components/offline_pages/core/background/request_queue.h @@ -10,6 +10,7 @@ #include <memory> #include <set> #include <string> +#include <utility> #include <vector> #include "base/callback.h" @@ -29,6 +30,7 @@ namespace offline_pages { class CleanupTaskFactory; +class ClientPolicyController; class RequestQueueStore; // Class responsible for managing save page requests. @@ -86,6 +88,10 @@ class RequestQueue : public TaskQueue::Delegate { // |callback|. void MarkAttemptAborted(int64_t request_id, UpdateCallback callback); + // Marks attempt with |request_id| as deferred. Results are returned through + // |callback|. + void MarkAttemptDeferred(int64_t request_id, UpdateCallback callback); + // Marks attempt with |request_id| as completed. The attempt may have // completed with either success or failure (stored in FailState). Results are // returned through |callback|. @@ -97,6 +103,7 @@ class RequestQueue : public TaskQueue::Delegate { // callbacks. void PickNextRequest( OfflinerPolicy* policy, + ClientPolicyController* policy_controller, PickRequestTask::RequestPickedCallback picked_callback, PickRequestTask::RequestNotPickedCallback not_picked_callback, PickRequestTask::RequestCountCallback request_count_callback, @@ -119,6 +126,8 @@ class RequestQueue : public TaskQueue::Delegate { cleanup_factory_ = std::move(factory); } + RequestQueueStore* GetStoreForTesting() { return store_.get(); } + private: // Store initialization functions. void Initialize(); diff --git a/chromium/components/offline_pages/core/background/request_queue_store.cc b/chromium/components/offline_pages/core/background/request_queue_store.cc index 3262182c1d9..778ee215348 100644 --- a/chromium/components/offline_pages/core/background/request_queue_store.cc +++ b/chromium/components/offline_pages/core/background/request_queue_store.cc @@ -4,6 +4,7 @@ #include "components/offline_pages/core/background/request_queue_store.h" +#include <string> #include <unordered_set> #include <utility> @@ -254,6 +255,7 @@ ItemActionStatus Update(sql::Database* db, const SavePageRequest& request) { " WHERE request_id = ?"; sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); + // SET columns: statement.BindInt64(0, store_utils::ToDatabaseTime(request.creation_time())); statement.BindInt64(1, 0); statement.BindInt64(2, @@ -267,6 +269,7 @@ ItemActionStatus Update(sql::Database* db, const SavePageRequest& request) { statement.BindString(9, request.original_url().spec()); statement.BindString(10, request.request_origin()); statement.BindInt64(11, static_cast<int64_t>(request.fail_state())); + // WHERE: statement.BindInt64(12, request.request_id()); if (!statement.Run()) @@ -284,7 +287,7 @@ void PostStoreUpdateResultForIds( RequestQueueStore::UpdateCallback callback) { UpdateRequestsResult result(store_state); for (const auto& item_id : item_ids) - result.item_statuses.push_back(std::make_pair(item_id, action_status)); + result.item_statuses.emplace_back(item_id, action_status); runner->PostTask(FROM_HERE, base::BindOnce(std::move(callback), std::move(result))); } @@ -376,7 +379,7 @@ void GetRequestsByIdsSync(sql::Database* db, result.updated_items.push_back(*request); ItemActionStatus status = request ? ItemActionStatus::SUCCESS : ItemActionStatus::NOT_FOUND; - result.item_statuses.push_back(std::make_pair(request_id, status)); + result.item_statuses.emplace_back(request_id, status); } if (!transaction.Commit()) { @@ -410,8 +413,7 @@ void UpdateRequestsSync(sql::Database* db, for (const auto& request : requests) { ItemActionStatus status = Update(db, request); - result.item_statuses.push_back( - std::make_pair(request.request_id(), status)); + result.item_statuses.emplace_back(request.request_id(), status); if (status == ItemActionStatus::SUCCESS) result.updated_items.push_back(request); } diff --git a/chromium/components/offline_pages/core/background/request_queue_store_unittest.cc b/chromium/components/offline_pages/core/background/request_queue_store_unittest.cc index fc0da9fd9e1..c448a6dbebe 100644 --- a/chromium/components/offline_pages/core/background/request_queue_store_unittest.cc +++ b/chromium/components/offline_pages/core/background/request_queue_store_unittest.cc @@ -5,14 +5,18 @@ #include "components/offline_pages/core/background/request_queue_store.h" #include <memory> +#include <string> +#include <utility> #include "base/bind.h" #include "base/files/file_path.h" #include "base/files/scoped_temp_dir.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/request_queue.h" #include "components/offline_pages/core/background/save_page_request.h" +#include "components/offline_pages/core/offline_clock.h" #include "sql/database.h" #include "sql/statement.h" #include "testing/gtest/include/gtest/gtest.h" @@ -32,7 +36,7 @@ const GURL kUrl2("http://another-example.com"); const ClientId kClientId("bookmark", "1234"); const ClientId kClientId2("async", "5678"); const bool kUserRequested = true; -const std::string kRequestOrigin = "abc.xyz"; +const char kRequestOrigin[] = "abc.xyz"; enum class LastResult { RESULT_NONE, @@ -40,6 +44,23 @@ enum class LastResult { RESULT_TRUE, }; +SavePageRequest GetTestRequest() { + SavePageRequest request(kRequestId, kUrl, kClientId, + base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromSeconds(1000)), + kUserRequested); + // Set fields to non-default values. + request.set_fail_state(offline_items_collection::FailState::FILE_NO_SPACE); + request.set_started_attempt_count(2); + request.set_completed_attempt_count(3); + request.set_last_attempt_time(base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromSeconds(400))); + request.set_request_origin("http://www.origin.com"); + // Note: pending_state is not stored. + request.set_original_url(kUrl2); + return request; +} + void BuildTestStoreWithSchemaFromM57(const base::FilePath& file) { sql::Database connection; ASSERT_TRUE( @@ -390,7 +411,7 @@ TEST_F(RequestQueueStoreTest, GetRequestsByIds) { std::unique_ptr<RequestQueueStore> store(this->BuildStore()); this->InitializeStore(store.get()); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId, kUrl, kClientId, creation_time, kUserRequested); store->AddRequest(request1, @@ -451,7 +472,7 @@ TEST_F(RequestQueueStoreTest, AddRequest) { std::unique_ptr<RequestQueueStore> store(this->BuildStore()); this->InitializeStore(store.get()); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); request.set_original_url(kUrl2); @@ -492,11 +513,28 @@ TEST_F(RequestQueueStoreTest, AddRequest) { ASSERT_EQ(1ul, this->last_requests().size()); } +TEST_F(RequestQueueStoreTest, AddAndGetRequestsMatch) { + std::unique_ptr<RequestQueueStore> store(this->BuildStore()); + this->InitializeStore(store.get()); + const SavePageRequest request = GetTestRequest(); + store->AddRequest(request, + base::BindOnce(&RequestQueueStoreTestBase::AddRequestDone, + base::Unretained(this))); + store->GetRequests(base::BindOnce(&RequestQueueStoreTestBase::GetRequestsDone, + base::Unretained(this))); + this->PumpLoop(); + + ASSERT_EQ(ItemActionStatus::SUCCESS, this->last_add_status()); + ASSERT_EQ(LastResult::RESULT_TRUE, this->last_result()); + ASSERT_EQ(1ul, this->last_requests().size()); + EXPECT_EQ(request.ToString(), this->last_requests()[0]->ToString()); +} + TEST_F(RequestQueueStoreTest, UpdateRequest) { std::unique_ptr<RequestQueueStore> store(this->BuildStore()); this->InitializeStore(store.get()); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest original_request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); store->AddRequest(original_request, @@ -532,6 +570,8 @@ TEST_F(RequestQueueStoreTest, UpdateRequest) { EXPECT_EQ(ItemActionStatus::NOT_FOUND, this->last_update_result()->item_statuses[1].second); EXPECT_EQ(1UL, this->last_update_result()->updated_items.size()); + EXPECT_EQ(updated_request.ToString(), + this->last_update_result()->updated_items.begin()->ToString()); EXPECT_EQ(updated_request, *(this->last_update_result()->updated_items.begin())); @@ -550,7 +590,7 @@ TEST_F(RequestQueueStoreTest, RemoveRequests) { std::unique_ptr<RequestQueueStore> store(this->BuildStore()); this->InitializeStore(store.get()); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId, kUrl, kClientId, creation_time, kUserRequested); store->AddRequest(request1, @@ -614,7 +654,7 @@ TEST_F(RequestQueueStoreTest, ResetStore) { std::unique_ptr<RequestQueueStore> store(this->BuildStore()); this->InitializeStore(store.get()); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest original_request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); store->AddRequest(original_request, @@ -644,7 +684,7 @@ TEST_F(RequestQueueStoreTest, SaveCloseReopenRead) { std::unique_ptr<RequestQueueStore> store(BuildStore()); this->InitializeStore(store.get()); - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest original_request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); store->AddRequest(original_request, diff --git a/chromium/components/offline_pages/core/background/request_queue_unittest.cc b/chromium/components/offline_pages/core/background/request_queue_unittest.cc index 0ea44f982e7..c07dd3d30e8 100644 --- a/chromium/components/offline_pages/core/background/request_queue_unittest.cc +++ b/chromium/components/offline_pages/core/background/request_queue_unittest.cc @@ -10,6 +10,7 @@ #include "base/bind.h" #include "base/test/test_mock_time_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/clock.h" #include "components/offline_pages/core/background/device_conditions.h" #include "components/offline_pages/core/background/offliner_policy.h" #include "components/offline_pages/core/background/request_coordinator.h" @@ -18,6 +19,7 @@ #include "components/offline_pages/core/background/request_queue_store.h" #include "components/offline_pages/core/background/save_page_request.h" #include "components/offline_pages/core/background/test_request_queue_store.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -208,7 +210,7 @@ TEST_F(RequestQueueTest, GetRequestsEmpty) { } TEST_F(RequestQueueTest, AddRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, @@ -226,7 +228,7 @@ TEST_F(RequestQueueTest, AddRequest) { } TEST_F(RequestQueueTest, RemoveRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, @@ -254,7 +256,7 @@ TEST_F(RequestQueueTest, RemoveRequest) { } TEST_F(RequestQueueTest, RemoveSeveralRequests) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, @@ -302,7 +304,7 @@ TEST_F(RequestQueueTest, RemoveSeveralRequests) { } TEST_F(RequestQueueTest, PauseAndResume) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, @@ -371,7 +373,7 @@ TEST_F(RequestQueueTest, PauseAndResume) { // A longer test populating the request queue with more than one item, properly // listing multiple items and removing the right item. TEST_F(RequestQueueTest, MultipleRequestsAddGetRemove) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request1(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request1, @@ -414,14 +416,14 @@ TEST_F(RequestQueueTest, MultipleRequestsAddGetRemove) { TEST_F(RequestQueueTest, MarkAttemptStarted) { // First add a request. Retry count will be set to 0. - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, base::Unretained(this))); PumpLoop(); - base::Time before_time = base::Time::Now(); + base::Time before_time = OfflineClock()->Now(); // Update the request, ensure it succeeded. queue()->MarkAttemptStarted( kRequestId, base::BindOnce(&RequestQueueTest::UpdateRequestsDone, @@ -434,7 +436,7 @@ TEST_F(RequestQueueTest, MarkAttemptStarted) { EXPECT_EQ(1UL, update_requests_result()->updated_items.size()); EXPECT_LE(before_time, update_requests_result()->updated_items.at(0).last_attempt_time()); - EXPECT_GE(base::Time::Now(), + EXPECT_GE(OfflineClock()->Now(), update_requests_result()->updated_items.at(0).last_attempt_time()); EXPECT_EQ( 1, update_requests_result()->updated_items.at(0).started_attempt_count()); @@ -452,7 +454,7 @@ TEST_F(RequestQueueTest, MarkAttemptStarted) { TEST_F(RequestQueueTest, MarkAttempStartedRequestNotPresent) { // First add a request. Retry count will be set to 0. - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // This request is never put into the queue. SavePageRequest request1(kRequestId, kUrl, kClientId, creation_time, kUserRequested); @@ -469,7 +471,7 @@ TEST_F(RequestQueueTest, MarkAttempStartedRequestNotPresent) { } TEST_F(RequestQueueTest, MarkAttemptAborted) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, @@ -500,7 +502,7 @@ TEST_F(RequestQueueTest, MarkAttemptAborted) { TEST_F(RequestQueueTest, MarkAttemptAbortedRequestNotPresent) { // First add a request. Retry count will be set to 0. - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); // This request is never put into the queue. SavePageRequest request1(kRequestId, kUrl, kClientId, creation_time, kUserRequested); @@ -517,7 +519,7 @@ TEST_F(RequestQueueTest, MarkAttemptAbortedRequestNotPresent) { } TEST_F(RequestQueueTest, MarkAttemptCompleted) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); queue()->AddRequest(request, base::BindOnce(&RequestQueueTest::AddRequestDone, @@ -550,7 +552,8 @@ TEST_F(RequestQueueTest, MarkAttemptCompleted) { TEST_F(RequestQueueTest, CleanStaleRequests) { // Create a request that is already expired. base::Time creation_time = - base::Time::Now() - base::TimeDelta::FromSeconds(2 * kOneWeekInSeconds); + OfflineClock()->Now() - + base::TimeDelta::FromSeconds(2 * kOneWeekInSeconds); SavePageRequest original_request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); diff --git a/chromium/components/offline_pages/core/background/save_page_request.cc b/chromium/components/offline_pages/core/background/save_page_request.cc index 56b391abe50..33b08266a94 100644 --- a/chromium/components/offline_pages/core/background/save_page_request.cc +++ b/chromium/components/offline_pages/core/background/save_page_request.cc @@ -4,6 +4,8 @@ #include "components/offline_pages/core/background/save_page_request.h" +#include <string> + namespace offline_pages { SavePageRequest::SavePageRequest(int64_t request_id, @@ -63,6 +65,13 @@ void SavePageRequest::MarkAttemptPaused() { state_ = RequestState::PAUSED; } +void SavePageRequest::MarkAttemptDeferred(const base::Time& attempt_time) { + ++started_attempt_count_; + ++completed_attempt_count_; + last_attempt_time_ = attempt_time; + state_ = RequestState::AVAILABLE; +} + void SavePageRequest::UpdateFailState(FailState fail_state) { // The order of precedence for failure errors related to offline page // downloads is as follows: NO_FAILURE, Failures that are not recoverable and diff --git a/chromium/components/offline_pages/core/background/save_page_request.h b/chromium/components/offline_pages/core/background/save_page_request.h index ed9d4534c66..e24ba2d1cad 100644 --- a/chromium/components/offline_pages/core/background/save_page_request.h +++ b/chromium/components/offline_pages/core/background/save_page_request.h @@ -53,6 +53,10 @@ class SavePageRequest { // loading until it has been explicitly unpaused. void MarkAttemptPaused(); + // Mark the attempt as deferred. This counts as a failed attempt so that + // deferred attempts are not unlimited. + void MarkAttemptDeferred(const base::Time& attempt_time); + int64_t request_id() const { return request_id_; } const GURL& url() const { return url_; } @@ -100,6 +104,9 @@ class SavePageRequest { request_origin_ = request_origin; } + // Implemented in test_util.cc. + std::string ToString() const; + private: // ID of this request. int64_t request_id_; diff --git a/chromium/components/offline_pages/core/background/save_page_request_unittest.cc b/chromium/components/offline_pages/core/background/save_page_request_unittest.cc index ab7e7223ef6..e2853177c02 100644 --- a/chromium/components/offline_pages/core/background/save_page_request_unittest.cc +++ b/chromium/components/offline_pages/core/background/save_page_request_unittest.cc @@ -4,6 +4,8 @@ #include "components/offline_pages/core/background/save_page_request.h" +#include "base/time/clock.h" +#include "components/offline_pages/core/offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -25,7 +27,7 @@ class SavePageRequestTest : public testing::Test { SavePageRequestTest::~SavePageRequestTest() {} TEST_F(SavePageRequestTest, CreatePendingReqeust) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); request.set_original_url(kUrl2); @@ -43,7 +45,7 @@ TEST_F(SavePageRequestTest, CreatePendingReqeust) { } TEST_F(SavePageRequestTest, StartAndCompleteRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); request.set_request_origin(kRequestOrigin); @@ -77,7 +79,7 @@ TEST_F(SavePageRequestTest, StartAndCompleteRequest) { } TEST_F(SavePageRequestTest, StartAndAbortRequest) { - base::Time creation_time = base::Time::Now(); + base::Time creation_time = OfflineClock()->Now(); SavePageRequest request(kRequestId, kUrl, kClientId, creation_time, kUserRequested); diff --git a/chromium/components/offline_pages/core/background/scheduler_stub.h b/chromium/components/offline_pages/core/background/scheduler_stub.h index 56d2d8ae701..6badeac03d9 100644 --- a/chromium/components/offline_pages/core/background/scheduler_stub.h +++ b/chromium/components/offline_pages/core/background/scheduler_stub.h @@ -39,6 +39,8 @@ class SchedulerStub : public Scheduler { return &trigger_conditions_; } + int64_t schedule_delay() const { return schedule_delay_; } + private: bool schedule_called_; bool backup_schedule_called_; diff --git a/chromium/components/offline_pages/core/background/test_util.cc b/chromium/components/offline_pages/core/background/test_util.cc new file mode 100644 index 00000000000..a106010d65f --- /dev/null +++ b/chromium/components/offline_pages/core/background/test_util.cc @@ -0,0 +1,34 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/json/json_writer.h" +#include "base/values.h" +#include "components/offline_pages/core/background/save_page_request.h" + +namespace offline_pages { + +std::string SavePageRequest::ToString() const { + base::DictionaryValue result; + result.SetInteger("request_id", request_id_); + result.SetString("url", url_.spec()); + result.SetString("client_id", client_id_.ToString()); + result.SetInteger("creation_time", + creation_time_.ToDeltaSinceWindowsEpoch().InSeconds()); + result.SetInteger("started_attempt_count", started_attempt_count_); + result.SetInteger("completed_attempt_count", completed_attempt_count_); + result.SetInteger("last_attempt_time", + last_attempt_time_.ToDeltaSinceWindowsEpoch().InSeconds()); + result.SetBoolean("user_requested", user_requested_); + result.SetInteger("state", static_cast<int>(state_)); + result.SetInteger("fail_state", static_cast<int>(fail_state_)); + result.SetInteger("pending_state", static_cast<int>(pending_state_)); + result.SetString("original_url", original_url_.spec()); + result.SetString("request_origin", request_origin_); + + std::string result_string; + base::JSONWriter::Write(result, &result_string); + return result_string; +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/client_namespace_constants.cc b/chromium/components/offline_pages/core/client_namespace_constants.cc index 99b2270227f..778b1e11d83 100644 --- a/chromium/components/offline_pages/core/client_namespace_constants.cc +++ b/chromium/components/offline_pages/core/client_namespace_constants.cc @@ -17,6 +17,7 @@ const char kNTPSuggestionsNamespace[] = "ntp_suggestions"; const char kSuggestedArticlesNamespace[] = "suggested_articles"; const char kBrowserActionsNamespace[] = "browser_actions"; const char kLivePageSharingNamespace[] = "live_page_sharing"; +const char kAutoAsyncNamespace[] = "auto_async_loading"; const char kDefaultNamespace[] = "default"; diff --git a/chromium/components/offline_pages/core/client_namespace_constants.h b/chromium/components/offline_pages/core/client_namespace_constants.h index c02e37171c4..db3a12fda91 100644 --- a/chromium/components/offline_pages/core/client_namespace_constants.h +++ b/chromium/components/offline_pages/core/client_namespace_constants.h @@ -25,11 +25,12 @@ extern const char kNTPSuggestionsNamespace[]; extern const char kSuggestedArticlesNamespace[]; extern const char kBrowserActionsNamespace[]; extern const char kLivePageSharingNamespace[]; +extern const char kAutoAsyncNamespace[]; // Enum of namespaces used by metric collection. // See OfflinePagesNamespaceEnumeration in enums.xml for histogram usages. // Changes to this enum should be in sync with the changes to the namespace -// constants above. +// constants above and with the metrics enum. enum class OfflinePagesNamespaceEnumeration { DEFAULT = 0, BOOKMARK = 1, @@ -41,10 +42,8 @@ enum class OfflinePagesNamespaceEnumeration { SUGGESTED_ARTICLES = 7, BROWSER_ACTIONS = 8, LIVE_PAGE_SHARING = 9, - // NOTE: always keep this entry at the end. Add new result types only - // immediately above this line. Make sure to update the corresponding - // histogram enum accordingly. - RESULT_COUNT, + ASYNC_AUTO_LOADING = 10, + kMaxValue = ASYNC_AUTO_LOADING, }; } // namespace offline_pages diff --git a/chromium/components/offline_pages/core/client_policy_controller.cc b/chromium/components/offline_pages/core/client_policy_controller.cc index 9e3ea54d053..7f6f871b156 100644 --- a/chromium/components/offline_pages/core/client_policy_controller.cc +++ b/chromium/components/offline_pages/core/client_policy_controller.cc @@ -17,51 +17,50 @@ namespace offline_pages { ClientPolicyController::ClientPolicyController() { policies_.clear(); // Manually defining client policies for bookmark and last_n. - policies_.insert(std::make_pair( + policies_.emplace( kBookmarkNamespace, MakePolicy(kBookmarkNamespace, LifetimeType::TEMPORARY, - base::TimeDelta::FromDays(7), kUnlimitedPages, 1))); - policies_.insert(std::make_pair( + base::TimeDelta::FromDays(7), kUnlimitedPages, 1)); + policies_.emplace( kLastNNamespace, OfflinePageClientPolicyBuilder(kLastNNamespace, LifetimeType::TEMPORARY, kUnlimitedPages, kUnlimitedPages) .SetExpirePeriod(base::TimeDelta::FromDays(30)) .SetIsSupportedByRecentTabs(true) .SetIsRestrictedToTabFromClientId(true) - .Build())); - policies_.insert(std::make_pair( + .Build()); + policies_.emplace( kAsyncNamespace, OfflinePageClientPolicyBuilder(kAsyncNamespace, LifetimeType::PERSISTENT, kUnlimitedPages, kUnlimitedPages) .SetIsSupportedByDownload(true) .SetIsUserRequestedDownload(true) .SetIsRemovedOnCacheReset(false) - .Build())); - policies_.insert(std::make_pair( + .Build()); + policies_.emplace( kCCTNamespace, OfflinePageClientPolicyBuilder(kCCTNamespace, LifetimeType::TEMPORARY, kUnlimitedPages, 1) .SetExpirePeriod(base::TimeDelta::FromDays(2)) .SetIsDisabledWhenPrefetchDisabled(true) - .Build())); - policies_.insert(std::make_pair( - kDownloadNamespace, OfflinePageClientPolicyBuilder( - kDownloadNamespace, LifetimeType::PERSISTENT, - kUnlimitedPages, kUnlimitedPages) - .SetIsRemovedOnCacheReset(false) - .SetIsSupportedByDownload(true) - .SetIsUserRequestedDownload(true) - .Build())); - policies_.insert(std::make_pair( - kNTPSuggestionsNamespace, - OfflinePageClientPolicyBuilder(kNTPSuggestionsNamespace, - LifetimeType::PERSISTENT, kUnlimitedPages, - kUnlimitedPages) - .SetIsSupportedByDownload(true) - .SetIsUserRequestedDownload(true) - .SetIsRemovedOnCacheReset(false) - .Build())); - policies_.insert(std::make_pair( + .Build()); + policies_.emplace(kDownloadNamespace, + OfflinePageClientPolicyBuilder( + kDownloadNamespace, LifetimeType::PERSISTENT, + kUnlimitedPages, kUnlimitedPages) + .SetIsRemovedOnCacheReset(false) + .SetIsSupportedByDownload(true) + .SetIsUserRequestedDownload(true) + .Build()); + policies_.emplace(kNTPSuggestionsNamespace, + OfflinePageClientPolicyBuilder( + kNTPSuggestionsNamespace, LifetimeType::PERSISTENT, + kUnlimitedPages, kUnlimitedPages) + .SetIsSupportedByDownload(true) + .SetIsUserRequestedDownload(true) + .SetIsRemovedOnCacheReset(false) + .Build()); + policies_.emplace( kSuggestedArticlesNamespace, OfflinePageClientPolicyBuilder(kSuggestedArticlesNamespace, LifetimeType::TEMPORARY, kUnlimitedPages, @@ -71,31 +70,38 @@ ClientPolicyController::ClientPolicyController() { .SetExpirePeriod(base::TimeDelta::FromDays(30)) .SetIsSupportedByDownload(IsPrefetchingOfflinePagesEnabled()) .SetIsSuggested(true) - .Build())); - policies_.insert(std::make_pair( - kBrowserActionsNamespace, - OfflinePageClientPolicyBuilder(kBrowserActionsNamespace, - LifetimeType::PERSISTENT, kUnlimitedPages, - kUnlimitedPages) - .SetIsRemovedOnCacheReset(false) - .SetIsSupportedByDownload(true) - .SetIsUserRequestedDownload(true) - .SetShouldAllowDownload(true) - .Build())); - policies_.insert( - std::make_pair(kLivePageSharingNamespace, - OfflinePageClientPolicyBuilder(kLivePageSharingNamespace, - LifetimeType::TEMPORARY, - kUnlimitedPages, 1) - .SetIsRemovedOnCacheReset(true) - .SetExpirePeriod(base::TimeDelta::FromHours(1)) - .SetIsRestrictedToTabFromClientId(true) - .Build())); + .Build()); + policies_.emplace(kBrowserActionsNamespace, + OfflinePageClientPolicyBuilder( + kBrowserActionsNamespace, LifetimeType::PERSISTENT, + kUnlimitedPages, kUnlimitedPages) + .SetIsRemovedOnCacheReset(false) + .SetIsSupportedByDownload(true) + .SetIsUserRequestedDownload(true) + .SetShouldAllowDownload(true) + .Build()); + policies_.emplace(kLivePageSharingNamespace, + OfflinePageClientPolicyBuilder(kLivePageSharingNamespace, + LifetimeType::TEMPORARY, + kUnlimitedPages, 1) + .SetIsRemovedOnCacheReset(true) + .SetExpirePeriod(base::TimeDelta::FromHours(1)) + .SetIsRestrictedToTabFromClientId(true) + .Build()); + policies_.emplace( + kAutoAsyncNamespace, + OfflinePageClientPolicyBuilder( + kAutoAsyncNamespace, LifetimeType::TEMPORARY, kUnlimitedPages, 1) + .SetIsRemovedOnCacheReset(true) + .SetExpirePeriod(base::TimeDelta::FromDays(30)) + .SetIsUserRequestedDownload(false) + .SetDeferBackgroundFetchWhilePageIsActive(true) + .Build()); // Fallback policy. - policies_.insert(std::make_pair( - kDefaultNamespace, MakePolicy(kDefaultNamespace, LifetimeType::TEMPORARY, - base::TimeDelta::FromDays(1), 10, 1))); + policies_.emplace(kDefaultNamespace, + MakePolicy(kDefaultNamespace, LifetimeType::TEMPORARY, + base::TimeDelta::FromDays(1), 10, 1)); } ClientPolicyController::~ClientPolicyController() {} @@ -257,7 +263,7 @@ bool ClientPolicyController::ShouldAllowDownloads( void ClientPolicyController::AddPolicyForTest( const std::string& name_space, const OfflinePageClientPolicyBuilder& builder) { - policies_.insert(std::make_pair(name_space, builder.Build())); + policies_.emplace(name_space, builder.Build()); } } // namespace offline_pages diff --git a/chromium/components/offline_pages/core/model/add_page_task_unittest.cc b/chromium/components/offline_pages/core/model/add_page_task_unittest.cc index 3e305e0174b..bca504c8c85 100644 --- a/chromium/components/offline_pages/core/model/add_page_task_unittest.cc +++ b/chromium/components/offline_pages/core/model/add_page_task_unittest.cc @@ -12,6 +12,7 @@ #include "base/bind.h" #include "base/files/file_path.h" #include "base/memory/weak_ptr.h" +#include "base/optional.h" #include "base/strings/utf_string_conversions.h" #include "base/threading/thread_task_runner_handle.h" #include "components/offline_pages/core/model/model_task_test_base.h" @@ -48,14 +49,16 @@ class AddPageTaskTest : public ModelTaskTestBase { void AddPage(const OfflinePageItem& page); bool CheckPageStored(const OfflinePageItem& page); - AddPageResult last_add_page_result() { return last_add_page_result_; } + const base::Optional<AddPageResult>& last_add_page_result() { + return last_add_page_result_; + } private: - AddPageResult last_add_page_result_ = AddPageResult::RESULT_COUNT; + base::Optional<AddPageResult> last_add_page_result_; }; void AddPageTaskTest::ResetResults() { - last_add_page_result_ = AddPageResult::RESULT_COUNT; + last_add_page_result_.reset(); } void AddPageTaskTest::OnAddPageDone(AddPageResult result) { diff --git a/chromium/components/offline_pages/core/model/clear_storage_task.cc b/chromium/components/offline_pages/core/model/clear_storage_task.cc index d890abd8e75..1c0ea825ab4 100644 --- a/chromium/components/offline_pages/core/model/clear_storage_task.cc +++ b/chromium/components/offline_pages/core/model/clear_storage_task.cc @@ -215,8 +215,8 @@ std::map<std::string, LifetimePolicy> GetTempNamespacePolicyMap( std::map<std::string, LifetimePolicy> result; for (const auto& name_space : policy_controller->GetNamespacesRemovedOnCacheReset()) { - result.insert(std::make_pair( - name_space, policy_controller->GetPolicy(name_space).lifetime_policy)); + result.emplace(name_space, + policy_controller->GetPolicy(name_space).lifetime_policy); } return result; } diff --git a/chromium/components/offline_pages/core/model/clear_storage_task.h b/chromium/components/offline_pages/core/model/clear_storage_task.h index 76bc208e502..8d2904b84a4 100644 --- a/chromium/components/offline_pages/core/model/clear_storage_task.h +++ b/chromium/components/offline_pages/core/model/clear_storage_task.h @@ -36,10 +36,7 @@ class ClearStorageTask : public Task { DELETE_FAILURE, // Deletion failed. DEPRECATED_EXPIRE_AND_DELETE_FAILURES, // Both expiration and deletion // failed. (DEPRECATED) - // NOTE: always keep this entry at the end. Add new result types only - // immediately above this line. Make sure to update the corresponding - // histogram enum accordingly. - RESULT_COUNT, + kMaxValue = DEPRECATED_EXPIRE_AND_DELETE_FAILURES, }; // Callback used when calling ClearPagesIfNeeded. diff --git a/chromium/components/offline_pages/core/model/complete_offline_page_upgrade_task.cc b/chromium/components/offline_pages/core/model/complete_offline_page_upgrade_task.cc index 3e69e40beec..d189bf2ae70 100644 --- a/chromium/components/offline_pages/core/model/complete_offline_page_upgrade_task.cc +++ b/chromium/components/offline_pages/core/model/complete_offline_page_upgrade_task.cc @@ -6,7 +6,7 @@ #include "base/bind.h" #include "base/files/file_util.h" -#include "base/sys_info.h" +#include "base/system/sys_info.h" #include "components/offline_pages/core/offline_page_metadata_store.h" #include "components/offline_pages/core/offline_store_utils.h" #include "sql/database.h" diff --git a/chromium/components/offline_pages/core/model/delete_page_task_unittest.cc b/chromium/components/offline_pages/core/model/delete_page_task_unittest.cc index 4965a90e6e0..bb9ad9052a2 100644 --- a/chromium/components/offline_pages/core/model/delete_page_task_unittest.cc +++ b/chromium/components/offline_pages/core/model/delete_page_task_unittest.cc @@ -10,6 +10,7 @@ #include "base/bind.h" #include "base/files/file_util.h" #include "base/memory/weak_ptr.h" +#include "base/optional.h" #include "base/strings/utf_string_conversions.h" #include "base/test/metrics/histogram_tester.h" #include "components/offline_pages/core/client_namespace_constants.h" @@ -49,7 +50,7 @@ class DeletePageTaskTest : public ModelTaskTestBase { DeletePageTask::DeletePageTaskCallback delete_page_callback(); base::HistogramTester* histogram_tester() { return histogram_tester_.get(); } - DeletePageResult last_delete_page_result() { + const base::Optional<DeletePageResult>& last_delete_page_result() { return last_delete_page_result_; } const std::vector<DeletedPageInfo>& last_deleted_page_infos() { @@ -59,12 +60,11 @@ class DeletePageTaskTest : public ModelTaskTestBase { private: std::unique_ptr<base::HistogramTester> histogram_tester_; - DeletePageResult last_delete_page_result_; + base::Optional<DeletePageResult> last_delete_page_result_; std::vector<DeletedPageInfo> last_deleted_page_infos_; }; -DeletePageTaskTest::DeletePageTaskTest() - : last_delete_page_result_(DeletePageResult::RESULT_COUNT) {} +DeletePageTaskTest::DeletePageTaskTest() {} DeletePageTaskTest::~DeletePageTaskTest() {} diff --git a/chromium/components/offline_pages/core/model/mark_page_accessed_task.cc b/chromium/components/offline_pages/core/model/mark_page_accessed_task.cc index 5e70735ae11..9e3de269067 100644 --- a/chromium/components/offline_pages/core/model/mark_page_accessed_task.cc +++ b/chromium/components/offline_pages/core/model/mark_page_accessed_task.cc @@ -37,8 +37,7 @@ void ReportAccessHistogram(int64_t offline_id, if (statement.Step()) { std::string name_space = statement.ColumnString(0); UMA_HISTOGRAM_ENUMERATION("OfflinePages.AccessPageCount", - model_utils::ToNamespaceEnum(name_space), - OfflinePagesNamespaceEnumeration::RESULT_COUNT); + model_utils::ToNamespaceEnum(name_space)); base::Time last_access_time = store_utils::FromDatabaseTime(statement.ColumnInt64(1)); diff --git a/chromium/components/offline_pages/core/model/offline_page_model_taskified.cc b/chromium/components/offline_pages/core/model/offline_page_model_taskified.cc index 2b856b83aa5..a966e392377 100644 --- a/chromium/components/offline_pages/core/model/offline_page_model_taskified.cc +++ b/chromium/components/offline_pages/core/model/offline_page_model_taskified.cc @@ -89,8 +89,6 @@ SavePageResult AddPageResultToSavePageResult(AddPageResult add_page_result) { return SavePageResult::ALREADY_EXISTS; case AddPageResult::STORE_FAILURE: return SavePageResult::STORE_FAILURE; - case AddPageResult::RESULT_COUNT: - break; } NOTREACHED(); return SavePageResult::STORE_FAILURE; @@ -98,13 +96,11 @@ SavePageResult AddPageResultToSavePageResult(AddPageResult add_page_result) { void ReportPageHistogramAfterSuccessfulSaving( const OfflinePageItem& offline_page, - const base::Time& save_time) { - base::UmaHistogramCustomTimes( + base::Time save_time) { + base::UmaHistogramTimes( model_utils::AddHistogramSuffix(offline_page.client_id.name_space, "OfflinePages.SavePageTime"), - save_time - offline_page.creation_time, - base::TimeDelta::FromMilliseconds(1), base::TimeDelta::FromSeconds(10), - 50); + save_time - offline_page.creation_time); base::UmaHistogramCustomCounts( model_utils::AddHistogramSuffix(offline_page.client_id.name_space, @@ -243,7 +239,8 @@ void OfflinePageModelTaskified::SavePage( if (offline_id == kInvalidOfflineId) offline_id = store_utils::GenerateOfflineId(); - OfflinePageArchiver::CreateArchiveParams create_archive_params; + OfflinePageArchiver::CreateArchiveParams create_archive_params( + save_page_params.client_id.name_space); // If the page is being saved in the background, we should try to remove the // popup overlay that obstructs viewing the normal content. create_archive_params.remove_popup_overlay = save_page_params.is_background; @@ -470,12 +467,11 @@ void OfflinePageModelTaskified::InformSavePageDone(SavePageCallback callback, const ClientId& client_id, int64_t offline_id) { UMA_HISTOGRAM_ENUMERATION("OfflinePages.SavePageCount", - model_utils::ToNamespaceEnum(client_id.name_space), - OfflinePagesNamespaceEnumeration::RESULT_COUNT); + model_utils::ToNamespaceEnum(client_id.name_space)); base::UmaHistogramEnumeration( model_utils::AddHistogramSuffix(client_id.name_space, "OfflinePages.SavePageResult"), - result, SavePageResult::RESULT_COUNT); + result); // Report storage usage if saving page succeeded. if (result == SavePageResult::SUCCESS) @@ -490,7 +486,7 @@ void OfflinePageModelTaskified::InformSavePageDone(SavePageCallback callback, void OfflinePageModelTaskified::OnCreateArchiveDone( const SavePageParams& save_page_params, int64_t offline_id, - const base::Time& start_time, + base::Time start_time, std::unique_ptr<OfflinePageArchiver> archiver, SavePageCallback callback, ArchiverResult archiver_result, @@ -539,7 +535,7 @@ void OfflinePageModelTaskified::OnCreateArchiveDone( download_manager_.get(), base::BindOnce(&OfflinePageModelTaskified::PublishArchiveDone, weak_ptr_factory_.GetWeakPtr(), std::move(archiver), - std::move(callback))); + std::move(callback), GetCurrentTime())); return; } @@ -549,7 +545,7 @@ void OfflinePageModelTaskified::OnCreateArchiveDone( AddPage(offline_page, base::BindOnce(&OfflinePageModelTaskified::OnAddPageForSavePageDone, weak_ptr_factory_.GetWeakPtr(), std::move(callback), - offline_page)); + offline_page, GetCurrentTime())); // Note: If the archiver instance ownership was not transferred, it will be // deleted here. } @@ -557,25 +553,32 @@ void OfflinePageModelTaskified::OnCreateArchiveDone( void OfflinePageModelTaskified::PublishArchiveDone( std::unique_ptr<OfflinePageArchiver> archiver, SavePageCallback save_page_callback, + base::Time publish_start_time, const OfflinePageItem& offline_page, PublishArchiveResult publish_results) { if (publish_results.move_result != SavePageResult::SUCCESS) { // Add UMA for the failure reason. UMA_HISTOGRAM_ENUMERATION("OfflinePages.PublishPageResult", - publish_results.move_result, - SavePageResult::RESULT_COUNT); + publish_results.move_result); std::move(save_page_callback).Run(publish_results.move_result, 0LL); return; } + + const base::Time add_page_start_time = GetCurrentTime(); + base::UmaHistogramTimes(model_utils::AddHistogramSuffix( + offline_page.client_id.name_space, + "OfflinePages.SavePage.PublishArchiveTime"), + add_page_start_time - publish_start_time); + OfflinePageItem page = offline_page; page.file_path = publish_results.new_file_path; page.system_download_id = publish_results.download_id; - AddPage(page, - base::BindOnce(&OfflinePageModelTaskified::OnAddPageForSavePageDone, - weak_ptr_factory_.GetWeakPtr(), - std::move(save_page_callback), page)); + AddPage(page, base::BindOnce( + &OfflinePageModelTaskified::OnAddPageForSavePageDone, + weak_ptr_factory_.GetWeakPtr(), + std::move(save_page_callback), page, add_page_start_time)); } void OfflinePageModelTaskified::PublishInternalArchive( @@ -619,6 +622,7 @@ void OfflinePageModelTaskified::PublishInternalArchiveDone( void OfflinePageModelTaskified::OnAddPageForSavePageDone( SavePageCallback callback, const OfflinePageItem& page_attempted, + base::Time add_page_start_time, AddPageResult add_page_result, int64_t offline_id) { SavePageResult save_page_result = @@ -626,7 +630,14 @@ void OfflinePageModelTaskified::OnAddPageForSavePageDone( InformSavePageDone(std::move(callback), save_page_result, page_attempted.client_id, offline_id); if (save_page_result == SavePageResult::SUCCESS) { - ReportPageHistogramAfterSuccessfulSaving(page_attempted, GetCurrentTime()); + base::Time successful_finish_time = GetCurrentTime(); + base::UmaHistogramTimes( + model_utils::AddHistogramSuffix(page_attempted.client_id.name_space, + "OfflinePages.SavePage.AddPageTime"), + successful_finish_time - add_page_start_time); + + ReportPageHistogramAfterSuccessfulSaving(page_attempted, + successful_finish_time); // TODO(romax): Just keep the same with logic in OPMImpl (which was wrong). // This should be fixed once we have the new strategy for clearing pages. if (policy_controller_->GetPolicy(page_attempted.client_id.name_space) @@ -654,16 +665,14 @@ void OfflinePageModelTaskified::OnDeleteDone( DeletePageCallback callback, DeletePageResult result, const std::vector<OfflinePageModel::DeletedPageInfo>& infos) { - UMA_HISTOGRAM_ENUMERATION("OfflinePages.DeletePageResult", result, - DeletePageResult::RESULT_COUNT); + UMA_HISTOGRAM_ENUMERATION("OfflinePages.DeletePageResult", result); std::vector<int64_t> system_download_ids; // Notify observers and run callback. for (const auto& info : infos) { UMA_HISTOGRAM_ENUMERATION( "OfflinePages.DeletePageCount", - model_utils::ToNamespaceEnum(info.client_id.name_space), - OfflinePagesNamespaceEnumeration::RESULT_COUNT); + model_utils::ToNamespaceEnum(info.client_id.name_space)); offline_event_logger_.RecordPageDeleted(info.offline_id); for (Observer& observer : observers_) observer.OfflinePageDeleted(info); @@ -716,7 +725,7 @@ void OfflinePageModelTaskified::ScheduleMaintenanceTasks() { last_maintenance_tasks_schedule_time_ = now; } -void OfflinePageModelTaskified::RunMaintenanceTasks(const base::Time now, +void OfflinePageModelTaskified::RunMaintenanceTasks(base::Time now, bool first_run) { DCHECK(!skip_maintenance_tasks_for_testing_); // If this is the first run of this session, enqueue the startup maintenance @@ -756,8 +765,7 @@ void OfflinePageModelTaskified::OnPersistentPageConsistencyCheckDone( void OfflinePageModelTaskified::OnClearCachedPagesDone( size_t deleted_page_count, ClearStorageResult result) { - UMA_HISTOGRAM_ENUMERATION("OfflinePages.ClearTemporaryPages.Result", result, - ClearStorageResult::RESULT_COUNT); + UMA_HISTOGRAM_ENUMERATION("OfflinePages.ClearTemporaryPages.Result", result); if (deleted_page_count > 0) { UMA_HISTOGRAM_COUNTS_1M("OfflinePages.ClearTemporaryPages.BatchSize", deleted_page_count); diff --git a/chromium/components/offline_pages/core/model/offline_page_model_taskified.h b/chromium/components/offline_pages/core/model/offline_page_model_taskified.h index 0d5dc55b2c4..1cfa4d30130 100644 --- a/chromium/components/offline_pages/core/model/offline_page_model_taskified.h +++ b/chromium/components/offline_pages/core/model/offline_page_model_taskified.h @@ -29,6 +29,8 @@ class GURL; namespace base { class FilePath; class SequencedTaskRunner; +class Time; +class TimeDelta; } // namespace base namespace offline_pages { @@ -158,11 +160,12 @@ class OfflinePageModelTaskified : public OfflinePageModel, int64_t offline_id); void OnAddPageForSavePageDone(SavePageCallback callback, const OfflinePageItem& page_attempted, + base::Time add_page_start_time, AddPageResult add_page_result, int64_t offline_id); void OnCreateArchiveDone(const SavePageParams& save_page_params, int64_t offline_id, - const base::Time& start_time, + base::Time start_time, std::unique_ptr<OfflinePageArchiver> archiver, SavePageCallback callback, OfflinePageArchiver::ArchiverResult archiver_result, @@ -189,7 +192,7 @@ class OfflinePageModelTaskified : public OfflinePageModel, // Methods for clearing temporary pages and performing consistency checks. The // latter are executed only once per Chrome session. void ScheduleMaintenanceTasks(); - void RunMaintenanceTasks(const base::Time now, bool first_run); + void RunMaintenanceTasks(base::Time now, bool first_run); void OnClearCachedPagesDone(size_t deleted_page_count, ClearStorageTask::ClearStorageResult result); void OnPersistentPageConsistencyCheckDone( @@ -205,6 +208,7 @@ class OfflinePageModelTaskified : public OfflinePageModel, // Callback for when PublishArchive has completd. void PublishArchiveDone(std::unique_ptr<OfflinePageArchiver> archiver, SavePageCallback save_page_callback, + base::Time publish_start_time, const OfflinePageItem& offline_page, PublishArchiveResult publish_results); diff --git a/chromium/components/offline_pages/core/model/offline_page_model_taskified_unittest.cc b/chromium/components/offline_pages/core/model/offline_page_model_taskified_unittest.cc index 4bfa2afcf5c..708f73dc5b5 100644 --- a/chromium/components/offline_pages/core/model/offline_page_model_taskified_unittest.cc +++ b/chromium/components/offline_pages/core/model/offline_page_model_taskified_unittest.cc @@ -63,6 +63,7 @@ const ClientId kTestClientId1(kDefaultNamespace, "1234"); const ClientId kTestClientId2(kDefaultNamespace, "5678"); const ClientId kTestUserRequestedClientId(kDownloadNamespace, "714"); const ClientId kTestBrowserActionsClientId(kBrowserActionsNamespace, "999"); +const ClientId kTestLastNClientId(kLastNNamespace, "8989"); const int64_t kTestFileSize = 876543LL; const base::string16 kTestTitle = base::UTF8ToUTF16("a title"); const char kTestRequestOrigin[] = "abc.xyz"; @@ -351,6 +352,7 @@ void OfflinePageModelTaskifiedTest::CheckTaskQueueIdle() { EXPECT_FALSE(task_queue()->HasRunningTask()); } +// Tests saving successfully a non-user-requested offline page. TEST_F(OfflinePageModelTaskifiedTest, SavePageSuccessful) { auto archiver = BuildArchiver(kTestUrl, ArchiverResult::SUCCESSFULLY_CREATED); int64_t offline_id = SavePageWithExpectedResult( @@ -388,10 +390,6 @@ TEST_F(OfflinePageModelTaskifiedTest, SavePageSuccessful) { "OfflinePages.PageSize"), kTestFileSize / 1024, 1); histogram_tester()->ExpectTotalCount( - model_utils::AddHistogramSuffix(kTestClientId1.name_space, - "OfflinePages.SavePageTime"), - 1); - histogram_tester()->ExpectTotalCount( "OfflinePages.StorageInfo.InternalFreeSpaceMiB", 1); histogram_tester()->ExpectTotalCount( "OfflinePages.StorageInfo.ExternalFreeSpaceMiB", 1); @@ -403,6 +401,21 @@ TEST_F(OfflinePageModelTaskifiedTest, SavePageSuccessful) { "OfflinePages.StorageInfo.InternalArchiveSizeMiB", 1); histogram_tester()->ExpectTotalCount( "OfflinePages.StorageInfo.ExternalArchiveSizeMiB", 1); + + // Performance metrics. + histogram_tester()->ExpectTotalCount( + model_utils::AddHistogramSuffix(kTestClientId1.name_space, + "OfflinePages.SavePageTime"), + 1); + histogram_tester()->ExpectTotalCount( + model_utils::AddHistogramSuffix( + kTestClientId1.name_space, + "OfflinePages.SavePage.PublishArchiveTime"), + 0); + histogram_tester()->ExpectUniqueSample( + model_utils::AddHistogramSuffix(kTestClientId1.name_space, + "OfflinePages.SavePage.AddPageTime"), + 0, 1); } TEST_F(OfflinePageModelTaskifiedTest, SavePageSuccessfulWithSameOriginalUrl) { @@ -1193,83 +1206,78 @@ TEST_F(OfflinePageModelTaskifiedTest, GetPagesSupportedByDownloads) { // This test is affected by https://crbug.com/725685, which only affects windows // platform. #if defined(OS_WIN) -#define MAYBE_CheckPagesSavedInSeparateDirsPrivate \ - DISABLED_CheckPagesSavedInSeparateDirsPrivate +#define MAYBE_CheckTempPagesSavedInCorrectDir \ + DISABLED_CheckTempPagesSavedInCorrectDir #else -#define MAYBE_CheckPagesSavedInSeparateDirsPrivate \ - CheckPagesSavedInSeparateDirsPrivate +#define MAYBE_CheckTempPagesSavedInCorrectDir CheckTempPagesSavedInCorrectDir #endif -TEST_F(OfflinePageModelTaskifiedTest, - MAYBE_CheckPagesSavedInSeparateDirsPrivate) { +TEST_F(OfflinePageModelTaskifiedTest, MAYBE_CheckTempPagesSavedInCorrectDir) { // Save a temporary page. auto archiver = BuildArchiver(kTestUrl, ArchiverResult::SUCCESSFULLY_CREATED); int64_t temporary_id = SavePageWithExpectedResult( - kTestUrl, kTestClientId1, GURL(), kEmptyRequestOrigin, - std::move(archiver), SavePageResult::SUCCESS); - - // Save a persistent page that will not be published - archiver = BuildArchiver(kTestUrl2, ArchiverResult::SUCCESSFULLY_CREATED); - int64_t persistent_id = SavePageWithExpectedResult( - kTestUrl2, kTestBrowserActionsClientId, GURL(), kEmptyRequestOrigin, + kTestUrl, kTestLastNClientId, GURL(), kEmptyRequestOrigin, std::move(archiver), SavePageResult::SUCCESS); std::unique_ptr<OfflinePageItem> temporary_page = store_test_util()->GetPageByOfflineId(temporary_id); - std::unique_ptr<OfflinePageItem> persistent_page = - store_test_util()->GetPageByOfflineId(persistent_id); - ASSERT_TRUE(temporary_page); - ASSERT_TRUE(persistent_page); - base::FilePath temporary_page_path = temporary_page->file_path; - base::FilePath persistent_page_path = persistent_page->file_path; + EXPECT_TRUE(temporary_dir_path().IsParent(temporary_page->file_path)); - EXPECT_TRUE(temporary_dir_path().IsParent(temporary_page_path)); - // For a page in the prefetch namespace, it gets moved to the - // a private internal directory inside chromium. - EXPECT_TRUE(private_archive_dir_path().IsParent(persistent_page_path)); - EXPECT_NE(temporary_page_path.DirName(), persistent_page_path.DirName()); + // Performance metrics. + histogram_tester()->ExpectTotalCount( + model_utils::AddHistogramSuffix(kTestLastNClientId.name_space, + "OfflinePages.SavePageTime"), + 1); + histogram_tester()->ExpectTotalCount( + model_utils::AddHistogramSuffix( + kTestLastNClientId.name_space, + "OfflinePages.SavePage.PublishArchiveTime"), + 0); + histogram_tester()->ExpectUniqueSample( + model_utils::AddHistogramSuffix(kTestLastNClientId.name_space, + "OfflinePages.SavePage.AddPageTime"), + 0, 1); } // This test is affected by https://crbug.com/725685, which only affects windows // platform. #if defined(OS_WIN) -#define MAYBE_CheckPagesSavedInSeparateDirsPublic \ - DISABLED_CheckPagesSavedInSeparateDirsPublic +#define MAYBE_CheckPersistenPagesSavedInCorrectDir \ + DISABLED_CheckPersistenPagesSavedInCorrectDir #else -#define MAYBE_CheckPagesSavedInSeparateDirsPublic \ - CheckPagesSavedInSeparateDirsPublic +#define MAYBE_CheckPersistenPagesSavedInCorrectDir \ + CheckPersistenPagesSavedInCorrectDir #endif TEST_F(OfflinePageModelTaskifiedTest, - MAYBE_CheckPagesSavedInSeparateDirsPublic) { - // Save a temporary page. - auto archiver = BuildArchiver(kTestUrl, ArchiverResult::SUCCESSFULLY_CREATED); - int64_t temporary_id = SavePageWithExpectedResult( - kTestUrl, kTestClientId1, GURL(), kEmptyRequestOrigin, - std::move(archiver), SavePageResult::SUCCESS); - - // Save a persistent page that will be published. - archiver = BuildArchiver(kTestUrl2, ArchiverResult::SUCCESSFULLY_CREATED); + MAYBE_CheckPersistenPagesSavedInCorrectDir) { + // Save a persistent page that will be published to the public folder. + auto archiver = + BuildArchiver(kTestUrl2, ArchiverResult::SUCCESSFULLY_CREATED); int64_t persistent_id = SavePageWithExpectedResult( kTestUrl2, kTestUserRequestedClientId, GURL(), kEmptyRequestOrigin, std::move(archiver), SavePageResult::SUCCESS); - std::unique_ptr<OfflinePageItem> temporary_page = - store_test_util()->GetPageByOfflineId(temporary_id); std::unique_ptr<OfflinePageItem> persistent_page = store_test_util()->GetPageByOfflineId(persistent_id); - - ASSERT_TRUE(temporary_page); ASSERT_TRUE(persistent_page); - base::FilePath temporary_page_path = temporary_page->file_path; - base::FilePath persistent_page_path = persistent_page->file_path; + EXPECT_TRUE(public_archive_dir_path().IsParent(persistent_page->file_path)); - EXPECT_TRUE(temporary_dir_path().IsParent(temporary_page_path)); - // TODO(petewil): It might be better to replace the check below with a check - // that the file ended up in the correct place instead of just not the wrong - // place. - EXPECT_NE(temporary_page_path.DirName(), persistent_page_path.DirName()); + // Performance metrics. + histogram_tester()->ExpectTotalCount( + model_utils::AddHistogramSuffix(kTestUserRequestedClientId.name_space, + "OfflinePages.SavePageTime"), + 1); + histogram_tester()->ExpectUniqueSample( + model_utils::AddHistogramSuffix( + kTestUserRequestedClientId.name_space, + "OfflinePages.SavePage.PublishArchiveTime"), + 0, 1); + histogram_tester()->ExpectUniqueSample( + model_utils::AddHistogramSuffix(kTestUserRequestedClientId.name_space, + "OfflinePages.SavePage.AddPageTime"), + 0, 1); } // This test is affected by https://crbug.com/725685, which only affects windows @@ -1311,14 +1319,11 @@ TEST_F(OfflinePageModelTaskifiedTest, MAYBE_CheckPublishInternalArchive) { std::unique_ptr<OfflinePageItem> persistent_page = store_test_util()->GetPageByOfflineId(persistent_id); - ASSERT_TRUE(persistent_page); - base::FilePath persistent_page_path = persistent_page->file_path; - // For a page in the browser actions namespace, it gets moved to the - // a private internal directory inside chromium. - EXPECT_TRUE(private_archive_dir_path().IsParent(persistent_page_path)); + // a public downloads directory. + EXPECT_TRUE(public_archive_dir_path().IsParent(persistent_page->file_path)); // Make another archiver, since SavePageWithExpectedResult deleted the first // one. diff --git a/chromium/components/offline_pages/core/model/offline_page_model_utils.cc b/chromium/components/offline_pages/core/model/offline_page_model_utils.cc index f6829d14d03..6785f1a4d55 100644 --- a/chromium/components/offline_pages/core/model/offline_page_model_utils.cc +++ b/chromium/components/offline_pages/core/model/offline_page_model_utils.cc @@ -43,6 +43,8 @@ OfflinePagesNamespaceEnumeration ToNamespaceEnum( return OfflinePagesNamespaceEnumeration::BROWSER_ACTIONS; else if (name_space == kLivePageSharingNamespace) return OfflinePagesNamespaceEnumeration::LIVE_PAGE_SHARING; + else if (name_space == kAutoAsyncNamespace) + return OfflinePagesNamespaceEnumeration::ASYNC_AUTO_LOADING; NOTREACHED(); return OfflinePagesNamespaceEnumeration::DEFAULT; diff --git a/chromium/components/offline_pages/core/model/persistent_page_consistency_check_task.cc b/chromium/components/offline_pages/core/model/persistent_page_consistency_check_task.cc index 40a697171ca..fe8f69fb771 100644 --- a/chromium/components/offline_pages/core/model/persistent_page_consistency_check_task.cc +++ b/chromium/components/offline_pages/core/model/persistent_page_consistency_check_task.cc @@ -233,8 +233,7 @@ void PersistentPageConsistencyCheckTask::Run() { void PersistentPageConsistencyCheckTask::OnPersistentPageConsistencyCheckDone( CheckResult check_result) { UMA_HISTOGRAM_ENUMERATION("OfflinePages.ConsistencyCheck.Persistent.Result", - check_result.result, - SyncOperationResult::RESULT_COUNT); + check_result.result); // If sync operation failed, invoke the callback with an empty list of // download ids. if (check_result.result != SyncOperationResult::SUCCESS) { diff --git a/chromium/components/offline_pages/core/model/start_offline_page_upgrade_task.cc b/chromium/components/offline_pages/core/model/start_offline_page_upgrade_task.cc index 2c71f87c40b..28cdaf678c1 100644 --- a/chromium/components/offline_pages/core/model/start_offline_page_upgrade_task.cc +++ b/chromium/components/offline_pages/core/model/start_offline_page_upgrade_task.cc @@ -6,7 +6,7 @@ #include "base/bind.h" #include "base/files/file_util.h" -#include "base/sys_info.h" +#include "base/system/sys_info.h" #include "components/offline_pages/core/offline_page_metadata_store.h" #include "components/offline_pages/core/offline_store_utils.h" #include "sql/database.h" diff --git a/chromium/components/offline_pages/core/model/startup_maintenance_task.cc b/chromium/components/offline_pages/core/model/startup_maintenance_task.cc index 0e1d961604b..e0a612e6e73 100644 --- a/chromium/components/offline_pages/core/model/startup_maintenance_task.cc +++ b/chromium/components/offline_pages/core/model/startup_maintenance_task.cc @@ -257,7 +257,7 @@ bool StartupMaintenanceSync( result = CheckTemporaryPageConsistencySync(db, temporary_namespaces, temporary_archives_dir); UMA_HISTOGRAM_ENUMERATION("OfflinePages.ConsistencyCheck.Temporary.Result", - result, SyncOperationResult::RESULT_COUNT); + result); // Report storage usage UMA, |temporary_namespaces| + |persistent_namespaces| // should be all namespaces. This is implicitly checked by the diff --git a/chromium/components/offline_pages/core/offline_clock.cc b/chromium/components/offline_pages/core/offline_clock.cc new file mode 100644 index 00000000000..bb4418446f5 --- /dev/null +++ b/chromium/components/offline_pages/core/offline_clock.cc @@ -0,0 +1,29 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/offline_clock.h" + +#include "base/logging.h" +#include "base/time/default_clock.h" + +namespace offline_pages { + +namespace { +base::Clock* custom_clock_ = nullptr; +} + +base::Clock* OfflineClock() { + if (custom_clock_) + return custom_clock_; + return base::DefaultClock::GetInstance(); +} + +void SetOfflineClockForTesting(base::Clock* clock) { + DCHECK(clock == nullptr || custom_clock_ == nullptr) + << "Offline clock is being overridden a second time, which might " + "indicate a bug."; + custom_clock_ = clock; +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/offline_clock.h b/chromium/components/offline_pages/core/offline_clock.h new file mode 100644 index 00000000000..5a15d6e090a --- /dev/null +++ b/chromium/components/offline_pages/core/offline_clock.h @@ -0,0 +1,24 @@ +// Copyright (c) 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_OFFLINE_CLOCK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_OFFLINE_CLOCK_H_ + +namespace base { +class Clock; +} + +namespace offline_pages { + +// Returns the clock to be used for obtaining the current time. This function +// can be called from any threads. +base::Clock* OfflineClock(); + +// Allows tests to override the clock returned by |OfflineClock()|. For safety, +// use |TestScopedOfflineClock| instead if possible. +void SetOfflineClockForTesting(base::Clock* clock); + +} // namespace offline_pages + +#endif // COMPONENTS_OFFLINE_PAGES_CORE_OFFLINE_CLOCK_H_ diff --git a/chromium/components/offline_pages/core/offline_page_archiver.cc b/chromium/components/offline_pages/core/offline_page_archiver.cc index 231fe1fd90c..6f49b2dbdab 100644 --- a/chromium/components/offline_pages/core/offline_page_archiver.cc +++ b/chromium/components/offline_pages/core/offline_page_archiver.cc @@ -11,6 +11,7 @@ #include "base/files/file_util.h" #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" +#include "base/sequenced_task_runner.h" #include "base/strings/utf_string_conversions.h" #include "base/task_runner_util.h" #include "components/offline_pages/core/model/offline_page_model_taskified.h" @@ -102,6 +103,10 @@ PublishArchiveResult MoveAndRegisterArchive( } // namespace +OfflinePageArchiver::CreateArchiveParams::CreateArchiveParams( + const std::string& name_space) + : name_space(name_space) {} + void OfflinePageArchiver::PublishArchive( const OfflinePageItem& offline_page, const scoped_refptr<base::SequencedTaskRunner>& background_task_runner, diff --git a/chromium/components/offline_pages/core/offline_page_archiver.h b/chromium/components/offline_pages/core/offline_page_archiver.h index 8bffda9bc28..b314fb21e40 100644 --- a/chromium/components/offline_pages/core/offline_page_archiver.h +++ b/chromium/components/offline_pages/core/offline_page_archiver.h @@ -80,16 +80,21 @@ class OfflinePageArchiver { // Describes the parameters to control how to create an archive. struct CreateArchiveParams { - CreateArchiveParams() - : remove_popup_overlay(false), use_page_problem_detectors(false) {} + explicit CreateArchiveParams(const std::string& name_space); + + // The offline page namespace associated with the archive to be created. + std::string name_space; // Whether to remove popup overlay that obstructs viewing normal content. - bool remove_popup_overlay; + bool remove_popup_overlay = false; // Run page problem detectors while generating MTHML if true. - bool use_page_problem_detectors; + bool use_page_problem_detectors = false; }; + // Callback for the final result of an attempt to generate of offline page + // archive. All parameters after |result| are only set in the case of a + // successful archive creation. using CreateArchiveCallback = base::OnceCallback<void(ArchiverResult /* result */, const GURL& /* url */, diff --git a/chromium/components/offline_pages/core/offline_page_client_policy.h b/chromium/components/offline_pages/core/offline_page_client_policy.h index de73ea2b0cc..08204a4b3e3 100644 --- a/chromium/components/offline_pages/core/offline_page_client_policy.h +++ b/chromium/components/offline_pages/core/offline_page_client_policy.h @@ -94,6 +94,10 @@ struct OfflinePageClientPolicy { FeaturePolicy feature_policy; + // Whether background fetches are deferred while the active tab matches the + // SavePageRequestURL. + bool defer_background_fetch_while_page_is_active = false; + OfflinePageClientPolicy(std::string namespace_val, LifetimePolicy lifetime_policy_val, size_t pages_allowed_per_url_val, @@ -184,6 +188,12 @@ class OfflinePageClientPolicyBuilder { return *this; } + OfflinePageClientPolicyBuilder& SetDeferBackgroundFetchWhilePageIsActive( + bool defer) { + policy_.defer_background_fetch_while_page_is_active = defer; + return *this; + } + private: OfflinePageClientPolicy policy_; diff --git a/chromium/components/offline_pages/core/offline_page_metadata_store.cc b/chromium/components/offline_pages/core/offline_page_metadata_store.cc index 4fd59e7db56..1e130ea722c 100644 --- a/chromium/components/offline_pages/core/offline_page_metadata_store.cc +++ b/chromium/components/offline_pages/core/offline_page_metadata_store.cc @@ -35,8 +35,7 @@ namespace { #define OFFLINE_PAGES_TABLE_NAME "offlinepages_v1" void ReportStoreEvent(OfflinePagesStoreEvent event) { - UMA_HISTOGRAM_ENUMERATION("OfflinePages.SQLStorage.StoreEvent", event, - OfflinePagesStoreEvent::STORE_EVENT_COUNT); + UMA_HISTOGRAM_ENUMERATION("OfflinePages.SQLStorage.StoreEvent", event); } bool CreateOfflinePagesTable(sql::Database* db) { @@ -410,14 +409,14 @@ void OfflinePageMetadataStore::InitializeInternal( DCHECK_EQ(state_, StoreState::NOT_LOADED); if (!last_closing_time_.is_null()) { - ReportStoreEvent(OfflinePagesStoreEvent::STORE_REOPENED); + ReportStoreEvent(OfflinePagesStoreEvent::kReopened); UMA_HISTOGRAM_CUSTOM_TIMES("OfflinePages.SQLStorage.TimeFromCloseToOpen", base::Time::Now() - last_closing_time_, base::TimeDelta::FromMilliseconds(10), base::TimeDelta::FromMinutes(10), 50 /* buckets */); } else { - ReportStoreEvent(OfflinePagesStoreEvent::STORE_OPENED_FIRST_TIME); + ReportStoreEvent(OfflinePagesStoreEvent::kOpenedFirstTime); } state_ = StoreState::INITIALIZING; @@ -462,13 +461,13 @@ void OfflinePageMetadataStore::OnInitializeInternalDone( void OfflinePageMetadataStore::CloseInternal() { if (state_ != StoreState::LOADED) { - ReportStoreEvent(OfflinePagesStoreEvent::STORE_CLOSE_SKIPPED); + ReportStoreEvent(OfflinePagesStoreEvent::kCloseSkipped); return; } TRACE_EVENT_ASYNC_STEP_PAST0("offline_pages", "Metadata Store", this, "Open"); last_closing_time_ = base::Time::Now(); - ReportStoreEvent(OfflinePagesStoreEvent::STORE_CLOSED); + ReportStoreEvent(OfflinePagesStoreEvent::kClosed); state_ = StoreState::NOT_LOADED; background_task_runner_->PostTask( diff --git a/chromium/components/offline_pages/core/offline_page_model.h b/chromium/components/offline_pages/core/offline_page_model.h index d6f9d428221..cdd0ded2619 100644 --- a/chromium/components/offline_pages/core/offline_page_model.h +++ b/chromium/components/offline_pages/core/offline_page_model.h @@ -19,8 +19,7 @@ #include "components/offline_pages/core/offline_page_archiver.h" #include "components/offline_pages/core/offline_page_thumbnail.h" #include "components/offline_pages/core/offline_page_types.h" - -class GURL; +#include "url/gurl.h" namespace offline_pages { diff --git a/chromium/components/offline_pages/core/offline_page_test_archiver.cc b/chromium/components/offline_pages/core/offline_page_test_archiver.cc index b87e12f5efb..4071ddab265 100644 --- a/chromium/components/offline_pages/core/offline_page_test_archiver.cc +++ b/chromium/components/offline_pages/core/offline_page_test_archiver.cc @@ -4,6 +4,9 @@ #include "components/offline_pages/core/offline_page_test_archiver.h" +#include <string> +#include <utility> + #include "base/bind.h" #include "base/files/file_util.h" #include "base/location.h" @@ -22,6 +25,7 @@ OfflinePageTestArchiver::OfflinePageTestArchiver( const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) : observer_(observer), url_(url), + create_archive_params_(std::string()), result_(result), size_to_report_(size_to_report), create_archive_called_(false), @@ -58,12 +62,14 @@ void OfflinePageTestArchiver::PublishArchive( PublishArchiveDoneCallback publish_done_callback) { publish_archive_called_ = true; PublishArchiveResult publish_archive_result; - publish_archive_result.move_result = SavePageResult::SUCCESS; - publish_archive_result.new_file_path = offline_page.file_path; - publish_archive_result.download_id = 0; if (archive_attempt_failure_) { publish_archive_result.move_result = SavePageResult::FILE_MOVE_FAILED; + } else { + publish_archive_result.move_result = SavePageResult::SUCCESS; + publish_archive_result.new_file_path = + new_file_path.Append(offline_page.file_path.BaseName()); + publish_archive_result.download_id = 0; } // Note: once the |publish_done_callback| is invoked it is very likely that diff --git a/chromium/components/offline_pages/core/offline_page_test_archiver.h b/chromium/components/offline_pages/core/offline_page_test_archiver.h index 6cae827ff5e..ce7a30ee8c1 100644 --- a/chromium/components/offline_pages/core/offline_page_test_archiver.h +++ b/chromium/components/offline_pages/core/offline_page_test_archiver.h @@ -19,7 +19,7 @@ class GURL; namespace base { class FilePath; -} // namespace +} // namespace base namespace offline_pages { diff --git a/chromium/components/offline_pages/core/offline_page_types.h b/chromium/components/offline_pages/core/offline_page_types.h index b655281fa7f..96879b9c436 100644 --- a/chromium/components/offline_pages/core/offline_page_types.h +++ b/chromium/components/offline_pages/core/offline_page_types.h @@ -20,7 +20,8 @@ class GURL; // temporary step to refactor and interface of the model out of the // implementation. namespace offline_pages { -// Result of saving a page offline. +// Result of saving a page offline. Must be kept with sync with +// OfflinePagesSavePageResult in metrics' enum.xml // A Java counterpart will be generated for this enum. // GENERATED_JAVA_ENUM_PACKAGE: org.chromium.components.offlinepages enum class SavePageResult { @@ -47,10 +48,7 @@ enum class SavePageResult { ADD_TO_DOWNLOAD_MANAGER_FAILED, // Unable to get write permission on public directory. PERMISSION_DENIED, - // NOTE: always keep this entry at the end. Add new result types only - // immediately above this line. Make sure to update the corresponding - // histogram enum accordingly. - RESULT_COUNT, + kMaxValue = PERMISSION_DENIED, }; // Result of adding an offline page. @@ -58,13 +56,11 @@ enum class AddPageResult { SUCCESS, STORE_FAILURE, ALREADY_EXISTS, - // NOTE: always keep this entry at the end. Add new result types only - // immediately above this line. Make sure to update the corresponding - // histogram enum accordingly. - RESULT_COUNT, + kMaxValue = ALREADY_EXISTS, }; -// Result of deleting an offline page. +// Result of deleting an offline page. Must be kept with sync with +// OfflinePagesDeletePageResult in metrics' enum.xml. // A Java counterpart will be generated for this enum. // GENERATED_JAVA_ENUM_PACKAGE: org.chromium.components.offlinepages enum class DeletePageResult { @@ -74,11 +70,8 @@ enum class DeletePageResult { DEVICE_FAILURE, // Deprecated. Deleting pages which are not in metadata store would be // returing |SUCCESS|. Should not be used anymore. - NOT_FOUND, - // NOTE: always keep this entry at the end. Add new result types only - // immediately above this line. Make sure to update the corresponding - // histogram enum accordingly. - RESULT_COUNT, + DEPRECATED_NOT_FOUND, + kMaxValue = DEPRECATED_NOT_FOUND, }; // The result when trying to share offline page to other apps. diff --git a/chromium/components/offline_pages/core/offline_store_types.h b/chromium/components/offline_pages/core/offline_store_types.h index 7a1faeb45f2..81b034881ab 100644 --- a/chromium/components/offline_pages/core/offline_store_types.h +++ b/chromium/components/offline_pages/core/offline_store_types.h @@ -44,10 +44,7 @@ enum class SyncOperationResult { TRANSACTION_COMMIT_ERROR, // Failed when commiting a DB transaction DB_OPERATION_ERROR, // Failed when executing a DB statement FILE_OPERATION_ERROR, // Failed while doing file operations - // NOTE: always keep this entry at the end. Add new result types only - // immediately above this line. Make sure to update the corresponding - // histogram enum accordingly. - RESULT_COUNT, + kMaxValue = FILE_OPERATION_ERROR, }; // List of item action statuses mapped to item ID. @@ -82,14 +79,12 @@ class StoreUpdateResult { // This enum is backed by a UMA histogram therefore its entries should not be // deleted or re-ordered and new ones should only be appended. // See enum definition with the same name in tools/metrics/histograms/enum.xml. -enum OfflinePagesStoreEvent { - STORE_OPENED_FIRST_TIME = 0, - STORE_REOPENED = 1, - STORE_CLOSED = 2, - STORE_CLOSE_SKIPPED = 3, - - // NOTE: always keep this entry at the end. - STORE_EVENT_COUNT +enum class OfflinePagesStoreEvent { + kOpenedFirstTime = 0, + kReopened = 1, + kClosed = 2, + kCloseSkipped = 3, + kMaxValue = kCloseSkipped, }; } // namespace offline_pages diff --git a/chromium/components/offline_pages/core/offline_store_utils.cc b/chromium/components/offline_pages/core/offline_store_utils.cc index 6064bc17e3a..46196def775 100644 --- a/chromium/components/offline_pages/core/offline_store_utils.cc +++ b/chromium/components/offline_pages/core/offline_store_utils.cc @@ -16,11 +16,12 @@ namespace offline_pages { namespace store_utils { int64_t ToDatabaseTime(base::Time time) { - return time.since_origin().InMicroseconds(); + return time.ToDeltaSinceWindowsEpoch().InMicroseconds(); } base::Time FromDatabaseTime(int64_t serialized_time) { - return base::Time() + base::TimeDelta::FromMicroseconds(serialized_time); + return base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(serialized_time)); } std::string ToDatabaseFilePath(const base::FilePath& file_path) { diff --git a/chromium/components/offline_pages/core/prefetch/BUILD.gn b/chromium/components/offline_pages/core/prefetch/BUILD.gn index c1449600380..39a0fe93e67 100644 --- a/chromium/components/offline_pages/core/prefetch/BUILD.gn +++ b/chromium/components/offline_pages/core/prefetch/BUILD.gn @@ -9,39 +9,11 @@ import("//third_party/protobuf/proto_library.gni") static_library("prefetch") { sources = [ - "add_unique_urls_task.cc", - "add_unique_urls_task.h", - "download_archives_task.cc", - "download_archives_task.h", - "download_cleanup_task.cc", - "download_cleanup_task.h", - "download_completed_task.cc", - "download_completed_task.h", - "finalize_dismissed_url_suggestion_task.cc", - "finalize_dismissed_url_suggestion_task.h", - "generate_page_bundle_reconcile_task.cc", - "generate_page_bundle_reconcile_task.h", "generate_page_bundle_request.cc", "generate_page_bundle_request.h", - "generate_page_bundle_task.cc", - "generate_page_bundle_task.h", "get_operation_request.cc", "get_operation_request.h", - "get_operation_task.cc", - "get_operation_task.h", - "import_archives_task.cc", - "import_archives_task.h", - "import_cleanup_task.cc", - "import_cleanup_task.h", - "import_completed_task.cc", - "import_completed_task.h", - "mark_operation_done_task.cc", - "mark_operation_done_task.h", - "metrics_finalization_task.cc", - "metrics_finalization_task.h", "offline_metrics_collector.h", - "page_bundle_update_task.cc", - "page_bundle_update_task.h", "prefetch_background_task.cc", "prefetch_background_task.h", "prefetch_background_task_handler.h", @@ -74,10 +46,6 @@ static_library("prefetch") { "prefetch_service_impl.h", "prefetch_types.cc", "prefetch_types.h", - "sent_get_operation_cleanup_task.cc", - "sent_get_operation_cleanup_task.h", - "stale_entry_finalizer_task.cc", - "stale_entry_finalizer_task.h", "store/prefetch_downloader_quota.cc", "store/prefetch_downloader_quota.h", "store/prefetch_store.cc", @@ -90,6 +58,42 @@ static_library("prefetch") { "suggested_articles_observer.h", "suggestions_provider.cc", "suggestions_provider.h", + "tasks/add_unique_urls_task.cc", + "tasks/add_unique_urls_task.h", + "tasks/download_archives_task.cc", + "tasks/download_archives_task.h", + "tasks/download_cleanup_task.cc", + "tasks/download_cleanup_task.h", + "tasks/download_completed_task.cc", + "tasks/download_completed_task.h", + "tasks/finalize_dismissed_url_suggestion_task.cc", + "tasks/finalize_dismissed_url_suggestion_task.h", + "tasks/generate_page_bundle_reconcile_task.cc", + "tasks/generate_page_bundle_reconcile_task.h", + "tasks/generate_page_bundle_task.cc", + "tasks/generate_page_bundle_task.h", + "tasks/get_operation_task.cc", + "tasks/get_operation_task.h", + "tasks/get_thumbnail_info_task.cc", + "tasks/get_thumbnail_info_task.h", + "tasks/import_archives_task.cc", + "tasks/import_archives_task.h", + "tasks/import_cleanup_task.cc", + "tasks/import_cleanup_task.h", + "tasks/import_completed_task.cc", + "tasks/import_completed_task.h", + "tasks/mark_operation_done_task.cc", + "tasks/mark_operation_done_task.h", + "tasks/metrics_finalization_task.cc", + "tasks/metrics_finalization_task.h", + "tasks/page_bundle_update_task.cc", + "tasks/page_bundle_update_task.h", + "tasks/sent_get_operation_cleanup_task.cc", + "tasks/sent_get_operation_cleanup_task.h", + "tasks/stale_entry_finalizer_task.cc", + "tasks/stale_entry_finalizer_task.h", + "thumbnail_fetch_by_url.cc", + "thumbnail_fetch_by_url.h", "thumbnail_fetcher.h", ] @@ -102,6 +106,7 @@ static_library("prefetch") { "//components/download/public/background_service:public", "//components/gcm_driver", "//components/gcm_driver/common", + "//components/image_fetcher/core", "//components/keyed_service/core", "//components/ntp_snippets", "//components/offline_pages/core", @@ -133,12 +138,12 @@ static_library("test_support") { "prefetch_request_test_base.h", "prefetch_service_test_taco.cc", "prefetch_service_test_taco.h", - "prefetch_task_test_base.cc", - "prefetch_task_test_base.h", "store/prefetch_store_test_util.cc", "store/prefetch_store_test_util.h", "stub_prefetch_service.cc", "stub_prefetch_service.h", + "tasks/prefetch_task_test_base.cc", + "tasks/prefetch_task_test_base.h", "test_download_client.cc", "test_download_client.h", "test_download_service.cc", @@ -163,6 +168,7 @@ static_library("test_support") { "//components/download/public/background_service:public", "//components/download/public/background_service/test:test_support", "//components/gcm_driver/instance_id", + "//components/image_fetcher/core:test_support", "//components/keyed_service/core", "//components/offline_pages/core", "//components/offline_pages/core:switches", @@ -203,22 +209,8 @@ if (is_android) { source_set("unit_tests") { testonly = true sources = [ - "add_unique_urls_task_unittest.cc", - "download_archives_task_unittest.cc", - "download_cleanup_task_unittest.cc", - "download_completed_task_unittest.cc", - "finalize_dismissed_url_suggestion_task_unittest.cc", - "generate_page_bundle_reconcile_task_unittest.cc", "generate_page_bundle_request_unittest.cc", - "generate_page_bundle_task_unittest.cc", "get_operation_request_unittest.cc", - "get_operation_task_unittest.cc", - "import_archives_task_unittest.cc", - "import_cleanup_task_unittest.cc", - "import_completed_task_unittest.cc", - "mark_operation_done_task_unittest.cc", - "metrics_finalization_task_unittest.cc", - "page_bundle_update_task_unittest.cc", "prefetch_dispatcher_impl_unittest.cc", "prefetch_download_flow_unittest.cc", "prefetch_downloader_impl_unittest.cc", @@ -229,13 +221,28 @@ source_set("unit_tests") { "prefetch_request_fetcher_unittest.cc", "prefetch_request_operation_response_unittest.cc", "prefetch_server_urls_unittest.cc", - "prefetch_task_test_base_unittest.cc", - "sent_get_operation_cleanup_task_unittest.cc", - "stale_entry_finalizer_task_unittest.cc", "store/prefetch_downloader_quota_unittest.cc", "store/prefetch_store_schema_unittest.cc", "store/prefetch_store_unittest.cc", "suggested_articles_observer_unittest.cc", + "tasks/add_unique_urls_task_unittest.cc", + "tasks/download_archives_task_unittest.cc", + "tasks/download_cleanup_task_unittest.cc", + "tasks/download_completed_task_unittest.cc", + "tasks/finalize_dismissed_url_suggestion_task_unittest.cc", + "tasks/generate_page_bundle_reconcile_task_unittest.cc", + "tasks/generate_page_bundle_task_unittest.cc", + "tasks/get_operation_task_unittest.cc", + "tasks/get_thumbnail_info_task_unittest.cc", + "tasks/import_archives_task_unittest.cc", + "tasks/import_cleanup_task_unittest.cc", + "tasks/import_completed_task_unittest.cc", + "tasks/mark_operation_done_task_unittest.cc", + "tasks/metrics_finalization_task_unittest.cc", + "tasks/page_bundle_update_task_unittest.cc", + "tasks/prefetch_task_test_base_unittest.cc", + "tasks/sent_get_operation_cleanup_task_unittest.cc", + "tasks/stale_entry_finalizer_task_unittest.cc", ] deps = [ @@ -245,6 +252,7 @@ source_set("unit_tests") { "//components/download/public/background_service:public", "//components/download/public/background_service/test:test_support", "//components/gcm_driver/instance_id", + "//components/image_fetcher/core:test_support", "//components/offline_pages/core", "//components/offline_pages/core:switches", "//components/offline_pages/core:test_support", diff --git a/chromium/components/offline_pages/core/prefetch/DEPS b/chromium/components/offline_pages/core/prefetch/DEPS index 3df3ed2737e..25d8791efa6 100644 --- a/chromium/components/offline_pages/core/prefetch/DEPS +++ b/chromium/components/offline_pages/core/prefetch/DEPS @@ -1,12 +1,13 @@ include_rules = [ "+components/download/internal/test", "+components/download/public", - "+components/prefs", - "+google_apis", - "+components/variations", "+components/gcm_driver", + "+components/image_fetcher", "+components/ntp_snippets", + "+components/prefs", + "+components/variations", "+components/version_info", + "+google_apis", "+net", "+services/network/public/cpp", "+services/network/test", diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.cc b/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.cc index e6f651249ac..acdff5b1e0c 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.cc @@ -17,21 +17,7 @@ #include "components/offline_pages/core/offline_event_logger.h" #include "components/offline_pages/core/offline_page_feature.h" #include "components/offline_pages/core/offline_page_model.h" -#include "components/offline_pages/core/prefetch/add_unique_urls_task.h" -#include "components/offline_pages/core/prefetch/download_archives_task.h" -#include "components/offline_pages/core/prefetch/download_cleanup_task.h" -#include "components/offline_pages/core/prefetch/download_completed_task.h" -#include "components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.h" -#include "components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.h" -#include "components/offline_pages/core/prefetch/generate_page_bundle_task.h" -#include "components/offline_pages/core/prefetch/get_operation_task.h" -#include "components/offline_pages/core/prefetch/import_archives_task.h" -#include "components/offline_pages/core/prefetch/import_cleanup_task.h" -#include "components/offline_pages/core/prefetch/import_completed_task.h" -#include "components/offline_pages/core/prefetch/mark_operation_done_task.h" -#include "components/offline_pages/core/prefetch/metrics_finalization_task.h" #include "components/offline_pages/core/prefetch/offline_metrics_collector.h" -#include "components/offline_pages/core/prefetch/page_bundle_update_task.h" #include "components/offline_pages/core/prefetch/prefetch_background_task.h" #include "components/offline_pages/core/prefetch/prefetch_background_task_handler.h" #include "components/offline_pages/core/prefetch/prefetch_downloader.h" @@ -41,10 +27,25 @@ #include "components/offline_pages/core/prefetch/prefetch_prefs.h" #include "components/offline_pages/core/prefetch/prefetch_service.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" -#include "components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.h" -#include "components/offline_pages/core/prefetch/stale_entry_finalizer_task.h" #include "components/offline_pages/core/prefetch/suggested_articles_observer.h" #include "components/offline_pages/core/prefetch/suggestions_provider.h" +#include "components/offline_pages/core/prefetch/tasks/add_unique_urls_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_archives_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_completed_task.h" +#include "components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.h" +#include "components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.h" +#include "components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.h" +#include "components/offline_pages/core/prefetch/tasks/get_operation_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_archives_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_completed_task.h" +#include "components/offline_pages/core/prefetch/tasks/mark_operation_done_task.h" +#include "components/offline_pages/core/prefetch/tasks/metrics_finalization_task.h" +#include "components/offline_pages/core/prefetch/tasks/page_bundle_update_task.h" +#include "components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.h" +#include "components/offline_pages/core/prefetch/thumbnail_fetch_by_url.h" #include "components/offline_pages/core/prefetch/thumbnail_fetcher.h" #include "components/prefs/pref_service.h" #include "url/gurl.h" @@ -58,8 +59,10 @@ void DeleteBackgroundTaskHelper(std::unique_ptr<PrefetchBackgroundTask> task) { } PrefetchURL SuggestionToPrefetchURL(PrefetchSuggestion suggestion) { - return PrefetchURL(suggestion.article_url.spec(), suggestion.article_url, + PrefetchURL result(suggestion.article_url.spec(), suggestion.article_url, base::UTF8ToUTF16(suggestion.article_title)); + result.thumbnail_url = suggestion.thumbnail_url; + return result; } } // namespace @@ -125,8 +128,8 @@ void PrefetchDispatcherImpl::NewSuggestionsAvailable( SuggestionsProvider* suggestions_provider) { if (!prefetch_prefs::IsEnabled(pref_service_)) return; - suggestions_provider->GetCurrentArticleSuggestions(base::BindOnce( - &PrefetchDispatcherImpl::AddSuggestions, weak_factory_.GetWeakPtr())); + suggestions_provider->GetCurrentArticleSuggestions( + base::BindOnce(&PrefetchDispatcherImpl::AddSuggestions, GetWeakPtr())); } void PrefetchDispatcherImpl::RemoveSuggestion(const GURL& url) { @@ -233,7 +236,7 @@ void PrefetchDispatcherImpl::QueueActionTasks() { service_->GetPrefetchNetworkRequestFactory(), base::BindOnce( &PrefetchDispatcherImpl::DidGenerateBundleOrGetOperationRequest, - weak_factory_.GetWeakPtr(), "GetOperationRequest")); + GetWeakPtr(), "GetOperationRequest")); task_queue_.AddTask(std::move(get_operation_task)); std::unique_ptr<Task> generate_page_bundle_task = @@ -242,7 +245,7 @@ void PrefetchDispatcherImpl::QueueActionTasks() { service_->GetPrefetchNetworkRequestFactory(), base::BindOnce( &PrefetchDispatcherImpl::DidGenerateBundleOrGetOperationRequest, - weak_factory_.GetWeakPtr(), "GeneratePageBundleRequest")); + GetWeakPtr(), "GeneratePageBundleRequest")); task_queue_.AddTask(std::move(generate_page_bundle_task)); } @@ -428,11 +431,6 @@ void PrefetchDispatcherImpl::FetchThumbnails( if (remaining_ids->empty()) return; - // Zine/Feed - // TODO(https://crbug.com/841516): Implement thumbnail fetching with the Feed. - if (!service_->GetThumbnailFetcher()) - return; - int64_t offline_id = remaining_ids->back().first; ClientId client_id = std::move(remaining_ids->back().second); DCHECK(client_id.name_space == kSuggestedArticlesNamespace); @@ -441,7 +439,7 @@ void PrefetchDispatcherImpl::FetchThumbnails( service_->GetOfflinePageModel()->HasThumbnailForOfflineId( offline_id, base::BindOnce(&PrefetchDispatcherImpl::ThumbnailExistenceChecked, - base::Unretained(this), offline_id, std::move(client_id), + GetWeakPtr(), offline_id, std::move(client_id), std::move(remaining_ids), is_first_attempt)); } @@ -454,12 +452,38 @@ void PrefetchDispatcherImpl::ThumbnailExistenceChecked( if (thumbnail_exists) { FetchThumbnails(std::move(remaining_ids), is_first_attempt); } else { - auto complete_callback = base::BindOnce( - &PrefetchDispatcherImpl::ThumbnailFetchComplete, base::Unretained(this), - offline_id, std::move(remaining_ids), is_first_attempt); - service_->GetThumbnailFetcher()->FetchSuggestionImageData( - client_id, is_first_attempt, std::move(complete_callback)); + // Zine/Feed: thumbnail_fetcher is non-null only with Zine. + ThumbnailFetcher* thumbnail_fetcher = service_->GetThumbnailFetcher(); + if (thumbnail_fetcher) { + auto complete_callback = base::BindOnce( + &PrefetchDispatcherImpl::ThumbnailFetchComplete, GetWeakPtr(), + offline_id, std::move(remaining_ids), is_first_attempt); + thumbnail_fetcher->FetchSuggestionImageData(client_id, is_first_attempt, + std::move(complete_callback)); + } else { + task_queue_.AddTask(std::make_unique<GetThumbnailInfoTask>( + service_->GetPrefetchStore(), offline_id, + base::BindOnce(&PrefetchDispatcherImpl::ThumbnailInfoReceived, + GetWeakPtr(), offline_id, std::move(remaining_ids), + is_first_attempt))); + } + } +} + +void PrefetchDispatcherImpl::ThumbnailInfoReceived( + const int64_t offline_id, + std::unique_ptr<IdsVector> remaining_ids, + bool is_first_attempt, + GetThumbnailInfoTask::Result result) { + if (result.thumbnail_url.is_empty()) { + FetchThumbnails(std::move(remaining_ids), is_first_attempt); + return; // No thumbnail url was given to us for this page. } + FetchThumbnailByURL( + base::BindOnce(&PrefetchDispatcherImpl::ThumbnailFetchComplete, + GetWeakPtr(), offline_id, std::move(remaining_ids), + is_first_attempt), + service_->GetThumbnailImageFetcher(), result.thumbnail_url); } void PrefetchDispatcherImpl::ThumbnailFetchComplete( diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.h b/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.h index 28987a3583d..b2d08d99a37 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.h +++ b/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl.h @@ -15,6 +15,8 @@ #include "base/macros.h" #include "base/memory/weak_ptr.h" #include "components/offline_pages/core/prefetch/prefetch_dispatcher.h" +#include "components/offline_pages/core/prefetch/suggestions_provider.h" +#include "components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.h" #include "components/offline_pages/task/task_queue.h" #include "components/version_info/channel.h" #include "net/url_request/url_request_context_getter.h" @@ -64,6 +66,10 @@ class PrefetchDispatcherImpl : public PrefetchDispatcher, private: friend class PrefetchDispatcherTest; + base::WeakPtr<PrefetchDispatcherImpl> GetWeakPtr() { + return weak_factory_.GetWeakPtr(); + } + void DisposeTask(); // Callbacks for network requests. @@ -112,6 +118,10 @@ class PrefetchDispatcherImpl : public PrefetchDispatcher, std::unique_ptr<IdsVector> remaining_ids, bool is_first_attempt, bool thumbnail_exists); + void ThumbnailInfoReceived(const int64_t offline_id, + std::unique_ptr<IdsVector> remaining_ids, + bool is_first_attempt, + GetThumbnailInfoTask::Result result); void ThumbnailFetchComplete(const int64_t offline_id, std::unique_ptr<IdsVector> remaining_ids, bool is_first_attempt, diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl_unittest.cc b/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl_unittest.cc index 3aba74c0601..492e9980d2a 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_dispatcher_impl_unittest.cc @@ -4,6 +4,7 @@ #include "components/offline_pages/core/prefetch/prefetch_dispatcher_impl.h" +#include <set> #include <utility> #include "base/files/file_util.h" @@ -11,12 +12,15 @@ #include "base/memory/ptr_util.h" #include "base/strings/utf_string_conversions.h" #include "base/test/scoped_feature_list.h" +#include "components/image_fetcher/core/mock_image_fetcher.h" +#include "components/image_fetcher/core/request_metadata.h" #include "components/offline_pages/core/client_namespace_constants.h" #include "components/offline_pages/core/offline_event_logger.h" #include "components/offline_pages/core/offline_page_feature.h" #include "components/offline_pages/core/prefetch/fake_suggestions_provider.h" #include "components/offline_pages/core/prefetch/generate_page_bundle_request.h" #include "components/offline_pages/core/prefetch/get_operation_request.h" +#include "components/offline_pages/core/prefetch/mock_prefetch_item_generator.h" #include "components/offline_pages/core/prefetch/mock_thumbnail_fetcher.h" #include "components/offline_pages/core/prefetch/prefetch_background_task.h" #include "components/offline_pages/core/prefetch/prefetch_importer_impl.h" @@ -59,6 +63,8 @@ const char kOperationName[] = "operation-1"; const char kBodyName[] = "body-1"; const int64_t kBodyLength = 10; const char kBodyContent[] = "abcde12345"; +const char kThumbnailUrl[] = "http://www.thumbnail.com/"; +const char kThumbnailData[] = "thumbnail_data"; const base::Time kRenderTime = base::Time::Now(); PrefetchSuggestion TestSuggestion1() { @@ -72,6 +78,52 @@ PrefetchSuggestion TestSuggestion1() { return suggestion; } +PrefetchSuggestion TestSuggestion2() { + PrefetchSuggestion suggestion; + suggestion.article_url = kTestURL2; + suggestion.article_title = "Second Title"; + suggestion.article_attribution = "From fun.com"; + suggestion.article_snippet = "More fun stuff"; + suggestion.thumbnail_url = GURL("http://google.com/funthumbnail"); + suggestion.favicon_url = GURL("http://fun.com/favicon"); + return suggestion; +} + +PrefetchSuggestion TestSuggestion3() { + PrefetchSuggestion suggestion; + suggestion.article_url = GURL("http://www.google.com/3"); + suggestion.article_title = "Third Title"; + suggestion.article_attribution = "From google.com"; + suggestion.article_snippet = "I'm feeling lucky"; + suggestion.thumbnail_url = GURL("http://google.com/googlethumbnail"); + suggestion.favicon_url = GURL("http://google.com/favicon"); + return suggestion; +} + +PrefetchSuggestion TestSuggestion4() { + PrefetchSuggestion suggestion; + suggestion.article_url = GURL("http://www.four.com"); + suggestion.article_title = "Fourth title"; + suggestion.article_attribution = "From four.com"; + suggestion.article_snippet = "I'm four"; + suggestion.thumbnail_url = GURL("http://google.com/fourthumbnail"); + suggestion.favicon_url = GURL("http://four.com/favicon"); + return suggestion; +} + +ClientId SuggestionClientId(const PrefetchSuggestion& suggestion) { + return {kSuggestedArticlesNamespace, suggestion.article_url.spec()}; +} + +const PrefetchItem* FindByUrl(const std::set<PrefetchItem>& items, + const GURL& url) { + for (const auto& item : items) { + if (item.url == url) + return &item; + } + return nullptr; +} + RenderPageInfo RenderInfo(const std::string& url) { RenderPageInfo info; info.url = url; @@ -83,18 +135,64 @@ RenderPageInfo RenderInfo(const std::string& url) { return info; } +OfflinePageThumbnail FakeThumbnail(int64_t offline_id) { + return OfflinePageThumbnail(offline_id, kRenderTime, kThumbnailData); +} + +// This class is a mix between a mock and fake. class MockOfflinePageModel : public StubOfflinePageModel { public: explicit MockOfflinePageModel(const base::FilePath& archive_directory) { SetArchiveDirectory(archive_directory); } ~MockOfflinePageModel() override = default; - MOCK_METHOD1(StoreThumbnail, void(const OfflinePageThumbnail& thumb)); - MOCK_METHOD2(HasThumbnailForOfflineId, - void(int64_t offline_id, - base::OnceCallback<void(bool)> callback)); + + // OfflinePageModel implementation. + MOCK_METHOD2(AddPage, void(const OfflinePageItem& page, AddPageCallback callback)); + + void StoreThumbnail(const OfflinePageThumbnail& thumb) override { + thumbnails_.insert(thumb); + } + + void HasThumbnailForOfflineId( + int64_t offline_id, + base::OnceCallback<void(bool)> callback) override { + has_thumbnail_for_offline_id_calls_.insert(offline_id); + bool found = false; + for (const OfflinePageThumbnail& thumbnail : thumbnails_) { + if (thumbnail.offline_id == offline_id) + found = true; + } + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), found)); + } + + // Returns the thumbnails stored with StoreThumbnail. + const std::set<OfflinePageThumbnail>& thumbnails() const { + return thumbnails_; + } + + const OfflinePageThumbnail* FindThumbnail(int64_t offline_id) const { + for (const auto& thumbnail : thumbnails_) { + if (thumbnail.offline_id == offline_id) + return &thumbnail; + } + return nullptr; + } + + void set_thumbnails(std::set<OfflinePageThumbnail> thumbnails) { + thumbnails_ = std::move(thumbnails); + } + + const std::set<int64_t>& has_thumbnail_for_offline_id_calls() const { + return has_thumbnail_for_offline_id_calls_; + } + + private: + std::set<OfflinePageThumbnail> thumbnails_; + std::set<int64_t> has_thumbnail_for_offline_id_calls_; }; class TestPrefetchBackgroundTask : public PrefetchBackgroundTask { @@ -165,8 +263,6 @@ class PrefetchDispatcherTest : public PrefetchRequestTestBase { void TearDown() override; // Configures the fixture for the test. Must be called before each test. - // feed_configuration=true configures the fixture as if Feed were the - // suggestion source. Otherwise, it's configured as if Zine was the source. // Zine/Feed TODO(harringtond): Collapse this method with SetUp() after Zine // code is removed. void Configure(PrefetchServiceTestTaco::SuggestionSource suggestion_source) { @@ -183,8 +279,9 @@ class PrefetchDispatcherTest : public PrefetchRequestTestBase { taco_->SetPrefetchNetworkRequestFactory( base::WrapUnique(network_request_factory_)); if (suggestion_source == PrefetchServiceTestTaco::kFeed) { - taco_->SetThumbnailFetcher(std::unique_ptr<MockThumbnailFetcher>()); - + auto image_fetcher = std::make_unique<image_fetcher::MockImageFetcher>(); + thumbnail_image_fetcher_ = image_fetcher.get(); + taco_->SetThumbnailImageFetcher(std::move(image_fetcher)); } else { auto thumbnail_fetcher = std::make_unique<MockThumbnailFetcher>(); thumbnail_fetcher_ = thumbnail_fetcher.get(); @@ -261,14 +358,19 @@ class PrefetchDispatcherTest : public PrefetchRequestTestBase { }); } - void ExpectHasThumbnailForOfflineId(int64_t offline_id, bool to_return) { - EXPECT_CALL(*offline_model_, HasThumbnailForOfflineId(offline_id, _)) - .WillOnce( - [&, offline_id, to_return]( - int64_t offline_id, base::OnceCallback<void(bool)> callback) { - task_runner()->PostTask( - FROM_HERE, base::BindOnce(std::move(callback), to_return)); - }); + void ExpectFetchThumbnailImage(const std::string& thumbnail_data, + const GURL& thumbnail_url) { + ASSERT_TRUE(thumbnail_image_fetcher_) << "Not configured in kFeed mode"; + EXPECT_CALL(*thumbnail_image_fetcher_, + FetchImageAndData_(std::string(), thumbnail_url, _, _, _)) + .WillOnce([=](const std::string& id, const GURL& image_url, + image_fetcher::ImageDataFetcherCallback* data_callback, + image_fetcher::ImageFetcherCallback* image_callback, + const net::NetworkTrafficAnnotationTag&) { + ASSERT_TRUE(image_callback->is_null()); + std::move(*data_callback) + .Run(thumbnail_data, image_fetcher::RequestMetadata()); + }); } PrefetchDispatcherImpl* dispatcher() { return dispatcher_; } @@ -283,8 +385,10 @@ class PrefetchDispatcherTest : public PrefetchRequestTestBase { // Owned by |taco_|, may be null. MockThumbnailFetcher* thumbnail_fetcher_; + image_fetcher::MockImageFetcher* thumbnail_image_fetcher_; PrefetchStoreTestUtil store_util_{task_runner()}; + MockPrefetchItemGenerator item_generator_; base::ScopedTempDir archive_directory_; TestingPrefServiceSimple prefs_; std::unique_ptr<FakeSuggestionsProvider> suggestions_provider_; @@ -631,6 +735,64 @@ TEST_F(PrefetchDispatcherTest, ZineNoNetworkRequestsAfterNewURLs) { EXPECT_EQ(nullptr, GetPendingRequest()); } +TEST_F(PrefetchDispatcherTest, ThumbnailImageFetchFailure_ItemDownloaded) { + Configure(PrefetchServiceTestTaco::kFeed); + suggestions_provider_->SetSuggestions({TestSuggestion1()}); + + PrefetchItem item = item_generator_.CreateItem(PrefetchItemState::DOWNLOADED); + item.thumbnail_url = GURL(kThumbnailUrl); + item.client_id.id = kClientID; + item.offline_id = kTestOfflineID; + store_util_.InsertPrefetchItem(item); + + ExpectFetchThumbnailImage("", GURL(kThumbnailUrl)); + prefetch_dispatcher()->ItemDownloaded( + kTestOfflineID, ClientId(kSuggestedArticlesNamespace, kClientID)); + RunUntilIdle(); + + EXPECT_TRUE(offline_model_->thumbnails().empty()) + << "Stored thumbnails: " + << ::testing::PrintToString(offline_model_->thumbnails()); +} + +// Test attempting to fetch several suggested article thumbnails. This verifies +// that multiple fetches are attempted. +TEST_F(PrefetchDispatcherTest, ThumbnailImageFetch_SeveralThumbnailDownloads) { + Configure(PrefetchServiceTestTaco::kFeed); + // Suggestion 1: No thumbnail fetch because there is no thumbnail_url. + testing::InSequence sequence; + PrefetchSuggestion suggestion1 = TestSuggestion1(); + suggestion1.thumbnail_url = GURL(); + // Suggestion 2: Thumbnail fetch fails. + const PrefetchSuggestion suggestion2 = TestSuggestion2(); + ExpectFetchThumbnailImage("", suggestion2.thumbnail_url); + // Suggestions 3&4: Successful thumbnail fetch. + const PrefetchSuggestion suggestion3 = TestSuggestion3(); + ExpectFetchThumbnailImage(kThumbnailData, suggestion3.thumbnail_url); + const PrefetchSuggestion suggestion4 = TestSuggestion4(); + ExpectFetchThumbnailImage(kThumbnailData, suggestion4.thumbnail_url); + std::vector<PrefetchSuggestion> suggestions = {suggestion1, suggestion2, + suggestion3, suggestion4}; + suggestions_provider_->SetSuggestions(suggestions); + + prefetch_service()->NewSuggestionsAvailable(); + RunUntilIdle(); + + // Pull out the items to find the OfflineIDs, and continue processing those + // IDs. + std::set<PrefetchItem> items; + store_util_.GetAllItems(&items); + auto generate_ids = std::make_unique<PrefetchDispatcher::IdsVector>(); + for (const auto& suggestion : suggestions) { + const PrefetchItem* item = FindByUrl(items, suggestion.article_url); + ASSERT_TRUE(item) << " item url=" << suggestion.article_url; + generate_ids->push_back( + std::make_pair(item->offline_id, SuggestionClientId(suggestion))); + } + prefetch_dispatcher()->GeneratePageBundleRequested(std::move(generate_ids)); + RunUntilIdle(); +} + TEST_F(PrefetchDispatcherTest, FeedNoNetworkRequestsAfterNewURLs) { Configure(PrefetchServiceTestTaco::kFeed); suggestions_provider_->SetSuggestions({TestSuggestion1()}); @@ -647,31 +809,36 @@ TEST_F(PrefetchDispatcherTest, ThumbnailFetchFailure_ItemDownloaded) { Configure(PrefetchServiceTestTaco::kContentSuggestions); ExpectFetchThumbnail("", false, kClientID); - ExpectHasThumbnailForOfflineId(kTestOfflineID, false); - EXPECT_CALL(*offline_model_, StoreThumbnail(_)).Times(0); prefetch_dispatcher()->ItemDownloaded( kTestOfflineID, ClientId(kSuggestedArticlesNamespace, kClientID)); + + EXPECT_TRUE(offline_model_->thumbnails().empty()) + << "Stored thumbnails: " + << ::testing::PrintToString(offline_model_->thumbnails()); } TEST_F(PrefetchDispatcherTest, ThumbnailFetchSuccess_ItemDownloaded) { Configure(PrefetchServiceTestTaco::kContentSuggestions); - std::string kThumbnailData = "abc"; - ExpectHasThumbnailForOfflineId(kTestOfflineID, false); - EXPECT_CALL(*offline_model_, StoreThumbnail(ValidThumbnail())); ExpectFetchThumbnail(kThumbnailData, false, kClientID); prefetch_dispatcher()->ItemDownloaded( kTestOfflineID, ClientId(kSuggestedArticlesNamespace, kClientID)); + RunUntilIdle(); + + const OfflinePageThumbnail* stored_thumbnail = + offline_model_->FindThumbnail(kTestOfflineID); + ASSERT_TRUE(stored_thumbnail); + EXPECT_EQ(kThumbnailData, stored_thumbnail->thumbnail); } TEST_F(PrefetchDispatcherTest, ThumbnailAlreadyExists_ItemDownloaded) { Configure(PrefetchServiceTestTaco::kContentSuggestions); - ExpectHasThumbnailForOfflineId(kTestOfflineID, true); + offline_model_->set_thumbnails({FakeThumbnail(kTestOfflineID)}); EXPECT_CALL(*thumbnail_fetcher_, FetchSuggestionImageData(_, _, _)).Times(0); - EXPECT_CALL(*offline_model_, StoreThumbnail(_)).Times(0); prefetch_dispatcher()->ItemDownloaded( kTestOfflineID, ClientId(kSuggestedArticlesNamespace, kClientID)); + RunUntilIdle(); } TEST_F(PrefetchDispatcherTest, @@ -690,14 +857,11 @@ TEST_F(PrefetchDispatcherTest, InSequence in_sequence; // Case #1. - ExpectHasThumbnailForOfflineId(kTestOfflineID1, false); - ExpectFetchThumbnail("abc", true, kClientID1); - EXPECT_CALL(*offline_model_, StoreThumbnail(_)).Times(1); + ExpectFetchThumbnail(kThumbnailData, true, kClientID1); // Case #2. - ExpectHasThumbnailForOfflineId(kTestOfflineID2, false); ExpectFetchThumbnail("", true, kClientID2); - // Case #3. - ExpectHasThumbnailForOfflineId(kTestOfflineID3, true); + // Case #3: thumbnail already exists + offline_model_->set_thumbnails({FakeThumbnail(kTestOfflineID3)}); auto prefetch_item_ids = std::make_unique<PrefetchDispatcher::IdsVector>(); prefetch_item_ids->emplace_back( @@ -708,6 +872,12 @@ TEST_F(PrefetchDispatcherTest, kTestOfflineID3, ClientId(kSuggestedArticlesNamespace, kClientID3)); prefetch_dispatcher()->GeneratePageBundleRequested( std::move(prefetch_item_ids)); + RunUntilIdle(); + + EXPECT_TRUE(offline_model_->FindThumbnail(kTestOfflineID1)) + << "Thumbnails: " + << ::testing::PrintToString(offline_model_->thumbnails()); + EXPECT_FALSE(offline_model_->FindThumbnail(kTestOfflineID2)); } // Runs through the entire lifecycle of a successful prefetch item, diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_download_flow_unittest.cc b/chromium/components/offline_pages/core/prefetch/prefetch_download_flow_unittest.cc index 529a5fdc076..5fee6f1cdf9 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_download_flow_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_download_flow_unittest.cc @@ -15,7 +15,7 @@ #include "components/offline_pages/core/prefetch/prefetch_dispatcher_impl.h" #include "components/offline_pages/core/prefetch/prefetch_service.h" #include "components/offline_pages/core/prefetch/prefetch_service_test_taco.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_download_client.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_downloader_impl.cc b/chromium/components/offline_pages/core/prefetch/prefetch_downloader_impl.cc index cd62c3af2fb..ded2ef757f8 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_downloader_impl.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_downloader_impl.cc @@ -96,10 +96,9 @@ void PrefetchDownloaderImpl::StartDownload(const std::string& download_id, policy { cookies_allowed: NO setting: - "Users can enable or disable the offline prefetch on desktop by " - "toggling 'Use a prediction service to load pages more quickly' in " - "settings under Privacy and security, or on Android by toggling " - "chrome://flags#offline-prefetch." + "Users can enable or disable offline prefetch by toggling " + "'Download articles for you' in settings under Downloads or " + "by toggling chrome://flags#offline-prefetch." chrome_policy { NetworkPredictionOptions { NetworkPredictionOptions: 2 diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_importer_impl.cc b/chromium/components/offline_pages/core/prefetch/prefetch_importer_impl.cc index 7ecf97b4e5a..069b3a0c4a1 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_importer_impl.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_importer_impl.cc @@ -28,7 +28,7 @@ enum class PageImportResult { OFFLINE_STORE_FAILURE = 3, OFFLINE_ITEM_ALREADY_EXISTS = 4, // Always leave this item last. Update if the actual last item changes. - MAX = OFFLINE_ITEM_ALREADY_EXISTS + kMaxValue = OFFLINE_ITEM_ALREADY_EXISTS }; PageImportResult FromAddPageResult(AddPageResult result) { @@ -39,8 +39,6 @@ PageImportResult FromAddPageResult(AddPageResult result) { return PageImportResult::OFFLINE_STORE_FAILURE; case AddPageResult::ALREADY_EXISTS: return PageImportResult::OFFLINE_ITEM_ALREADY_EXISTS; - case AddPageResult::RESULT_COUNT: - NOTREACHED(); } NOTREACHED(); return PageImportResult::UNKNOWN; @@ -48,7 +46,7 @@ PageImportResult FromAddPageResult(AddPageResult result) { void ReportPageImportResult(PageImportResult result) { UMA_HISTOGRAM_ENUMERATION("OfflinePages.Prefetching.OfflinePageImportResult", - result, PageImportResult::MAX); + result); } void MoveFile(const base::FilePath& src_path, diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_item.h b/chromium/components/offline_pages/core/prefetch/prefetch_item.h index e45066eb8a8..0a7c340de63 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_item.h +++ b/chromium/components/offline_pages/core/prefetch/prefetch_item.h @@ -22,6 +22,8 @@ namespace offline_pages { // successfully or not. // Instances of this class are in-memory representations of items in (or to be // inserted into) the persistent prefetching data store. +// +// Only used in tests. struct PrefetchItem { PrefetchItem(); PrefetchItem(PrefetchItem&& other); @@ -61,6 +63,18 @@ struct PrefetchItem { // left empty if they are the same. GURL final_archived_url; + // The URL to the thumbnail image representing the article. + GURL thumbnail_url; + + // The URL to the favicon image of the article's hosting web site. + GURL favicon_url; + + // A snippet of the article's contents. + std::string snippet; + + // The publisher name/web site the article is attributed to. + std::string attribution; + // Number of attempts to request OPS to generate an archive for this item. int generate_bundle_attempts = 0; @@ -91,8 +105,10 @@ struct PrefetchItem { // item. It holds a negative value otherwise. int64_t archive_body_length = -1; - // Time when this item was inserted into the store with the URL to be - // prefetched. + // The last time the URL was attempted to be added to the store. Normally this + // is just the time the item was added. If the same URL is added multiple + // times, this is the timestamp of the last time. creation_time + // is used as a proxy for priority. base::Time creation_time; // Time used for the expiration of the item depending on the applicable policy diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_item_unittest.cc b/chromium/components/offline_pages/core/prefetch/prefetch_item_unittest.cc index 7a9e6ca57b7..8e01d2d01b9 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_item_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_item_unittest.cc @@ -97,6 +97,18 @@ TEST_F(PrefetchItemTest, OperatorEqualsCopyConstructorAndToString) { item1.final_archived_url = GURL("http://test.com/final"); CheckFieldAndResetItem(item1, "final_archived_url"); + item1.thumbnail_url = GURL("http://thumbnail"); + CheckFieldAndResetItem(item1, "thumbnail_url"); + + item1.favicon_url = GURL("http://favicon"); + CheckFieldAndResetItem(item1, "favicon_url"); + + item1.snippet = "snippet"; + CheckFieldAndResetItem(item1, "snippet"); + + item1.attribution = "attribution"; + CheckFieldAndResetItem(item1, "attribution"); + item1.generate_bundle_attempts = 10; CheckFieldAndResetItem(item1, "generate_bundle_attempts"); diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_request_fetcher.cc b/chromium/components/offline_pages/core/prefetch/prefetch_request_fetcher.cc index 63d2c995689..841aab93f0f 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_request_fetcher.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_request_fetcher.cc @@ -68,8 +68,9 @@ PrefetchRequestFetcher::PrefetchRequestFetcher( policy { cookies_allowed: NO setting: - "Users can enable or disable the offline prefetch by toggling" - "chrome://flags#offline-prefetch in Chromium on Android." + "Users can enable or disable offline prefetch by toggling " + "'Download articles for you' in settings under Downloads or " + "by toggling chrome://flags#offline-prefetch." policy_exception_justification: "Not implemented, considered not useful." })"); diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_service.h b/chromium/components/offline_pages/core/prefetch/prefetch_service.h index 477ead7e8cf..42abcfd4d56 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_service.h +++ b/chromium/components/offline_pages/core/prefetch/prefetch_service.h @@ -9,6 +9,9 @@ class GURL; +namespace image_fetcher { +class ImageFetcher; +} namespace ntp_snippets { class ContentSuggestionsService; } @@ -92,6 +95,7 @@ class PrefetchService : public KeyedService { // Zine/Feed: Null when using the Feed. virtual ThumbnailFetcher* GetThumbnailFetcher() = 0; virtual OfflinePageModel* GetOfflinePageModel() = 0; + virtual image_fetcher::ImageFetcher* GetThumbnailImageFetcher() = 0; // Test-only methods. diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.cc b/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.cc index ab1cf99a615..c15fba58576 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.cc @@ -8,6 +8,7 @@ #include "base/bind.h" #include "base/logging.h" +#include "components/image_fetcher/core/image_fetcher.h" #include "components/offline_pages/core/client_id.h" #include "components/offline_pages/core/client_namespace_constants.h" #include "components/offline_pages/core/prefetch/offline_metrics_collector.h" @@ -36,7 +37,8 @@ PrefetchServiceImpl::PrefetchServiceImpl( std::unique_ptr<PrefetchImporter> prefetch_importer, std::unique_ptr<PrefetchBackgroundTaskHandler> prefetch_background_task_handler, - std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher) + std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher, + std::unique_ptr<image_fetcher::ImageFetcher> thumbnail_image_fetcher) : offline_metrics_collector_(std::move(offline_metrics_collector)), prefetch_dispatcher_(std::move(dispatcher)), prefetch_gcm_handler_(std::move(gcm_handler)), @@ -48,7 +50,8 @@ PrefetchServiceImpl::PrefetchServiceImpl( prefetch_background_task_handler_( std::move(prefetch_background_task_handler)), suggested_articles_observer_(std::move(suggested_articles_observer)), - thumbnail_fetcher_(std::move(thumbnail_fetcher)) { + thumbnail_fetcher_(std::move(thumbnail_fetcher)), + thumbnail_image_fetcher_(std::move(thumbnail_image_fetcher)) { prefetch_dispatcher_->SetService(this); prefetch_downloader_->SetPrefetchService(this); prefetch_gcm_handler_->SetService(this); @@ -73,6 +76,7 @@ void PrefetchServiceImpl::SetContentSuggestionsService( DCHECK(suggested_articles_observer_); DCHECK(!suggestions_provider_); DCHECK(thumbnail_fetcher_); + DCHECK(!thumbnail_image_fetcher_); suggested_articles_observer_->SetContentSuggestionsServiceAndObserve( content_suggestions); thumbnail_fetcher_->SetContentSuggestionsService(content_suggestions); @@ -82,6 +86,7 @@ void PrefetchServiceImpl::SetSuggestionProvider( SuggestionsProvider* suggestions_provider) { DCHECK(!suggested_articles_observer_); DCHECK(!thumbnail_fetcher_); + DCHECK(thumbnail_image_fetcher_); suggestions_provider_ = suggestions_provider; } @@ -146,6 +151,10 @@ ThumbnailFetcher* PrefetchServiceImpl::GetThumbnailFetcher() { return thumbnail_fetcher_.get(); } +image_fetcher::ImageFetcher* PrefetchServiceImpl::GetThumbnailImageFetcher() { + return thumbnail_image_fetcher_.get(); +} + void PrefetchServiceImpl::Shutdown() { suggested_articles_observer_.reset(); prefetch_downloader_.reset(); diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.h b/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.h index c72fcce77e6..93fa2ef8519 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.h +++ b/chromium/components/offline_pages/core/prefetch/prefetch_service_impl.h @@ -30,7 +30,8 @@ class PrefetchServiceImpl : public PrefetchService { std::unique_ptr<PrefetchDownloader> prefetch_downloader, std::unique_ptr<PrefetchImporter> prefetch_importer, std::unique_ptr<PrefetchBackgroundTaskHandler> background_task_handler, - std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher); + std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher, + std::unique_ptr<image_fetcher::ImageFetcher> thumbnail_image_fetcher_); ~PrefetchServiceImpl() override; @@ -54,9 +55,14 @@ class PrefetchServiceImpl : public PrefetchService { PrefetchDownloader* GetPrefetchDownloader() override; PrefetchImporter* GetPrefetchImporter() override; PrefetchBackgroundTaskHandler* GetPrefetchBackgroundTaskHandler() override; + + // Thumbnail fetchers. With Feed, GetThumbnailImageFetcher() is available + // and GetThumbnailFetcher() is null. ThumbnailFetcher* GetThumbnailFetcher() override; + image_fetcher::ImageFetcher* GetThumbnailImageFetcher() override; SuggestedArticlesObserver* GetSuggestedArticlesObserverForTesting() override; + // KeyedService implementation: void Shutdown() override; @@ -77,6 +83,7 @@ class PrefetchServiceImpl : public PrefetchService { // Zine/Feed: only non-null when using Zine. std::unique_ptr<SuggestedArticlesObserver> suggested_articles_observer_; std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher_; + std::unique_ptr<image_fetcher::ImageFetcher> thumbnail_image_fetcher_; // Zine/Feed: only non-null when using Feed. SuggestionsProvider* suggestions_provider_ = nullptr; diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.cc b/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.cc index d3a323f5e65..9b16e55c652 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.cc @@ -9,6 +9,8 @@ #include "base/macros.h" #include "base/memory/ptr_util.h" #include "base/threading/thread_task_runner_handle.h" +#include "components/image_fetcher/core/image_fetcher.h" +#include "components/image_fetcher/core/mock_image_fetcher.h" #include "components/offline_pages/core/offline_page_model.h" #include "components/offline_pages/core/prefetch/mock_thumbnail_fetcher.h" #include "components/offline_pages/core/prefetch/offline_metrics_collector.h" @@ -79,6 +81,9 @@ PrefetchServiceTestTaco::PrefetchServiceTestTaco(SuggestionSource source) { // result here. This allows us to not create a ContentSuggestionsService. suggested_articles_observer_->GetTestingArticles(); thumbnail_fetcher_ = std::make_unique<MockThumbnailFetcher>(); + } else { + thumbnail_image_fetcher_ = + std::make_unique<image_fetcher::MockImageFetcher>(); } prefetch_background_task_handler_ = @@ -156,6 +161,12 @@ void PrefetchServiceTestTaco::SetThumbnailFetcher( thumbnail_fetcher_ = std::move(thumbnail_fetcher); } +void PrefetchServiceTestTaco::SetThumbnailImageFetcher( + std::unique_ptr<image_fetcher::ImageFetcher> thumbnail_image_fetcher) { + CHECK(!prefetch_service_); + thumbnail_image_fetcher_ = std::move(thumbnail_image_fetcher); +} + void PrefetchServiceTestTaco::SetOfflinePageModel( std::unique_ptr<OfflinePageModel> offline_page_model) { CHECK(!prefetch_service_); @@ -171,7 +182,7 @@ void PrefetchServiceTestTaco::CreatePrefetchService() { std::move(suggested_articles_observer_), std::move(prefetch_downloader_), std::move(prefetch_importer_), std::move(prefetch_background_task_handler_), - std::move(thumbnail_fetcher_)); + std::move(thumbnail_fetcher_), std::move(thumbnail_image_fetcher_)); } std::unique_ptr<PrefetchService> diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.h b/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.h index 1b9e98bd71b..bc64980fadb 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.h +++ b/chromium/components/offline_pages/core/prefetch/prefetch_service_test_taco.h @@ -11,6 +11,10 @@ #include "base/memory/ref_counted.h" #include "base/threading/thread_task_runner_handle.h" +namespace image_fetcher { +class ImageFetcher; +} + namespace offline_pages { class OfflineMetricsCollector; class OfflinePageModel; @@ -71,6 +75,9 @@ class PrefetchServiceTestTaco { prefetch_background_task_handler); // Default type: MockThumbnailFetcher. void SetThumbnailFetcher(std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher); + // Default type: image_fetcher::MockImageFetcher. + void SetThumbnailImageFetcher( + std::unique_ptr<image_fetcher::ImageFetcher> thumbnail_image_fetcher); void SetOfflinePageModel( std::unique_ptr<OfflinePageModel> offline_page_model); @@ -104,6 +111,7 @@ class PrefetchServiceTestTaco { prefetch_background_task_handler_; std::unique_ptr<PrefetchService> prefetch_service_; std::unique_ptr<ThumbnailFetcher> thumbnail_fetcher_; + std::unique_ptr<image_fetcher::ImageFetcher> thumbnail_image_fetcher_; std::unique_ptr<OfflinePageModel> offline_page_model_; std::unique_ptr<TestDownloadService> download_service_; std::unique_ptr<TestDownloadClient> download_client_; diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_types.cc b/chromium/components/offline_pages/core/prefetch/prefetch_types.cc index 98777749fd0..5fc940c39c1 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_types.cc +++ b/chromium/components/offline_pages/core/prefetch/prefetch_types.cc @@ -135,6 +135,15 @@ RenderPageInfo::RenderPageInfo() = default; RenderPageInfo::RenderPageInfo(const RenderPageInfo& other) = default; +PrefetchURL::PrefetchURL(const std::string& id, + const GURL& url, + const base::string16& title) + : id(id), url(url), title(title) {} + +PrefetchURL::~PrefetchURL() = default; + +PrefetchURL::PrefetchURL(const PrefetchURL& other) = default; + PrefetchDownloadResult::PrefetchDownloadResult() = default; PrefetchDownloadResult::PrefetchDownloadResult(const std::string& download_id, diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_types.h b/chromium/components/offline_pages/core/prefetch/prefetch_types.h index 65d3e3ea535..1147d7b332f 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_types.h +++ b/chromium/components/offline_pages/core/prefetch/prefetch_types.h @@ -137,7 +137,7 @@ enum class PrefetchItemState { // client. ZOMBIE = 100, // Max item state, needed for histograms - MAX = ZOMBIE + kMaxValue = ZOMBIE }; // Error codes used to identify the reason why a prefetch entry has finished @@ -147,7 +147,7 @@ enum class PrefetchItemState { // New entries can be added anywhere as long as they are assigned unique values // and kept in ascending order. Deprecated entries should be labeled as such but // never removed. Assigned values should never be reused. Remember to update the -// MAX value if adding a new trailing item. +// kMaxValue value if adding a new trailing item. // // Changes to this enum must be reflected in the respective metrics enum named // OflinePrefetchItemErrorCode in enums.xml. Use the exact same integer value @@ -182,8 +182,8 @@ enum class PrefetchItemErrorCode { STALE_AT_DOWNLOADING = 1000, STALE_AT_IMPORTING = 1050, STALE_AT_UNKNOWN = 1100, - // The item was terminated due to not being concluded after being more than 7 - // days in the pipeline. + // The item was terminated because it remained in the pipeline for more than + // 7 days. STUCK = 1150, // Exceeded maximum retries for get operation request. GET_OPERATION_MAX_ATTEMPTS_REACHED = 1200, @@ -200,7 +200,7 @@ enum class PrefetchItemErrorCode { // downloaded. SUGGESTION_INVALIDATED = 1700, // Note: Must always have the same value as the last actual entry. - MAX = SUGGESTION_INVALIDATED + kMaxValue = SUGGESTION_INVALIDATED }; // Callback invoked upon completion of a prefetch request. @@ -213,8 +213,9 @@ using PrefetchRequestFinishedCallback = struct PrefetchURL { PrefetchURL(const std::string& id, const GURL& url, - const base::string16& title) - : id(id), url(url), title(title) {} + const base::string16& title); + ~PrefetchURL(); + PrefetchURL(const PrefetchURL& other); // Client provided ID to allow the matching of provided URLs to the respective // work item in the prefetching system within that client's assigned @@ -227,6 +228,10 @@ struct PrefetchURL { // The title of the page. base::string16 title; + + // URL for a thumbnail that represents the page. May be empty if no thumbnail + // is available. + GURL thumbnail_url; }; // Result of a completed download. diff --git a/chromium/components/offline_pages/core/prefetch/store/README.md b/chromium/components/offline_pages/core/prefetch/store/README.md index 60821cc78fa..dc2e4163d2d 100644 --- a/chromium/components/offline_pages/core/prefetch/store/README.md +++ b/chromium/components/offline_pages/core/prefetch/store/README.md @@ -146,3 +146,30 @@ Furthermore, when adding or removing columns, any existing column ordering might not be kept. This means that any query must not presume column ordering and must always explicitly refer to them by name. Using <code>SELECT * FROM ...</code> for obtaining data in all columns is therefore *unsafe and forbidden*. + +## Schema History / Test Data + +The components/test/data/offline_pages/prefetch/version_schemas directory contains +data used in testing the prefetch database schema, +see prefetch_store_schema_unittest.cc for the tests. In this directory, there +are two files for every version of the prefetch database schema: + +- v#.sql + +SQL that creates the database schema for this version and inserts rows in each +table for testing. This defines the initial state for each migration test. Data +inserted here should attempt to cover edge cases specific to that version (like +a change in a default value). + +- v#.data + +Represents the expected result of running the initial state defined in the .sql +file through the migration logic up to the current version of the schema. +Or in pseudo-code: + + migrated_db = MigrateToCurrentSchema(BuildDbFromSqlFile(old_version_sql_file)); + EXPECT_EQ(GetDataFromDataFile(old_version_data_file), + GetDataFromDb(migrated_db)); + +Whenever a new version is created, existing .data might need to be updated to +account for the added migration step. diff --git a/chromium/components/offline_pages/core/prefetch/store/prefetch_store.cc b/chromium/components/offline_pages/core/prefetch/store/prefetch_store.cc index be6090c00cf..7f724003aee 100644 --- a/chromium/components/offline_pages/core/prefetch/store/prefetch_store.cc +++ b/chromium/components/offline_pages/core/prefetch/store/prefetch_store.cc @@ -76,8 +76,7 @@ void CloseDatabaseSync( } void ReportStoreEvent(OfflinePagesStoreEvent event) { - UMA_HISTOGRAM_ENUMERATION("OfflinePages.PrefetchStore.StoreEvent", event, - OfflinePagesStoreEvent::STORE_EVENT_COUNT); + UMA_HISTOGRAM_ENUMERATION("OfflinePages.PrefetchStore.StoreEvent", event); } } // namespace @@ -114,14 +113,14 @@ void PrefetchStore::Initialize(base::OnceClosure pending_command) { initialization_status_ = InitializationStatus::INITIALIZING; if (!last_closing_time_.is_null()) { - ReportStoreEvent(OfflinePagesStoreEvent::STORE_REOPENED); + ReportStoreEvent(OfflinePagesStoreEvent::kReopened); UMA_HISTOGRAM_CUSTOM_TIMES("OfflinePages.PrefetchStore.TimeFromCloseToOpen", base::Time::Now() - last_closing_time_, base::TimeDelta::FromMilliseconds(10), base::TimeDelta::FromMinutes(10), 50 /* buckets */); } else { - ReportStoreEvent(OfflinePagesStoreEvent::STORE_OPENED_FIRST_TIME); + ReportStoreEvent(OfflinePagesStoreEvent::kOpenedFirstTime); } // This is how we reset a pointer and provide deleter. This is necessary to @@ -163,13 +162,13 @@ void PrefetchStore::OnInitializeDone(base::OnceClosure pending_command, void PrefetchStore::CloseInternal() { if (initialization_status_ != InitializationStatus::SUCCESS) { - ReportStoreEvent(OfflinePagesStoreEvent::STORE_CLOSE_SKIPPED); + ReportStoreEvent(OfflinePagesStoreEvent::kCloseSkipped); return; } TRACE_EVENT_ASYNC_STEP_PAST0("offline_pages", "Prefetch Store", this, "Open"); last_closing_time_ = base::Time::Now(); - ReportStoreEvent(OfflinePagesStoreEvent::STORE_CLOSED); + ReportStoreEvent(OfflinePagesStoreEvent::kClosed); initialization_status_ = InitializationStatus::NOT_INITIALIZED; blocking_task_runner_->PostTask( diff --git a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.cc b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.cc index d58053eb9eb..8e0479848dd 100644 --- a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.cc +++ b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.cc @@ -15,11 +15,12 @@ namespace offline_pages { // tables. // * 2: Changes prefetch_items.file_size to have a default value of -1 (instead // of 0). +// * 3: Add thumbnail_url, favicon_url, snippet, and attribution. // static -const int PrefetchStoreSchema::kCurrentVersion = 2; +constexpr int PrefetchStoreSchema::kCurrentVersion; // static -const int PrefetchStoreSchema::kCompatibleVersion = 1; +constexpr int PrefetchStoreSchema::kCompatibleVersion; namespace { @@ -58,47 +59,54 @@ int GetCompatibleVersionNumber(sql::MetaTable* meta_table) { // IMPORTANT #1: when making changes to these columns please also reflect them // into: // - PrefetchItem: update existing fields and all method implementations -// (operator=, operator<<, ToString, etc). +// (operator==, ToString, etc). // - PrefetchItemTest, PrefetchStoreTestUtil: update test related code to cover // the changed set of columns and PrefetchItem members. // - MockPrefetchItemGenerator: so that its generated items consider all fields. -// IMPORTANT #2: the ordering of column types is important in SQLite 3 tables to -// simplify data retrieval. Columns with fixed length types must come first and -// variable length types must come later. +// IMPORTANT #2: Commonly used columns should appear first, as SQLite can stop +// reading the row early if later columns are not being read. static const char kItemsTableCreationSql[] = - "CREATE TABLE IF NOT EXISTS prefetch_items " // Fixed length columns come first. - "(offline_id INTEGER PRIMARY KEY NOT NULL," - " state INTEGER NOT NULL DEFAULT 0," - " generate_bundle_attempts INTEGER NOT NULL DEFAULT 0," - " get_operation_attempts INTEGER NOT NULL DEFAULT 0," - " download_initiation_attempts INTEGER NOT NULL DEFAULT 0," - " archive_body_length INTEGER_NOT_NULL DEFAULT -1," - " creation_time INTEGER NOT NULL," - " freshness_time INTEGER NOT NULL," - " error_code INTEGER NOT NULL DEFAULT 0," - " file_size INTEGER NOT NULL DEFAULT -1," - // Variable length columns come later. - " guid VARCHAR NOT NULL DEFAULT ''," - " client_namespace VARCHAR NOT NULL DEFAULT ''," - " client_id VARCHAR NOT NULL DEFAULT ''," - " requested_url VARCHAR NOT NULL DEFAULT ''," - " final_archived_url VARCHAR NOT NULL DEFAULT ''," - " operation_name VARCHAR NOT NULL DEFAULT ''," - " archive_body_name VARCHAR NOT NULL DEFAULT ''," - " title VARCHAR NOT NULL DEFAULT ''," - " file_path VARCHAR NOT NULL DEFAULT ''" - ")"; + R"sql( +CREATE TABLE IF NOT EXISTS prefetch_items( +offline_id INTEGER PRIMARY KEY NOT NULL, +state INTEGER NOT NULL DEFAULT 0, +generate_bundle_attempts INTEGER NOT NULL DEFAULT 0, +get_operation_attempts INTEGER NOT NULL DEFAULT 0, +download_initiation_attempts INTEGER NOT NULL DEFAULT 0, +archive_body_length INTEGER_NOT_NULL DEFAULT -1, +creation_time INTEGER NOT NULL, +freshness_time INTEGER NOT NULL, +error_code INTEGER NOT NULL DEFAULT 0, +file_size INTEGER NOT NULL DEFAULT -1, +guid VARCHAR NOT NULL DEFAULT '', +client_namespace VARCHAR NOT NULL DEFAULT '', +client_id VARCHAR NOT NULL DEFAULT '', +requested_url VARCHAR NOT NULL DEFAULT '', +final_archived_url VARCHAR NOT NULL DEFAULT '', +operation_name VARCHAR NOT NULL DEFAULT '', +archive_body_name VARCHAR NOT NULL DEFAULT '', +title VARCHAR NOT NULL DEFAULT '', +file_path VARCHAR NOT NULL DEFAULT '', +thumbnail_url VARCHAR NOT NULL DEFAULT '', +favicon_url VARCHAR NOT NULL DEFAULT '', +snippet VARCHAR NOT NULL DEFAULT '', +attribution VARCHAR NOT NULL DEFAULT '' +) +)sql"; bool CreatePrefetchItemsTable(sql::Database* db) { return db->Execute(kItemsTableCreationSql); } static const char kQuotaTableCreationSql[] = - "CREATE TABLE IF NOT EXISTS prefetch_downloader_quota " - "(quota_id INTEGER PRIMARY KEY NOT NULL DEFAULT 1," - " update_time INTEGER NOT NULL," - " available_quota INTEGER NOT NULL DEFAULT 0)"; + R"sql( +CREATE TABLE IF NOT EXISTS prefetch_downloader_quota( +quota_id INTEGER PRIMARY KEY NOT NULL DEFAULT 1, +update_time INTEGER NOT NULL, +available_quota INTEGER NOT NULL DEFAULT 0 +) +)sql"; bool CreatePrefetchQuotaTable(sql::Database* db) { return db->Execute(kQuotaTableCreationSql); @@ -117,49 +125,57 @@ bool CreateLatestSchema(sql::Database* db) { } int MigrateFromVersion1To2(sql::Database* db, sql::MetaTable* meta_table) { + // Version 2 simply changes the default value of file_size from 0 to -1. + // Because SQLite doesn't support removing or modifying columns, we create + // a new table and insert data from the old table. const int target_version = 2; const int target_compatible_version = 1; + // 1. Rename the existing items table. + // 2. Create the new items table. + // 3. Copy existing rows to the new items table. + // 4. Drop the old items table. static const char kVersion1ToVersion2MigrationSql[] = - // Rename the existing items table. - "ALTER TABLE prefetch_items RENAME TO prefetch_items_old; " - // Creates the new items table. - "CREATE TABLE prefetch_items " - "(offline_id INTEGER PRIMARY KEY NOT NULL," - " state INTEGER NOT NULL DEFAULT 0," - " generate_bundle_attempts INTEGER NOT NULL DEFAULT 0," - " get_operation_attempts INTEGER NOT NULL DEFAULT 0," - " download_initiation_attempts INTEGER NOT NULL DEFAULT 0," - " archive_body_length INTEGER_NOT_NULL DEFAULT -1," - " creation_time INTEGER NOT NULL," - " freshness_time INTEGER NOT NULL," - " error_code INTEGER NOT NULL DEFAULT 0," - // Note: default value changed from 0 to -1. - " file_size INTEGER NOT NULL DEFAULT -1," - " guid VARCHAR NOT NULL DEFAULT ''," - " client_namespace VARCHAR NOT NULL DEFAULT ''," - " client_id VARCHAR NOT NULL DEFAULT ''," - " requested_url VARCHAR NOT NULL DEFAULT ''," - " final_archived_url VARCHAR NOT NULL DEFAULT ''," - " operation_name VARCHAR NOT NULL DEFAULT ''," - " archive_body_name VARCHAR NOT NULL DEFAULT ''," - " title VARCHAR NOT NULL DEFAULT ''," - " file_path VARCHAR NOT NULL DEFAULT ''); " - // Copy existing rows to the new items table. - "INSERT INTO prefetch_items " - " (offline_id, state, generate_bundle_attempts, get_operation_attempts," - " download_initiation_attempts, archive_body_length, creation_time," - " freshness_time, error_code, file_size, guid, client_namespace," - " client_id, requested_url, final_archived_url, operation_name," - " archive_body_name, title, file_path)" - " SELECT " - " offline_id, state, generate_bundle_attempts, get_operation_attempts," - " download_initiation_attempts, archive_body_length, creation_time," - " freshness_time, error_code, file_size, guid, client_namespace," - " client_id, requested_url, final_archived_url, operation_name," - " archive_body_name, title, file_path" - " FROM prefetch_items_old; " - // Drops the old items table. - "DROP TABLE prefetch_items_old; "; + R"sql( +ALTER TABLE prefetch_items RENAME TO prefetch_items_old; + +CREATE TABLE prefetch_items( +offline_id INTEGER PRIMARY KEY NOT NULL, +state INTEGER NOT NULL DEFAULT 0, +generate_bundle_attempts INTEGER NOT NULL DEFAULT 0, +get_operation_attempts INTEGER NOT NULL DEFAULT 0, +download_initiation_attempts INTEGER NOT NULL DEFAULT 0, +archive_body_length INTEGER_NOT_NULL DEFAULT -1, +creation_time INTEGER NOT NULL, +freshness_time INTEGER NOT NULL, +error_code INTEGER NOT NULL DEFAULT 0, +file_size INTEGER NOT NULL DEFAULT -1, +guid VARCHAR NOT NULL DEFAULT '', +client_namespace VARCHAR NOT NULL DEFAULT '', +client_id VARCHAR NOT NULL DEFAULT '', +requested_url VARCHAR NOT NULL DEFAULT '', +final_archived_url VARCHAR NOT NULL DEFAULT '', +operation_name VARCHAR NOT NULL DEFAULT '', +archive_body_name VARCHAR NOT NULL DEFAULT '', +title VARCHAR NOT NULL DEFAULT '', +file_path VARCHAR NOT NULL DEFAULT '' +); + +INSERT INTO prefetch_items +(offline_id, state, generate_bundle_attempts, get_operation_attempts, +download_initiation_attempts, archive_body_length, creation_time, +freshness_time, error_code, file_size, guid, client_namespace, +client_id, requested_url, final_archived_url, operation_name, +archive_body_name, title, file_path) +SELECT +offline_id, state, generate_bundle_attempts, get_operation_attempts, +download_initiation_attempts, archive_body_length, creation_time, +freshness_time, error_code, file_size, guid, client_namespace, +client_id, requested_url, final_archived_url, operation_name, +archive_body_name, title, file_path +FROM prefetch_items_old; + +DROP TABLE prefetch_items_old; +)sql"; sql::Transaction transaction(db); if (transaction.Begin() && db->Execute(kVersion1ToVersion2MigrationSql) && @@ -172,39 +188,71 @@ int MigrateFromVersion1To2(sql::Database* db, sql::MetaTable* meta_table) { return kVersionError; } -} // namespace - -// static -bool PrefetchStoreSchema::CreateOrUpgradeIfNeeded(sql::Database* db) { - DCHECK_GE(kCurrentVersion, kCompatibleVersion); - DCHECK(db); - if (!db) - return false; +int MigrateFromVersion2To3(sql::Database* db, sql::MetaTable* meta_table) { + const int target_version = 3; + const int target_compatible_version = 1; + static const char k2To3Sql[] = R"sql( +ALTER TABLE prefetch_items ADD COLUMN thumbnail_url VARCHAR NOT NULL DEFAULT ''; +ALTER TABLE prefetch_items ADD COLUMN favicon_url VARCHAR NOT NULL DEFAULT ''; +ALTER TABLE prefetch_items ADD COLUMN snippet VARCHAR NOT NULL DEFAULT ''; +ALTER TABLE prefetch_items ADD COLUMN attribution VARCHAR NOT NULL DEFAULT ''; +)sql"; - sql::MetaTable meta_table; - if (!meta_table.Init(db, kCurrentVersion, kCompatibleVersion)) - return false; + sql::Transaction transaction(db); + if (transaction.Begin() && db->Execute(k2To3Sql) && + SetVersionNumber(meta_table, target_version) && + SetCompatibleVersionNumber(meta_table, target_compatible_version) && + transaction.Commit()) + return target_version; - const int compatible_version = GetCompatibleVersionNumber(&meta_table); - int current_version = GetVersionNumber(&meta_table); - if (current_version == kVersionError || compatible_version == kVersionError) - return false; - DCHECK_GE(current_version, compatible_version); + return kVersionError; +} - // Stored database version is newer and incompatible with the current running - // code (Chrome was downgraded). The DB will never work until Chrome is - // re-upgraded. - if (compatible_version > kCurrentVersion) +// Returns true if the database has previously been initialized to a compatible +// version. +bool DatabaseIsValid(sql::Database* db, + sql::MetaTable* meta_table, + int* current_version, + int* compatible_version) { + if (!sql::MetaTable::DoesTableExist(db) || + !meta_table->Init(db, PrefetchStoreSchema::kCurrentVersion, + PrefetchStoreSchema::kCompatibleVersion)) return false; + *compatible_version = GetCompatibleVersionNumber(meta_table); + *current_version = GetVersionNumber(meta_table); + return // Sanity checks. + *current_version != kVersionError && + *compatible_version != kVersionError && *current_version >= 1 && + *compatible_version >= 1 && + // This can be false when Chrome is downgraded after a db change that's + // not backwards-compatible. + *compatible_version <= PrefetchStoreSchema::kCurrentVersion; +} - // Database is already at the latest version or has just been created. Create - // any missing tables and return. - if (current_version == kCurrentVersion) - return CreateLatestSchema(db); +} // namespace - // Versions 0 and below are unexpected. - if (current_version <= 0) - return false; +// static +bool PrefetchStoreSchema::CreateOrUpgradeIfNeeded(sql::Database* db) { + // TODO(harringtond): Add UMA to track errors and important actions here. + DCHECK(db); + sql::MetaTable meta_table; + int current_version, compatible_version; + if (!DatabaseIsValid(db, &meta_table, ¤t_version, + &compatible_version)) { + // Raze the database to get back to a working state. Note that this is + // considered dangerous, and may lose data that could be recovered later. + // The benefit of this is that we won't be stuck in a bad state. For + // prefetch, loss of the data in the database is not terrible: prefetching + // is best effort, and the user will get more prefetch suggestions later. + if (!db->Raze()) + return false; + sql::Transaction transaction(db); + sql::MetaTable new_meta; + return transaction.Begin() && + new_meta.Init(db, PrefetchStoreSchema::kCurrentVersion, + PrefetchStoreSchema::kCompatibleVersion) && + CreateLatestSchema(db) && transaction.Commit(); + } // Schema upgrade code starts here. // @@ -215,9 +263,11 @@ bool PrefetchStoreSchema::CreateOrUpgradeIfNeeded(sql::Database* db) { // should not. For instance, one should never refer to kCurrentVersion or // kCompatibleVersion when setting values for the current and compatible // versions as these are definitely going to change with each schema change. - if (current_version == 1) { + if (current_version == 1) current_version = MigrateFromVersion1To2(db, &meta_table); - } + + if (current_version == 2) + current_version = MigrateFromVersion2To3(db, &meta_table); return current_version == kCurrentVersion; } diff --git a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.h b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.h index f60bd1b0ed7..7334c58be60 100644 --- a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.h +++ b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema.h @@ -17,8 +17,11 @@ namespace offline_pages { // from any and all previous database versions to the latest. class PrefetchStoreSchema { public: - static const int kCurrentVersion; - static const int kCompatibleVersion; + // See version_schemas for a history of schema versions. + static constexpr int kCurrentVersion = 3; + static constexpr int kCompatibleVersion = 1; + static_assert(kCurrentVersion >= kCompatibleVersion, + "compatible version shouldn't be greater than the current one"); // Creates or upgrade the database schema as needed from information stored in // a metadata table. Returns |true| if the database is ready to be used, diff --git a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema_unittest.cc b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema_unittest.cc index 5199e4513d0..3d40391a621 100644 --- a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_schema_unittest.cc @@ -7,6 +7,13 @@ #include <limits> #include <memory> +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/json/string_escape.h" +#include "base/path_service.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" #include "sql/database.h" #include "sql/meta_table.h" #include "sql/statement.h" @@ -14,7 +21,7 @@ #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { - +namespace { static const char kSomeTableCreationSql[] = "CREATE TABLE some_table " "(id INTEGER PRIMARY KEY NOT NULL," @@ -25,6 +32,156 @@ static const char kAnotherTableCreationSql[] = "(id INTEGER PRIMARY KEY NOT NULL," " name VARCHAR NOT NULL)"; +std::vector<std::string> TableColumns(sql::Database* db, + const std::string table_name) { + std::vector<std::string> columns; + std::string sql = "PRAGMA TABLE_INFO(" + table_name + ")"; + sql::Statement table_info(db->GetUniqueStatement(sql.c_str())); + while (table_info.Step()) + columns.push_back(table_info.ColumnString(1)); + return columns; +} + +struct Table { + std::string ToString() const { + std::ostringstream ss; + ss << "-- TABLE " << name << " --\n"; + for (size_t row_index = 0; row_index < rows.size(); ++row_index) { + ss << "--- ROW " << row_index << " ---\n"; + const std::vector<std::string>& row = rows[row_index]; + for (size_t i = 0; i < row.size(); ++i) { + ss << column_names[i] << ": " << base::GetQuotedJSONString(row[i]) + << '\n'; + } + } + return ss.str(); + } + + std::string name; + std::vector<std::string> column_names; + // List of all values. Has size [row_count][column_count]. + std::vector<std::vector<std::string>> rows; +}; + +// Returns the value contained in a table cell, or nullptr if the cell row or +// column is invalid. +const std::string* TableCell(const Table& table, + const std::string& column, + size_t row) { + if (row >= table.rows.size()) + return nullptr; + for (size_t i = 0; i < table.column_names.size(); ++i) { + if (table.column_names[i] == column) { + return &table.rows[row][i]; + } + } + return nullptr; +} + +struct DatabaseTables { + std::string ToString() { + std::ostringstream ss; + for (auto i = tables.begin(); i != tables.end(); ++i) + ss << i->second.ToString(); + return ss.str(); + } + std::map<std::string, Table> tables; +}; + +Table ReadTable(sql::Database* db, const std::string table_name) { + Table table; + table.name = table_name; + table.column_names = TableColumns(db, table_name); + std::string sql = "SELECT * FROM " + table_name; + sql::Statement all_data(db->GetUniqueStatement(sql.c_str())); + while (all_data.Step()) { + std::vector<std::string> row; + for (size_t i = 0; i < table.column_names.size(); ++i) { + row.push_back(all_data.ColumnString(i)); + } + table.rows.push_back(std::move(row)); + } + return table; +} + +// Returns all tables in |db|, except the 'meta' table. We don't test the 'meta' +// table directly in this file, but instead use the MetaTable class. +DatabaseTables ReadTables(sql::Database* db) { + DatabaseTables database_tables; + std::stringstream ss; + sql::Statement table_names(db->GetUniqueStatement( + "SELECT name FROM sqlite_master WHERE type='table'")); + while (table_names.Step()) { + const std::string table_name = table_names.ColumnString(0); + if (table_name == "meta") + continue; + database_tables.tables[table_name] = ReadTable(db, table_name); + } + return database_tables; +} + +// Returns the SQL that defines a table. +std::string TableSql(sql::Database* db, const std::string& table_name) { + DatabaseTables database_tables; + std::stringstream ss; + sql::Statement table_sql(db->GetUniqueStatement( + "SELECT sql FROM sqlite_master WHERE type='table' AND name=?")); + table_sql.BindString(0, table_name); + if (!table_sql.Step()) + return std::string(); + // Try to normalize the SQL, since we use this to compare schemas. + std::string sql = + base::CollapseWhitespaceASCII(table_sql.ColumnString(0), true); + base::ReplaceSubstringsAfterOffset(&sql, 0, ", ", ","); + base::ReplaceSubstringsAfterOffset(&sql, 0, ",", ",\n"); + return sql; +} + +std::string ReadSchemaFile(const std::string& file_name) { + std::string data; + base::FilePath path; + CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &path)); + path = path.AppendASCII( + "components/test/data/offline_pages/prefetch/version_schemas/") + .AppendASCII(file_name); + CHECK(base::ReadFileToString(path, &data)) << path; + return data; +} + +std::unique_ptr<sql::Database> CreateTablesWithSampleRows(int version) { + auto db = std::make_unique<sql::Database>(); + CHECK(db->OpenInMemory()); + // Write a meta table. v*.sql overwrites version and last_compatible_version. + sql::MetaTable meta_table; + CHECK(meta_table.Init(db.get(), 1, 1)); + + const std::string schema = ReadSchemaFile( + base::StrCat({"v", base::NumberToString(version), ".sql"})); + CHECK(db->Execute(schema.c_str())); + return db; +} + +void ExpectDbIsCurrent(sql::Database* db) { + // Check the meta table. + sql::MetaTable meta_table; + EXPECT_TRUE(meta_table.Init(db, 1, 1)); + EXPECT_EQ(PrefetchStoreSchema::kCurrentVersion, + meta_table.GetVersionNumber()); + EXPECT_EQ(PrefetchStoreSchema::kCompatibleVersion, + meta_table.GetCompatibleVersionNumber()); + + std::unique_ptr<sql::Database> current_db = + CreateTablesWithSampleRows(PrefetchStoreSchema::kCurrentVersion); + + // Check that database schema is current. + for (auto name_and_table : ReadTables(db).tables) { + const std::string current_sql = + TableSql(current_db.get(), name_and_table.first); + const std::string real_sql = TableSql(db, name_and_table.first); + EXPECT_EQ(current_sql, real_sql); + } +} + TEST(PrefetchStoreSchemaPreconditionTest, TestSqliteCreateTableIsTransactional) { sql::Database db; @@ -90,200 +247,120 @@ TEST(PrefetchStoreSchemaPreconditionTest, EXPECT_TRUE(db.DoesColumnExist("some_table", "value")); } -class PrefetchStoreSchemaTest : public testing::Test { - public: - PrefetchStoreSchemaTest() = default; - ~PrefetchStoreSchemaTest() override = default; - - void SetUp() override { - db_ = std::make_unique<sql::Database>(); - ASSERT_TRUE(db_->OpenInMemory()); - ASSERT_FALSE(sql::MetaTable::DoesTableExist(db_.get())); - } - - void CheckTablesExistence() { - EXPECT_TRUE(db_->DoesTableExist("prefetch_items")); - EXPECT_TRUE(db_->DoesTableExist("prefetch_downloader_quota")); - EXPECT_FALSE(db_->DoesTableExist("prefetch_items_old")); - } - - protected: - std::unique_ptr<sql::Database> db_; - std::unique_ptr<PrefetchStoreSchema> schema_; -}; - -TEST_F(PrefetchStoreSchemaTest, TestSchemaCreationFromNothing) { - EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db_.get())); - CheckTablesExistence(); - sql::MetaTable meta_table; - EXPECT_TRUE(meta_table.Init(db_.get(), std::numeric_limits<int>::max(), - std::numeric_limits<int>::max())); - EXPECT_EQ(PrefetchStoreSchema::kCurrentVersion, - meta_table.GetVersionNumber()); - EXPECT_EQ(PrefetchStoreSchema::kCompatibleVersion, - meta_table.GetCompatibleVersionNumber()); -} - -TEST_F(PrefetchStoreSchemaTest, TestMissingTablesAreCreatedAtLatestVersion) { - sql::MetaTable meta_table; - EXPECT_TRUE(meta_table.Init(db_.get(), PrefetchStoreSchema::kCurrentVersion, - PrefetchStoreSchema::kCompatibleVersion)); - EXPECT_EQ(PrefetchStoreSchema::kCurrentVersion, - meta_table.GetVersionNumber()); - EXPECT_EQ(PrefetchStoreSchema::kCompatibleVersion, - meta_table.GetCompatibleVersionNumber()); +TEST(PrefetchStoreSchemaTest, TestInvalidMetaTable) { + // Verify CreateOrUpgradeIfNeeded will raze the db if it can't be used. + sql::Database db; + ASSERT_TRUE(db.OpenInMemory()); + sql::MetaTable meta; + meta.Init(&db, 100, 99); // Some future version. + ASSERT_TRUE(db.Execute("CREATE TABLE prefetch_items (x INTEGER DEFAULT 1);")); + ASSERT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(&db)); - EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db_.get())); - CheckTablesExistence(); + ExpectDbIsCurrent(&db); } -TEST_F(PrefetchStoreSchemaTest, TestMissingTablesAreRecreated) { - EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db_.get())); - CheckTablesExistence(); - - EXPECT_TRUE(db_->Execute("DROP TABLE prefetch_items")); - EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db_.get())); - CheckTablesExistence(); - - EXPECT_TRUE(db_->Execute("DROP TABLE prefetch_downloader_quota")); - EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db_.get())); - CheckTablesExistence(); -} +// Verify the latest v#.sql accurately represents the current schema. +// +// Note: We keep the creation code for the current schema version duplicated in +// PrefetchStoreSchema and in the latest version test file so that when we move +// on from the current schema we already know it's represented correctly in the +// test. +TEST(PrefetchStoreSchemaTest, TestCurrentSqlFileIsAccurate) { + // Create the database with the release code, and with v?.sql. + sql::Database db; + ASSERT_TRUE(db.OpenInMemory()); + ASSERT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(&db)); -void CreateVersion1TablesWithSampleRows(sql::Database* db) { - // Create version 1 tables. - static const char kV0ItemsTableCreationSql[] = - "CREATE TABLE prefetch_items" - "(offline_id INTEGER PRIMARY KEY NOT NULL," - " state INTEGER NOT NULL DEFAULT 0," - " generate_bundle_attempts INTEGER NOT NULL DEFAULT 0," - " get_operation_attempts INTEGER NOT NULL DEFAULT 0," - " download_initiation_attempts INTEGER NOT NULL DEFAULT 0," - " archive_body_length INTEGER_NOT_NULL DEFAULT -1," - " creation_time INTEGER NOT NULL," - " freshness_time INTEGER NOT NULL," - " error_code INTEGER NOT NULL DEFAULT 0," - " file_size INTEGER NOT NULL DEFAULT 0," - " guid VARCHAR NOT NULL DEFAULT ''," - " client_namespace VARCHAR NOT NULL DEFAULT ''," - " client_id VARCHAR NOT NULL DEFAULT ''," - " requested_url VARCHAR NOT NULL DEFAULT ''," - " final_archived_url VARCHAR NOT NULL DEFAULT ''," - " operation_name VARCHAR NOT NULL DEFAULT ''," - " archive_body_name VARCHAR NOT NULL DEFAULT ''," - " title VARCHAR NOT NULL DEFAULT ''," - " file_path VARCHAR NOT NULL DEFAULT ''" - ")"; - EXPECT_TRUE(db->Execute(kV0ItemsTableCreationSql)); - static const char kV0QuotaTableCreationSql[] = - "CREATE TABLE prefetch_downloader_quota" - "(quota_id INTEGER PRIMARY KEY NOT NULL DEFAULT 1," - " update_time INTEGER NOT NULL," - " available_quota INTEGER NOT NULL DEFAULT 0)"; - EXPECT_TRUE(db->Execute(kV0QuotaTableCreationSql)); - - // Insert one row with artificial values into the items table. - static const char kV0ItemInsertSql[] = - "INSERT INTO prefetch_items" - " (offline_id, state, generate_bundle_attempts, get_operation_attempts," - " download_initiation_attempts, archive_body_length, creation_time," - " freshness_time, error_code, file_size, guid, client_namespace," - " client_id, requested_url, final_archived_url, operation_name," - " archive_body_name, title, file_path)" - " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - sql::Statement insertStatement1(db->GetUniqueStatement(kV0ItemInsertSql)); - // Generates fake values for all integer columns starting at 1. - for (int i = 0; i <= 9; ++i) - insertStatement1.BindInt(i, i + 1); - // Generates fake values for all string columns starting at "a". - for (int i = 10; i <= 18; ++i) - insertStatement1.BindString(i, std::string(1, 'a' + i - 10)); - EXPECT_TRUE(insertStatement1.Run()); - - // Insert one row with artificial values into the quota table. - static const char kV0QuotaInsertSql[] = - "INSERT INTO prefetch_downloader_quota" - " (quota_id, update_time, available_quota)" - " VALUES (?, ?, ?)"; - sql::Statement insertStatement2(db->GetUniqueStatement(kV0QuotaInsertSql)); - // Generates fake values for all columns. - insertStatement2.BindInt(0, 1); - insertStatement2.BindInt(1, 2); - insertStatement2.BindInt(2, 3); - EXPECT_TRUE(insertStatement2.Run()); + ExpectDbIsCurrent(&db); } -void CheckSampleRowsAtCurrentVersion(sql::Database* db) { - // Checks the previously inserted item row was migrated correctly. - static const char kV0ItemSelectSql[] = - "SELECT " - " offline_id, state, generate_bundle_attempts, get_operation_attempts," - " download_initiation_attempts, archive_body_length, creation_time," - " freshness_time, error_code, file_size, guid, client_namespace," - " client_id, requested_url, final_archived_url, operation_name," - " archive_body_name, title, file_path" - " FROM prefetch_items"; - sql::Statement selectStatement1(db->GetUniqueStatement(kV0ItemSelectSql)); - ASSERT_TRUE(selectStatement1.Step()); - // Checks fake values for all integer columns. - for (int i = 0; i <= 9; ++i) - EXPECT_EQ(i + 1, selectStatement1.ColumnInt(i)) - << "Wrong integer value at items table's column " << i; - // Checks fake values for all string columns. - for (int i = 10; i <= 18; ++i) - EXPECT_EQ(std::string(1, 'a' + i - 10), selectStatement1.ColumnString(i)) - << "Wrong string value at items table's column " << i; - ; - EXPECT_FALSE(selectStatement1.Step()); - - // Checks the previously inserted quota row was migrated correctly. - static const char kV0QuotaSelectSql[] = - "SELECT quota_id, update_time, available_quota" - " FROM prefetch_downloader_quota"; - sql::Statement selectStatement2(db->GetUniqueStatement(kV0QuotaSelectSql)); - ASSERT_TRUE(selectStatement2.Step()); - // Checks fake values for all columns. - EXPECT_EQ(1, selectStatement2.ColumnInt(0)); - EXPECT_EQ(2, selectStatement2.ColumnInt(1)); - EXPECT_EQ(3, selectStatement2.ColumnInt(2)); - EXPECT_FALSE(selectStatement2.Step()); +// Tests database creation starting with all previous versions, or an empty +// state. +TEST(PrefetchStoreSchemaTest, TestCreateOrMigrate) { + for (int i = 0; i <= PrefetchStoreSchema::kCurrentVersion; ++i) { + SCOPED_TRACE(testing::Message() << "Testing migration from version " << i); + std::unique_ptr<sql::Database> db; + // When i==0, start from an empty state. + const int version = i > 0 ? i : PrefetchStoreSchema::kCurrentVersion; + if (i > 0) { + db = CreateTablesWithSampleRows(i); + // Executes the migration. + EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db.get())); + } else { + db = std::make_unique<sql::Database>(); + ASSERT_TRUE(db->OpenInMemory()); + // Creation from scratch. + EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db.get())); + // Tables are already created, this will just insert rows. + const std::string schema = ReadSchemaFile( + base::StrCat({"v", base::NumberToString(version), ".sql"})); + ASSERT_TRUE(db->Execute(schema.c_str())); + } + + // Check schema. + ExpectDbIsCurrent(db.get()); + + // Check the database contents. + std::string expected_data = ReadSchemaFile( + base::StrCat({"v", base::NumberToString(version), ".data"})); + EXPECT_EQ(expected_data, ReadTables(db.get()).ToString()); + } } -// Tests that a migration from the initially deployed version of the schema, -// as it was for chromium/src at 90113a2c01ca9ff77042daacd8282a4c16aade85, is -// correctly migrated to the final, current version without losing data. -TEST_F(PrefetchStoreSchemaTest, TestMigrationFromV0) { - // Set version numbers to 1. - sql::MetaTable meta_table; - EXPECT_TRUE(meta_table.Init(db_.get(), 1, 1)); - EXPECT_EQ(1, meta_table.GetVersionNumber()); - EXPECT_EQ(1, meta_table.GetCompatibleVersionNumber()); - - CreateVersion1TablesWithSampleRows(db_.get()); - - // Executes the migration. - EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(db_.get())); - EXPECT_EQ(2, meta_table.GetVersionNumber()); - EXPECT_EQ(1, meta_table.GetCompatibleVersionNumber()); - CheckTablesExistence(); - - CheckSampleRowsAtCurrentVersion(db_.get()); - - // Tests that the default value for file size is now -1. - sql::Statement fileSizeInsertStatement(db_->GetUniqueStatement( - "INSERT INTO prefetch_items (offline_id, creation_time, freshness_time)" - " VALUES (?, ?, ?)")); - fileSizeInsertStatement.BindInt(0, 100); - fileSizeInsertStatement.BindInt(1, 101); - fileSizeInsertStatement.BindInt(2, 102); - EXPECT_TRUE(fileSizeInsertStatement.Run()); - - sql::Statement fileSizeSelectStatement(db_->GetUniqueStatement( - "SELECT file_size FROM prefetch_items WHERE offline_id = ?")); - fileSizeSelectStatement.BindInt(0, 100); - ASSERT_TRUE(fileSizeSelectStatement.Step()); - EXPECT_EQ(-1, fileSizeSelectStatement.ColumnInt(0)); - EXPECT_FALSE(fileSizeSelectStatement.Step()); +// Test that the current database version can be used by all compatible +// versions. +TEST(PrefetchStoreSchemaTest, TestRevert) { + static_assert(PrefetchStoreSchema::kCompatibleVersion == 1, + "If compatible version is changed, add a test to verify the " + "database is correctly razed and recreated!"); + + // This test simply runs the insert operations in v*.sql on a database + // with the current schema. + for (int version = PrefetchStoreSchema::kCompatibleVersion; + version < PrefetchStoreSchema::kCurrentVersion; ++version) { + SCOPED_TRACE(testing::Message() << "Testing revert to version " << version); + // First, extract the expected state after running v*.sql. + DatabaseTables original_state; + { + std::unique_ptr<sql::Database> db = CreateTablesWithSampleRows(version); + original_state = ReadTables(db.get()); + } + + // Create a new database at the current version. + sql::Database db; + ASSERT_TRUE(db.OpenInMemory()); + EXPECT_TRUE(PrefetchStoreSchema::CreateOrUpgradeIfNeeded(&db)); + + // Attempt to insert a row using the old SQL. + const std::string schema = ReadSchemaFile( + base::StrCat({"v", base::NumberToString(version), ".sql"})); + EXPECT_TRUE(db.Execute(schema.c_str())); + + // Check the database contents. + // We should find every value from original_state present in the db. + std::string expected_data = ReadSchemaFile( + base::StrCat({"v", base::NumberToString(version), ".data"})); + const DatabaseTables new_state = ReadTables(&db); + for (auto name_and_table : original_state.tables) { + const Table& original_table = name_and_table.second; + ASSERT_EQ(1ul, new_state.tables.count(name_and_table.first)); + const Table& new_table = + new_state.tables.find(name_and_table.first)->second; + for (size_t row = 0; row < original_table.rows.size(); ++row) { + for (const std::string& column_name : original_table.column_names) { + const std::string* old_value = + TableCell(original_table, column_name, row); + const std::string* new_value = TableCell(new_table, column_name, row); + ASSERT_TRUE(old_value); + EXPECT_TRUE(new_value) << "new table does not have old value"; + if (new_value) { + EXPECT_EQ(*old_value, *new_value); + } + } + } + } + } } +} // namespace } // namespace offline_pages diff --git a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_test_util.cc b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_test_util.cc index b8787387b70..1f7209691ff 100644 --- a/chromium/components/offline_pages/core/prefetch/store/prefetch_store_test_util.cc +++ b/chromium/components/offline_pages/core/prefetch/store/prefetch_store_test_util.cc @@ -45,12 +45,17 @@ const char* kSqlAllColumnNames = "archive_body_name, " "title, " "file_path, " - "file_size"; + "file_size, " + "thumbnail_url, " + "favicon_url, " + "snippet, " + "attribution"; bool InsertPrefetchItemSync(const PrefetchItem& item, sql::Database* db) { static const std::string kSql = base::StringPrintf( "INSERT INTO prefetch_items (%s)" - " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + " ?, ?, ?)", kSqlAllColumnNames); sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql.c_str())); statement.BindInt64(0, item.offline_id); @@ -72,6 +77,10 @@ bool InsertPrefetchItemSync(const PrefetchItem& item, sql::Database* db) { statement.BindString16(16, item.title); statement.BindString(17, store_utils::ToDatabaseFilePath(item.file_path)); statement.BindInt64(18, item.file_size); + statement.BindString(19, item.thumbnail_url.spec()); + statement.BindString(20, item.favicon_url.spec()); + statement.BindString(21, item.snippet); + statement.BindString(22, item.attribution); return statement.Run(); } @@ -89,7 +98,7 @@ int CountPrefetchItemsSync(sql::Database* db) { // Populates the PrefetchItem with the data from the current row of the passed // in statement following the natural column ordering. void PopulatePrefetchItem(const sql::Statement& statement, PrefetchItem* item) { - DCHECK_EQ(19, statement.ColumnCount()); + DCHECK_EQ(23, statement.ColumnCount()); DCHECK(item); // Fields are assigned to the item in the order they are stored in the SQL @@ -115,6 +124,10 @@ void PopulatePrefetchItem(const sql::Statement& statement, PrefetchItem* item) { item->file_path = store_utils::FromDatabaseFilePath(statement.ColumnString(17)); item->file_size = statement.ColumnInt64(18); + item->thumbnail_url = GURL(statement.ColumnString(19)); + item->favicon_url = GURL(statement.ColumnString(20)); + item->snippet = statement.ColumnString(21); + item->attribution = statement.ColumnString(22); } std::unique_ptr<PrefetchItem> GetPrefetchItemSync(int64_t offline_id, diff --git a/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.cc b/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.cc index bf78025e3df..d15e88ccccd 100644 --- a/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.cc +++ b/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.cc @@ -64,6 +64,10 @@ OfflinePageModel* StubPrefetchService::GetOfflinePageModel() { return nullptr; } +image_fetcher::ImageFetcher* StubPrefetchService::GetThumbnailImageFetcher() { + return nullptr; +} + SuggestedArticlesObserver* StubPrefetchService::GetSuggestedArticlesObserverForTesting() { return nullptr; diff --git a/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.h b/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.h index 8ba27e267d1..8457d27d522 100644 --- a/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.h +++ b/chromium/components/offline_pages/core/prefetch/stub_prefetch_service.h @@ -29,6 +29,7 @@ class StubPrefetchService : public PrefetchService { PrefetchBackgroundTaskHandler* GetPrefetchBackgroundTaskHandler() override; ThumbnailFetcher* GetThumbnailFetcher() override; OfflinePageModel* GetOfflinePageModel() override; + image_fetcher::ImageFetcher* GetThumbnailImageFetcher() override; SuggestedArticlesObserver* GetSuggestedArticlesObserverForTesting() override; }; diff --git a/chromium/components/offline_pages/core/prefetch/add_unique_urls_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/add_unique_urls_task.cc index a8083132cc7..1813cebc246 100644 --- a/chromium/components/offline_pages/core/prefetch/add_unique_urls_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/add_unique_urls_task.cc @@ -2,18 +2,19 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/add_unique_urls_task.h" +#include "components/offline_pages/core/prefetch/tasks/add_unique_urls_task.h" #include <map> #include <memory> -#include <set> #include <utility> #include "base/bind.h" #include "base/callback.h" #include "base/logging.h" #include "base/metrics/histogram_macros.h" +#include "base/time/clock.h" #include "base/time/time.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" @@ -30,22 +31,19 @@ using Result = AddUniqueUrlsTask::Result; namespace { -std::map<std::string, std::pair<int64_t, PrefetchItemState>> -FindExistingPrefetchItemsInNamespaceSync(sql::Database* db, - const std::string& name_space) { +// Returns a map of URL to offline_id for all existing prefetch items. +std::map<std::string, int64_t> GetAllUrlsAndIdsFromNamespaceSync( + sql::Database* db, + const std::string& name_space) { static const char kSql[] = - "SELECT offline_id, state, requested_url FROM prefetch_items" + "SELECT requested_url, offline_id FROM prefetch_items" " WHERE client_namespace = ?"; sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindString(0, name_space); - std::map<std::string, std::pair<int64_t, PrefetchItemState>> result; - while (statement.Step()) { - result.insert(std::make_pair( - statement.ColumnString(2), - std::make_pair(statement.ColumnInt64(0), static_cast<PrefetchItemState>( - statement.ColumnInt(1))))); - } + std::map<std::string, int64_t> result; + while (statement.Step()) + result.emplace(statement.ColumnString(0), statement.ColumnInt64(1)); return result; } @@ -57,9 +55,9 @@ bool CreatePrefetchItemSync(sql::Database* db, static const char kSql[] = "INSERT INTO prefetch_items" " (offline_id, requested_url, client_namespace, client_id, creation_time," - " freshness_time, title)" + " freshness_time, title, thumbnail_url)" " VALUES" - " (?, ?, ?, ?, ?, ?, ?)"; + " (?, ?, ?, ?, ?, ?, ?, ?)"; sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindInt64(0, store_utils::GenerateOfflineId()); @@ -69,16 +67,31 @@ bool CreatePrefetchItemSync(sql::Database* db, statement.BindInt64(4, now_db_time); statement.BindInt64(5, now_db_time); statement.BindString16(6, prefetch_url.title); + statement.BindString(7, prefetch_url.thumbnail_url.spec()); + + return statement.Run(); +} + +bool UpdateItemTimeSync(sql::Database* db, + int64_t offline_id, + base::Time now_db_time) { + static const char kSql[] = + "UPDATE prefetch_items SET" + " freshness_time=?,creation_time=?" + " WHERE offline_id=?"; + + sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); + statement.BindInt64(0, store_utils::ToDatabaseTime(now_db_time)); + statement.BindInt64(1, store_utils::ToDatabaseTime(now_db_time)); + statement.BindInt64(2, offline_id); return statement.Run(); } // Adds new prefetch item entries to the store using the URLs and client IDs -// from |candidate_prefetch_urls| and the client's |name_space|. Also cleans up -// entries in the Zombie state from the client's |name_space| except for the -// ones whose URL is contained in |candidate_prefetch_urls|. -// Returns the number of added prefecth items. -Result AddUrlsAndCleanupZombiesSync( +// from |candidate_prefetch_urls| and the client's |name_space|. Returns the +// result of the attempt to add new URLs. +Result AddUniqueUrlsSync( const std::string& name_space, const std::vector<PrefetchURL>& candidate_prefetch_urls, sql::Database* db) { @@ -86,41 +99,38 @@ Result AddUrlsAndCleanupZombiesSync( if (!transaction.Begin()) return Result::STORE_ERROR; - std::map<std::string, std::pair<int64_t, PrefetchItemState>> existing_items = - FindExistingPrefetchItemsInNamespaceSync(db, name_space); - + std::map<std::string, int64_t> existing_items = + GetAllUrlsAndIdsFromNamespaceSync(db, name_space); + std::set<std::string> added_urls; int added_row_count = 0; - base::Time now = base::Time::Now(); + base::Time now = OfflineClock()->Now(); // Insert rows in reverse order to ensure that the beginning of the list has - // the newest timestamp. This will cause it to be prefetched first. + // the most recent timestamps so that it is prefetched first. for (auto candidate_iter = candidate_prefetch_urls.rbegin(); candidate_iter != candidate_prefetch_urls.rend(); ++candidate_iter) { - PrefetchURL prefetch_url = *candidate_iter; - auto iter = existing_items.find(prefetch_url.url.spec()); - if (iter == existing_items.end()) { + const PrefetchURL& prefetch_url = *candidate_iter; + const std::string url_spec = prefetch_url.url.spec(); + // Don't add the same URL more than once. + if (!added_urls.insert(url_spec).second) + continue; + + auto existing_iter = existing_items.find(url_spec); + if (existing_iter != existing_items.end()) { + // An existing item is still being suggested so update its timestamps (and + // therefore priority). + if (!UpdateItemTimeSync(db, existing_iter->second, now)) + return Result::STORE_ERROR; // Transaction rollback. + } else { if (!CreatePrefetchItemSync(db, name_space, prefetch_url, - store_utils::ToDatabaseTime(now))) + store_utils::ToDatabaseTime(now))) { return Result::STORE_ERROR; // Transaction rollback. + } added_row_count++; - - // We artificially add a microsecond to ensure that the timestamp is - // different (and guarantee a particular order when sorting by timestamp). - now += base::TimeDelta::FromMicroseconds(1); - } else { - // Removing from the list of existing items if it was requested again, to - // prevent it from being removed in the next step. - existing_items.erase(iter); } - } - // Purge remaining zombie IDs. - for (const auto& existing_item : existing_items) { - if (existing_item.second.second != PrefetchItemState::ZOMBIE) - continue; - if (!PrefetchStoreUtils::DeletePrefetchItemByOfflineIdSync( - db, existing_item.second.first)) { - return Result::STORE_ERROR; // Transaction rollback. - } + // We artificially add a microsecond to ensure that the timestamp is + // different (and guarantee a particular order when sorting by timestamp). + now += base::TimeDelta::FromMicroseconds(1); } if (!transaction.Commit()) @@ -130,6 +140,7 @@ Result AddUrlsAndCleanupZombiesSync( added_row_count); return added_row_count > 0 ? Result::URLS_ADDED : Result::NOTHING_ADDED; } + } // namespace AddUniqueUrlsTask::AddUniqueUrlsTask( @@ -149,11 +160,11 @@ AddUniqueUrlsTask::AddUniqueUrlsTask( AddUniqueUrlsTask::~AddUniqueUrlsTask() {} void AddUniqueUrlsTask::Run() { - prefetch_store_->Execute(base::BindOnce(&AddUrlsAndCleanupZombiesSync, - name_space_, prefetch_urls_), - base::BindOnce(&AddUniqueUrlsTask::OnUrlsAdded, - weak_ptr_factory_.GetWeakPtr()), - Result::STORE_ERROR); + prefetch_store_->Execute( + base::BindOnce(&AddUniqueUrlsSync, name_space_, prefetch_urls_), + base::BindOnce(&AddUniqueUrlsTask::OnUrlsAdded, + weak_ptr_factory_.GetWeakPtr()), + Result::STORE_ERROR); } void AddUniqueUrlsTask::OnUrlsAdded(Result result) { diff --git a/chromium/components/offline_pages/core/prefetch/add_unique_urls_task.h b/chromium/components/offline_pages/core/prefetch/tasks/add_unique_urls_task.h index c7fb8a13237..ed3afa01ac9 100644 --- a/chromium/components/offline_pages/core/prefetch/add_unique_urls_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/add_unique_urls_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_ADD_UNIQUE_URLS_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_ADD_UNIQUE_URLS_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_ADD_UNIQUE_URLS_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_ADD_UNIQUE_URLS_TASK_H_ #include <string> #include <vector> @@ -20,12 +20,10 @@ struct PrefetchURL; // Task that adds new URL suggestions to the pipeline. URLs are matched against // existing ones from any stage of the process so that only new, unique ones are // actually added. -// Fully processed items are kept in the store in the PrefetchItemState::ZOMBIE -// state until it is confirmed that the client for its namespace is not -// recommending the same URL anymore to avoid processing it twice. So once the -// step described above is done, all same namespace items in the ZOMBIE state -// whose URL didn't match any of the just suggested ones are finally deleted -// from the store. +// Fully processed items are kept in store in the zombie state so that follow up +// recommendations of the same URL from the same client are not processed twice. +// Zombie items are then cleaned after a set period of time by the +// |StaleEntryFinalizerTask|. class AddUniqueUrlsTask : public Task { public: // Result of executing the command in the store. @@ -59,4 +57,4 @@ class AddUniqueUrlsTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_ADD_UNIQUE_URLS_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_ADD_UNIQUE_URLS_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/add_unique_urls_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/add_unique_urls_task_unittest.cc index 14a843d9977..86cbf7b8af6 100644 --- a/chromium/components/offline_pages/core/prefetch/add_unique_urls_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/add_unique_urls_task_unittest.cc @@ -2,9 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/add_unique_urls_task.h" +#include "components/offline_pages/core/prefetch/tasks/add_unique_urls_task.h" #include <map> +#include <memory> #include <set> #include <string> #include <vector> @@ -12,12 +13,14 @@ #include "base/strings/utf_string_conversions.h" #include "base/test/test_simple_task_runner.h" #include "base/threading/thread_task_runner_handle.h" +#include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" +#include "components/offline_pages/core/test_scoped_offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -26,24 +29,30 @@ const char kTestNamespace[] = "test"; const char kClientId1[] = "ID-1"; const char kClientId2[] = "ID-2"; const char kClientId3[] = "ID-3"; -const char kClientId4[] = "ID-5"; const GURL kTestURL1("https://www.google.com/"); const GURL kTestURL2("http://www.example.com/"); const GURL kTestURL3("https://news.google.com/"); -const GURL kTestURL4("https://chrome.google.com/"); +const char kTestThumbnailURL[] = "http://thumbnail.com/"; const base::string16 kTestTitle1 = base::ASCIIToUTF16("Title 1"); const base::string16 kTestTitle2 = base::ASCIIToUTF16("Title 2"); const base::string16 kTestTitle3 = base::ASCIIToUTF16("Title 3"); -const base::string16 kTestTitle4 = base::ASCIIToUTF16("Title 4"); } // namespace class AddUniqueUrlsTaskTest : public PrefetchTaskTestBase { public: - AddUniqueUrlsTaskTest(); + AddUniqueUrlsTaskTest() = default; ~AddUniqueUrlsTaskTest() override = default; // Returns all items stored in a map keyed with client id. - std::map<std::string, PrefetchItem> GetAllItems(); + std::map<std::string, PrefetchItem> GetAllItems() { + std::set<PrefetchItem> set; + store_util()->GetAllItems(&set); + + std::map<std::string, PrefetchItem> map; + for (const auto& item : set) + map[item.client_id.id] = item; + return map; + } TestPrefetchDispatcher* dispatcher() { return &dispatcher_; } @@ -51,17 +60,6 @@ class AddUniqueUrlsTaskTest : public PrefetchTaskTestBase { TestPrefetchDispatcher dispatcher_; }; -AddUniqueUrlsTaskTest::AddUniqueUrlsTaskTest() {} - -std::map<std::string, PrefetchItem> AddUniqueUrlsTaskTest::GetAllItems() { - std::set<PrefetchItem> set; - store_util()->GetAllItems(&set); - - std::map<std::string, PrefetchItem> map; - for (const auto& item : set) - map[item.client_id.id] = item; - return map; -} TEST_F(AddUniqueUrlsTaskTest, StoreFailure) { store_util()->SimulateInitializationError(); @@ -72,18 +70,22 @@ TEST_F(AddUniqueUrlsTaskTest, StoreFailure) { TEST_F(AddUniqueUrlsTaskTest, AddTaskInEmptyStore) { std::vector<PrefetchURL> urls; - urls.push_back(PrefetchURL{kClientId1, kTestURL1, kTestTitle1}); - urls.push_back(PrefetchURL{kClientId2, kTestURL2, kTestTitle2}); + PrefetchURL url1{kClientId1, kTestURL1, kTestTitle1}; + url1.thumbnail_url = GURL(kTestThumbnailURL); + urls.push_back(url1); + urls.emplace_back(kClientId2, kTestURL2, kTestTitle2); RunTask(std::make_unique<AddUniqueUrlsTask>(dispatcher(), store(), kTestNamespace, urls)); std::map<std::string, PrefetchItem> items = GetAllItems(); - ASSERT_EQ(2u, items.size()); - ASSERT_TRUE(items.count(kClientId1) > 0); + ASSERT_EQ(2U, items.size()); + ASSERT_GT(items.count(kClientId1), 0U); EXPECT_EQ(kTestURL1, items[kClientId1].url); EXPECT_EQ(kTestNamespace, items[kClientId1].client_id.name_space); EXPECT_EQ(kTestTitle1, items[kClientId1].title); - ASSERT_TRUE(items.count(kClientId2) > 0); + ASSERT_GT(items.count(kClientId2), 0U); + EXPECT_EQ(kTestThumbnailURL, items[kClientId1].thumbnail_url); + ASSERT_GT(items.count(kClientId2), 0ul); EXPECT_EQ(kTestURL2, items[kClientId2].url); EXPECT_EQ(kTestNamespace, items[kClientId2].client_id.name_space); EXPECT_EQ(kTestTitle2, items[kClientId2].title); @@ -94,8 +96,11 @@ TEST_F(AddUniqueUrlsTaskTest, AddTaskInEmptyStore) { } TEST_F(AddUniqueUrlsTaskTest, SingleDuplicateUrlNotAdded) { - std::vector<PrefetchURL> urls; - urls.push_back(PrefetchURL{kClientId1, kTestURL1, kTestTitle1}); + // Add the same URL twice in a single round. Only one entry should be added. + std::vector<PrefetchURL> urls = { + {kClientId1, kTestURL1, kTestTitle1}, + {kClientId2, kTestURL1, kTestTitle2}, + }; RunTask(std::make_unique<AddUniqueUrlsTask>(dispatcher(), store(), kTestNamespace, urls)); EXPECT_EQ(1, dispatcher()->task_schedule_count); @@ -110,80 +115,81 @@ TEST_F(AddUniqueUrlsTaskTest, SingleDuplicateUrlNotAdded) { kTestNamespace, urls)); // The task schedule count should not have changed with no new URLs. EXPECT_EQ(1, dispatcher()->task_schedule_count); + EXPECT_EQ(1UL, GetAllItems().size()); } -TEST_F(AddUniqueUrlsTaskTest, DontAddURLIfItExists) { - std::vector<PrefetchURL> urls; - urls.push_back(PrefetchURL{kClientId1, kTestURL1, kTestTitle1}); - urls.push_back(PrefetchURL{kClientId2, kTestURL2, kTestTitle2}); +TEST_F(AddUniqueUrlsTaskTest, DontAddURLIfItAlreadyExists) { + // Overrides and initializes a test clock. + TestScopedOfflineClock clock; + const base::Time start_time = base::Time::Now(); + clock.SetNow(start_time); + + // Populate the store with pre-existing items. + std::vector<PrefetchURL> urls = {{kClientId1, kTestURL1, kTestTitle1}, + {kClientId2, kTestURL2, kTestTitle2}}; RunTask(std::make_unique<AddUniqueUrlsTask>(dispatcher(), store(), kTestNamespace, urls)); EXPECT_EQ(1, dispatcher()->task_schedule_count); - urls.clear(); - // This PrefetchURL has a duplicate URL, should not be added. - urls.push_back(PrefetchURL{kClientId4, kTestURL1, kTestTitle4}); - urls.push_back(PrefetchURL{kClientId3, kTestURL3, kTestTitle3}); + // Advance time by 1 hour to verify that timestamp of ID-1 is updated on the + // next task execution. + clock.Advance(base::TimeDelta::FromHours(1)); + const base::Time later_time = clock.Now(); + // Turn ID-1 and ID-2 items into zombies. + // Note: ZombifyPrefetchItem returns the number of affected items. + EXPECT_EQ(1, store_util()->ZombifyPrefetchItems(kTestNamespace, kTestURL1)); + EXPECT_EQ(1, store_util()->ZombifyPrefetchItems(kTestNamespace, kTestURL2)); + + urls = {{kClientId1, kTestURL1, kTestTitle1}, + {kClientId3, kTestURL3, kTestTitle3}}; RunTask(std::make_unique<AddUniqueUrlsTask>(dispatcher(), store(), kTestNamespace, urls)); EXPECT_EQ(2, dispatcher()->task_schedule_count); std::map<std::string, PrefetchItem> items = GetAllItems(); - ASSERT_EQ(3u, items.size()); - ASSERT_TRUE(items.count(kClientId1) > 0); + ASSERT_EQ(3U, items.size()); + ASSERT_GT(items.count(kClientId1), 0U); + + // Re-suggested ID-1 should have its timestamp updated. EXPECT_EQ(kTestURL1, items[kClientId1].url); EXPECT_EQ(kTestNamespace, items[kClientId1].client_id.name_space); EXPECT_EQ(kTestTitle1, items[kClientId1].title); - ASSERT_TRUE(items.count(kClientId2) > 0); + EXPECT_EQ(PrefetchItemState::ZOMBIE, items[kClientId1].state); + // Note: as timestamps are inserted with microsecond variations, we're + // comparing them using a safe range of 1 second. + EXPECT_LE(later_time, items[kClientId1].creation_time); + EXPECT_GE(later_time + base::TimeDelta::FromSeconds(1), + items[kClientId1].creation_time); + EXPECT_LE(later_time, items[kClientId1].freshness_time); + EXPECT_GE(later_time + base::TimeDelta::FromSeconds(1), + items[kClientId1].freshness_time); + + // Previously existing ID-2 should not have been modified. + ASSERT_GT(items.count(kClientId2), 0U); EXPECT_EQ(kTestURL2, items[kClientId2].url); EXPECT_EQ(kTestNamespace, items[kClientId2].client_id.name_space); EXPECT_EQ(kTestTitle2, items[kClientId2].title); - ASSERT_TRUE(items.count(kClientId3) > 0); - EXPECT_EQ(kTestURL3, items[kClientId3].url); - EXPECT_EQ(kTestNamespace, items[kClientId3].client_id.name_space); - EXPECT_EQ(kTestTitle3, items[kClientId3].title); -} - -TEST_F(AddUniqueUrlsTaskTest, HandleZombiePrefetchItems) { - std::vector<PrefetchURL> urls; - urls.push_back(PrefetchURL{kClientId1, kTestURL1, kTestTitle1}); - urls.push_back(PrefetchURL{kClientId2, kTestURL2, kTestTitle2}); - urls.push_back(PrefetchURL{kClientId3, kTestURL3, kTestTitle3}); - RunTask(std::make_unique<AddUniqueUrlsTask>(dispatcher(), store(), - kTestNamespace, urls)); - EXPECT_EQ(1, dispatcher()->task_schedule_count); - - // ZombifyPrefetchItem returns the number of affected items. - EXPECT_EQ(1, store_util()->ZombifyPrefetchItems(kTestNamespace, urls[0].url)); - EXPECT_EQ(1, store_util()->ZombifyPrefetchItems(kTestNamespace, urls[1].url)); - - urls.clear(); - urls.push_back(PrefetchURL{kClientId1, kTestURL1, kTestTitle1}); - urls.push_back(PrefetchURL{kClientId3, kTestURL3, kTestTitle3}); - urls.push_back(PrefetchURL{kClientId4, kTestURL4, kTestTitle4}); - // ID-1 is expected to stay in zombie state. - // ID-2 is expected to be removed, because it is in zombie state. - // ID-3 is still requested, so it is ignored. - // ID-4 is added. - RunTask(std::make_unique<AddUniqueUrlsTask>(dispatcher(), store(), - kTestNamespace, urls)); - EXPECT_EQ(2, dispatcher()->task_schedule_count); - - std::map<std::string, PrefetchItem> items = GetAllItems(); - ASSERT_EQ(3u, items.size()); - ASSERT_TRUE(items.count(kClientId1) > 0); - EXPECT_EQ(kTestURL1, items[kClientId1].url); - EXPECT_EQ(kTestNamespace, items[kClientId1].client_id.name_space); - EXPECT_EQ(kTestTitle1, items[kClientId1].title); - ASSERT_TRUE(items.count(kClientId3) > 0); + EXPECT_EQ(PrefetchItemState::ZOMBIE, items[kClientId2].state); + EXPECT_LE(start_time, items[kClientId2].creation_time); + EXPECT_GE(start_time + base::TimeDelta::FromSeconds(1), + items[kClientId2].creation_time); + EXPECT_LE(start_time, items[kClientId2].freshness_time); + EXPECT_GE(start_time + base::TimeDelta::FromSeconds(1), + items[kClientId2].freshness_time); + + // Newly suggested ID-3 should be added. + ASSERT_GT(items.count(kClientId3), 0U); EXPECT_EQ(kTestURL3, items[kClientId3].url); EXPECT_EQ(kTestNamespace, items[kClientId3].client_id.name_space); EXPECT_EQ(kTestTitle3, items[kClientId3].title); - ASSERT_TRUE(items.count(kClientId4) > 0); - EXPECT_EQ(kTestURL4, items[kClientId4].url); - EXPECT_EQ(kTestNamespace, items[kClientId4].client_id.name_space); - EXPECT_EQ(kTestTitle4, items[kClientId4].title); + EXPECT_EQ(PrefetchItemState::NEW_REQUEST, items[kClientId3].state); + EXPECT_LE(later_time, items[kClientId3].creation_time); + EXPECT_GE(later_time + base::TimeDelta::FromSeconds(1), + items[kClientId3].creation_time); + EXPECT_LE(later_time, items[kClientId3].freshness_time); + EXPECT_GE(later_time + base::TimeDelta::FromSeconds(1), + items[kClientId3].freshness_time); } } // namespace offline_pages diff --git a/chromium/components/offline_pages/core/prefetch/download_archives_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/download_archives_task.cc index 9db843b1d27..46dd54533fd 100644 --- a/chromium/components/offline_pages/core/prefetch/download_archives_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_archives_task.cc @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/download_archives_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_archives_task.h" #include "base/bind.h" #include "base/guid.h" #include "base/metrics/histogram_macros.h" -#include "base/time/default_clock.h" -#include "base/time/time.h" +#include "base/time/clock.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_page_feature.h" #include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_downloader.h" @@ -76,7 +76,7 @@ bool MarkItemAsDownloading(sql::Database* db, sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindInt(0, static_cast<int>(PrefetchItemState::DOWNLOADING)); statement.BindString(1, guid); - statement.BindInt64(2, store_utils::ToDatabaseTime(base::Time::Now())); + statement.BindInt64(2, store_utils::ToDatabaseTime(OfflineClock()->Now())); statement.BindInt64(3, offline_id); return statement.Run(); } @@ -100,8 +100,7 @@ std::unique_ptr<ItemsToDownload> SelectAndMarkItemsForDownloadSync( return nullptr; } - PrefetchDownloaderQuota downloader_quota(db, - base::DefaultClock::GetInstance()); + PrefetchDownloaderQuota downloader_quota(db, OfflineClock()); int64_t available_quota = downloader_quota.GetAvailableQuotaBytes(); if (available_quota <= 0 && !IsLimitlessPrefetchingEnabled()) return nullptr; @@ -185,7 +184,7 @@ void DownloadArchivesTask::Run() { } prefetch_store_->Execute( - base::BindOnce(SelectAndMarkItemsForDownloadSync), + base::BindOnce(&SelectAndMarkItemsForDownloadSync), base::BindOnce(&DownloadArchivesTask::SendItemsToPrefetchDownloader, weak_ptr_factory_.GetWeakPtr()), std::unique_ptr<ItemsToDownload>()); diff --git a/chromium/components/offline_pages/core/prefetch/download_archives_task.h b/chromium/components/offline_pages/core/prefetch/tasks/download_archives_task.h index c77ca0edb54..1591996e0f1 100644 --- a/chromium/components/offline_pages/core/prefetch/download_archives_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_archives_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_ARCHIVES_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_ARCHIVES_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_ARCHIVES_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_ARCHIVES_TASK_H_ #include <memory> #include <string> @@ -63,4 +63,4 @@ class DownloadArchivesTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_ARCHIVES_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_ARCHIVES_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/download_archives_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/download_archives_task_unittest.cc index 613344a06b5..87a722194f2 100644 --- a/chromium/components/offline_pages/core/prefetch/download_archives_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_archives_task_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/download_archives_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_archives_task.h" #include <algorithm> #include <set> @@ -12,10 +12,10 @@ #include "base/test/metrics/histogram_tester.h" #include "base/test/scoped_feature_list.h" #include "components/offline_pages/core/offline_page_feature.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/store/prefetch_downloader_quota.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_utils.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_downloader.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/download_cleanup_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/download_cleanup_task.cc index 9c58f419012..439caf8a868 100644 --- a/chromium/components/offline_pages/core/prefetch/download_cleanup_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_cleanup_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/download_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_cleanup_task.h" #include <vector> diff --git a/chromium/components/offline_pages/core/prefetch/download_cleanup_task.h b/chromium/components/offline_pages/core/prefetch/tasks/download_cleanup_task.h index 9951f36e423..c3cd260e6a8 100644 --- a/chromium/components/offline_pages/core/prefetch/download_cleanup_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_cleanup_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_CLEANUP_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_CLEANUP_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_CLEANUP_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_CLEANUP_TASK_H_ #include <map> #include <set> @@ -52,4 +52,4 @@ class DownloadCleanupTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_CLEANUP_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_CLEANUP_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/download_cleanup_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/download_cleanup_task_unittest.cc index 57ff9b87d78..1bb47e2206c 100644 --- a/chromium/components/offline_pages/core/prefetch/download_cleanup_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_cleanup_task_unittest.cc @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/download_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_cleanup_task.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "sql/database.h" #include "sql/statement.h" diff --git a/chromium/components/offline_pages/core/prefetch/download_completed_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/download_completed_task.cc index b6b7e68e1ed..e64dee6fba0 100644 --- a/chromium/components/offline_pages/core/prefetch/download_completed_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_completed_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/download_completed_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_completed_task.h" #include "base/bind.h" #include "base/callback.h" @@ -28,7 +28,7 @@ enum class DownloadOutcome { DOWNLOAD_FAILED_ITEM_UPDATED, DOWNLOAD_SUCCEEDED_ITEM_NOT_FOUND, DOWNLOAD_FAILED_ITEM_NOT_FOUND, - COUNT // Must always be the last element. + kMaxValue = DOWNLOAD_FAILED_ITEM_NOT_FOUND, }; DownloadOutcome GetDownloadOutcome(bool successful_download, @@ -161,7 +161,7 @@ void DownloadCompletedTask::OnPrefetchItemUpdated(bool successful_download, DownloadOutcome status = GetDownloadOutcome(successful_download, update_info.success); UMA_HISTOGRAM_ENUMERATION("OfflinePages.Prefetching.DownloadFinishedUpdate", - status, DownloadOutcome::COUNT); + status); TaskComplete(); } diff --git a/chromium/components/offline_pages/core/prefetch/download_completed_task.h b/chromium/components/offline_pages/core/prefetch/tasks/download_completed_task.h index 8a45d7d43eb..d7ef7906ed7 100644 --- a/chromium/components/offline_pages/core/prefetch/download_completed_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_completed_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_COMPLETED_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_COMPLETED_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_COMPLETED_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_COMPLETED_TASK_H_ #include "base/macros.h" #include "base/memory/weak_ptr.h" @@ -45,4 +45,4 @@ class DownloadCompletedTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_DOWNLOAD_COMPLETED_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_DOWNLOAD_COMPLETED_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/download_completed_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/download_completed_task_unittest.cc index 829fc741990..b2a5da6bf3e 100644 --- a/chromium/components/offline_pages/core/prefetch/download_completed_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/download_completed_task_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/download_completed_task.h" +#include "components/offline_pages/core/prefetch/tasks/download_completed_task.h" #include <string> #include <vector> @@ -13,11 +13,11 @@ #include "base/time/time.h" #include "components/offline_pages/core/client_namespace_constants.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_utils.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.cc index ed2ad824d13..b5b7d30a06b 100644 --- a/chromium/components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.h" +#include "components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.h" #include <memory> diff --git a/chromium/components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.h b/chromium/components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.h index 22a1f8bca6e..9a2b0c9d6c1 100644 --- a/chromium/components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_FINALIZE_DISMISSED_URL_SUGGESTION_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_FINALIZE_DISMISSED_URL_SUGGESTION_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_FINALIZE_DISMISSED_URL_SUGGESTION_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_FINALIZE_DISMISSED_URL_SUGGESTION_TASK_H_ #include <array> @@ -47,4 +47,4 @@ class FinalizeDismissedUrlSuggestionTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_FINALIZE_DISMISSED_URL_SUGGESTION_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_FINALIZE_DISMISSED_URL_SUGGESTION_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task_unittest.cc index bc4e38493c0..36d645bac94 100644 --- a/chromium/components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task_unittest.cc @@ -2,17 +2,17 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/finalize_dismissed_url_suggestion_task.h" +#include "components/offline_pages/core/prefetch/tasks/finalize_dismissed_url_suggestion_task.h" #include <array> #include <set> #include <vector> #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { diff --git a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.cc index 4981b71de44..3594d427a05 100644 --- a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.h" +#include "components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.h" #include <utility> diff --git a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.h b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.h index 3f959379e78..269ec25af12 100644 --- a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GENERATE_PAGE_BUNDLE_RECONCILE_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GENERATE_PAGE_BUNDLE_RECONCILE_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GENERATE_PAGE_BUNDLE_RECONCILE_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GENERATE_PAGE_BUNDLE_RECONCILE_TASK_H_ #include "base/memory/weak_ptr.h" #include "components/offline_pages/task/task.h" @@ -40,4 +40,4 @@ class GeneratePageBundleReconcileTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GENERATE_PAGE_BUNDLE_RECONCILE_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GENERATE_PAGE_BUNDLE_RECONCILE_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task_unittest.cc index f8e49a288d2..d3fb188d18b 100644 --- a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/generate_page_bundle_reconcile_task.h" +#include "components/offline_pages/core/prefetch/tasks/generate_page_bundle_reconcile_task.h" #include <memory> #include <string> @@ -10,10 +10,10 @@ #include "base/time/time.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" #include "components/offline_pages/core/prefetch/prefetch_network_request_factory.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "sql/database.h" #include "sql/statement.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.cc index 4cdc5136f58..bf658f04786 100644 --- a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.cc @@ -2,15 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/generate_page_bundle_task.h" +#include "components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.h" #include <utility> #include "base/bind.h" #include "base/callback.h" #include "base/callback_helpers.h" -#include "base/time/default_clock.h" +#include "base/time/clock.h" #include "components/offline_pages/core/client_id.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_gcm_handler.h" #include "components/offline_pages/core/prefetch/prefetch_network_request_factory.h" @@ -53,9 +54,7 @@ struct FetchedUrl { // This is maximum URLs that Offline Page Service can take in one request. const int kMaxUrlsToSend = 100; -bool UpdateStateSync(sql::Database* db, - const int64_t offline_id, - base::Clock* clock) { +bool UpdateStateSync(sql::Database* db, const int64_t offline_id) { static const char kSql[] = "UPDATE prefetch_items" " SET state = ?," @@ -66,7 +65,7 @@ bool UpdateStateSync(sql::Database* db, sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindInt( 0, static_cast<int>(PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE)); - statement.BindInt64(1, store_utils::ToDatabaseTime(clock->Now())); + statement.BindInt64(1, store_utils::ToDatabaseTime(OfflineClock()->Now())); statement.BindInt64(2, offline_id); return statement.Run(); } @@ -109,8 +108,7 @@ bool MarkUrlFinishedWithError(sql::Database* db, const FetchedUrl& url) { return statement.Run(); } -std::unique_ptr<UrlAndIds> SelectUrlsToPrefetchSync(base::Clock* clock, - sql::Database* db) { +std::unique_ptr<UrlAndIds> SelectUrlsToPrefetchSync(sql::Database* db) { sql::Transaction transaction(db); if (!transaction.Begin()) return nullptr; @@ -131,7 +129,7 @@ std::unique_ptr<UrlAndIds> SelectUrlsToPrefetchSync(base::Clock* clock, auto url_and_ids = std::make_unique<UrlAndIds>(); for (const auto& url : *urls) { - if (!UpdateStateSync(db, url.offline_id, clock)) + if (!UpdateStateSync(db, url.offline_id)) return nullptr; url_and_ids->urls.push_back(std::move(url.requested_url)); url_and_ids->ids.push_back({url.offline_id, std::move(url.client_id)}); @@ -150,8 +148,7 @@ GeneratePageBundleTask::GeneratePageBundleTask( PrefetchGCMHandler* gcm_handler, PrefetchNetworkRequestFactory* request_factory, PrefetchRequestFinishedCallback callback) - : clock_(base::DefaultClock::GetInstance()), - prefetch_dispatcher_(prefetch_dispatcher), + : prefetch_dispatcher_(prefetch_dispatcher), prefetch_store_(prefetch_store), gcm_handler_(gcm_handler), request_factory_(request_factory), @@ -162,16 +159,12 @@ GeneratePageBundleTask::~GeneratePageBundleTask() {} void GeneratePageBundleTask::Run() { prefetch_store_->Execute( - base::BindOnce(&SelectUrlsToPrefetchSync, clock_), + base::BindOnce(&SelectUrlsToPrefetchSync), base::BindOnce(&GeneratePageBundleTask::StartGeneratePageBundle, weak_factory_.GetWeakPtr()), std::unique_ptr<UrlAndIds>()); } -void GeneratePageBundleTask::SetClockForTesting(base::Clock* clock) { - clock_ = clock; -} - void GeneratePageBundleTask::StartGeneratePageBundle( std::unique_ptr<UrlAndIds> url_and_ids) { if (!url_and_ids) { diff --git a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_task.h b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.h index 5dd6ce1aa9b..62c6a6ea450 100644 --- a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.h @@ -2,15 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GENERATE_PAGE_BUNDLE_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GENERATE_PAGE_BUNDLE_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GENERATE_PAGE_BUNDLE_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GENERATE_PAGE_BUNDLE_TASK_H_ #include <memory> #include <string> #include <vector> #include "base/memory/weak_ptr.h" -#include "base/time/clock.h" #include "components/gcm_driver/instance_id/instance_id.h" #include "components/offline_pages/core/prefetch/prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" @@ -37,16 +36,12 @@ class GeneratePageBundleTask : public Task { // Task implementation. void Run() override; - void SetClockForTesting(base::Clock* clock); - private: void StartGeneratePageBundle(std::unique_ptr<UrlAndIds> url_and_ids); void GotRegistrationId(std::unique_ptr<UrlAndIds> url_and_ids, const std::string& id, instance_id::InstanceID::Result result); - base::Clock* clock_; - PrefetchDispatcher* prefetch_dispatcher_; PrefetchStore* prefetch_store_; PrefetchGCMHandler* gcm_handler_; @@ -59,4 +54,4 @@ class GeneratePageBundleTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GENERATE_PAGE_BUNDLE_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GENERATE_PAGE_BUNDLE_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_task_unittest.cc index 0d0df142657..0c68d25be43 100644 --- a/chromium/components/offline_pages/core/prefetch/generate_page_bundle_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/generate_page_bundle_task_unittest.cc @@ -2,22 +2,22 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/generate_page_bundle_task.h" +#include "components/offline_pages/core/prefetch/tasks/generate_page_bundle_task.h" #include <utility> #include "base/logging.h" #include "base/test/mock_callback.h" -#include "base/test/simple_test_clock.h" #include "base/time/time.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_utils.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/test_prefetch_gcm_handler.h" +#include "components/offline_pages/core/test_scoped_offline_clock.h" #include "components/offline_pages/task/task.h" #include "services/network/test/test_utils.h" #include "testing/gmock/include/gmock/gmock.h" @@ -71,7 +71,7 @@ TEST_F(GeneratePageBundleTaskTest, EmptyTask) { TEST_F(GeneratePageBundleTaskTest, TaskMakesNetworkRequest) { base::MockCallback<PrefetchRequestFinishedCallback> request_callback; - base::SimpleTestClock clock; + TestScopedOfflineClock clock; // This item will be sent with the bundle request. PrefetchItem item1 = @@ -106,7 +106,6 @@ TEST_F(GeneratePageBundleTaskTest, TaskMakesNetworkRequest) { GeneratePageBundleTask task(dispatcher(), store(), gcm_handler(), prefetch_request_factory(), request_callback.Get()); - task.SetClockForTesting(&clock); RunTask(&task); // Note: even though the requested URLs checked further below are in undefined diff --git a/chromium/components/offline_pages/core/prefetch/get_operation_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/get_operation_task.cc index 3569987e5eb..a58428f9fc4 100644 --- a/chromium/components/offline_pages/core/prefetch/get_operation_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/get_operation_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/get_operation_task.h" +#include "components/offline_pages/core/prefetch/tasks/get_operation_task.h" #include <memory> #include <utility> diff --git a/chromium/components/offline_pages/core/prefetch/get_operation_task.h b/chromium/components/offline_pages/core/prefetch/tasks/get_operation_task.h index 10d2e966c1b..58e554769bc 100644 --- a/chromium/components/offline_pages/core/prefetch/get_operation_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/get_operation_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GET_OPERATION_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GET_OPERATION_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GET_OPERATION_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GET_OPERATION_TASK_H_ #include <memory> #include <string> @@ -46,4 +46,4 @@ class GetOperationTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_GET_OPERATION_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GET_OPERATION_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/get_operation_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/get_operation_task_unittest.cc index 3da268855a5..b81609ff881 100644 --- a/chromium/components/offline_pages/core/prefetch/get_operation_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/get_operation_task_unittest.cc @@ -2,12 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/get_operation_task.h" +#include "components/offline_pages/core/prefetch/tasks/get_operation_task.h" #include "base/test/mock_callback.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_gcm_handler.h" #include "components/offline_pages/task/task.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.cc new file mode 100644 index 00000000000..00a170778a1 --- /dev/null +++ b/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.cc @@ -0,0 +1,56 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.h" + +#include <utility> + +#include "components/offline_pages/core/prefetch/store/prefetch_store.h" +#include "sql/database.h" +#include "sql/statement.h" + +namespace offline_pages { +namespace { + +GetThumbnailInfoTask::Result GetThumbnailInfoSync(const int64_t offline_id, + sql::Database* db) { + static const char kSql[] = R"(SELECT thumbnail_url FROM prefetch_items + WHERE offline_id=?;)"; + + sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); + DCHECK(statement.is_valid()); + + GetThumbnailInfoTask::Result result; + statement.BindInt64(0, offline_id); + if (statement.Step()) + result.thumbnail_url = GURL(statement.ColumnString(0)); + + return result; +} + +} // namespace + +GetThumbnailInfoTask::GetThumbnailInfoTask(PrefetchStore* store, + const int64_t offline_id, + ResultCallback callback) + : prefetch_store_(store), + offline_id_(offline_id), + callback_(std::move(callback)) {} + +GetThumbnailInfoTask::~GetThumbnailInfoTask() = default; + +void GetThumbnailInfoTask::Run() { + prefetch_store_->Execute( + base::BindOnce(GetThumbnailInfoSync, offline_id_), + base::BindOnce(&GetThumbnailInfoTask::CompleteTaskAndForwardResult, + weak_factory_.GetWeakPtr()), + Result()); +} + +void GetThumbnailInfoTask::CompleteTaskAndForwardResult(Result result) { + TaskComplete(); + std::move(callback_).Run(result); +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.h b/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.h new file mode 100644 index 00000000000..e3ee3b5d7f2 --- /dev/null +++ b/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.h @@ -0,0 +1,51 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GET_THUMBNAIL_INFO_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GET_THUMBNAIL_INFO_TASK_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "base/memory/weak_ptr.h" +#include "components/offline_pages/task/task.h" +#include "url/gurl.h" + +namespace offline_pages { +class PrefetchStore; + +// Task that attempts to get thumbnail information about an offline item in the +// prefetch store. +class GetThumbnailInfoTask : public Task { + public: + struct Result { + // The thumbnail URL of the offline item. This is empty if the offline item + // was not found, or if the item had no thumbnail URL. + GURL thumbnail_url; + }; + using ResultCallback = base::OnceCallback<void(Result)>; + + GetThumbnailInfoTask(PrefetchStore* store, + const int64_t offline_id, + ResultCallback callback); + ~GetThumbnailInfoTask() override; + + // Task implementation. + void Run() override; + + private: + void CompleteTaskAndForwardResult(Result result); + PrefetchStore* prefetch_store_; + int64_t offline_id_; + ResultCallback callback_; + + base::WeakPtrFactory<GetThumbnailInfoTask> weak_factory_{this}; + + DISALLOW_COPY_AND_ASSIGN(GetThumbnailInfoTask); +}; + +} // namespace offline_pages + +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_GET_THUMBNAIL_INFO_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task_unittest.cc new file mode 100644 index 00000000000..1d93c31c82e --- /dev/null +++ b/chromium/components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task_unittest.cc @@ -0,0 +1,58 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/prefetch/tasks/get_thumbnail_info_task.h" + +#include "base/test/mock_callback.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace offline_pages { +namespace { +const char kTestUrl[] = "https://www.test_thumbnail.com/"; + +class GetThumbnailInfoTaskTest : public PrefetchTaskTestBase { + public: + GetThumbnailInfoTaskTest() = default; + ~GetThumbnailInfoTaskTest() override = default; + + PrefetchItem TestItem() { + PrefetchItem item = + item_generator()->CreateItem(PrefetchItemState::DOWNLOADED); + item.thumbnail_url = GURL(kTestUrl); + return item; + } +}; + +MATCHER(IsNullResult, "") { + return arg.thumbnail_url.is_empty(); +} + +MATCHER(IsTestUrl, "") { + return arg.thumbnail_url.possibly_invalid_spec() == kTestUrl; +} + +TEST_F(GetThumbnailInfoTaskTest, NotPresent) { + const PrefetchItem item = TestItem(); + store_util()->InsertPrefetchItem(item); + + base::MockCallback<GetThumbnailInfoTask::ResultCallback> callback; + EXPECT_CALL(callback, Run(IsNullResult())); + RunTask(std::make_unique<GetThumbnailInfoTask>(store(), item.offline_id + 1, + callback.Get())); +} + +TEST_F(GetThumbnailInfoTaskTest, Found) { + const PrefetchItem item = TestItem(); + store_util()->InsertPrefetchItem(item); + + base::MockCallback<GetThumbnailInfoTask::ResultCallback> callback; + EXPECT_CALL(callback, Run(IsTestUrl())); + RunTask(std::make_unique<GetThumbnailInfoTask>(store(), item.offline_id, + callback.Get())); +} + +} // namespace +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/prefetch/import_archives_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/import_archives_task.cc index 8af2196cbff..84fa73ca1bc 100644 --- a/chromium/components/offline_pages/core/prefetch/import_archives_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_archives_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/import_archives_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_archives_task.h" #include <memory> diff --git a/chromium/components/offline_pages/core/prefetch/import_archives_task.h b/chromium/components/offline_pages/core/prefetch/tasks/import_archives_task.h index 5a741f792d5..a6df6a6f58c 100644 --- a/chromium/components/offline_pages/core/prefetch/import_archives_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_archives_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_ARCHIVES_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_ARCHIVES_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_ARCHIVES_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_ARCHIVES_TASK_H_ #include <vector> @@ -40,4 +40,4 @@ class ImportArchivesTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_ARCHIVES_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_ARCHIVES_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/import_archives_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/import_archives_task_unittest.cc index 70f32d8285f..31e46fea5c8 100644 --- a/chromium/components/offline_pages/core/prefetch/import_archives_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_archives_task_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/import_archives_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_archives_task.h" #include <string> #include <vector> @@ -13,11 +13,11 @@ #include "base/time/time.h" #include "components/offline_pages/core/prefetch/prefetch_importer.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_utils.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "sql/database.h" #include "sql/statement.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/import_cleanup_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/import_cleanup_task.cc index e0c99994a40..4a498a35b16 100644 --- a/chromium/components/offline_pages/core/prefetch/import_cleanup_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_cleanup_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/import_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_cleanup_task.h" #include <algorithm> diff --git a/chromium/components/offline_pages/core/prefetch/import_cleanup_task.h b/chromium/components/offline_pages/core/prefetch/tasks/import_cleanup_task.h index 583a75abde4..56cd164c472 100644 --- a/chromium/components/offline_pages/core/prefetch/import_cleanup_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_cleanup_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_CLEANUP_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_CLEANUP_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_CLEANUP_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_CLEANUP_TASK_H_ #include "base/macros.h" #include "base/memory/weak_ptr.h" @@ -37,4 +37,4 @@ class ImportCleanupTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_CLEANUP_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_CLEANUP_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/import_cleanup_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/import_cleanup_task_unittest.cc index b79d66018cc..b75a2ac22be 100644 --- a/chromium/components/offline_pages/core/prefetch/import_cleanup_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_cleanup_task_unittest.cc @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/import_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_cleanup_task.h" #include "components/offline_pages/core/prefetch/prefetch_importer.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { diff --git a/chromium/components/offline_pages/core/prefetch/import_completed_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/import_completed_task.cc index 19316622435..4bc12a904bf 100644 --- a/chromium/components/offline_pages/core/prefetch/import_completed_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_completed_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/import_completed_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_completed_task.h" #include "base/bind.h" #include "base/callback.h" diff --git a/chromium/components/offline_pages/core/prefetch/import_completed_task.h b/chromium/components/offline_pages/core/prefetch/tasks/import_completed_task.h index d7ebefd7b21..0d7445c8e83 100644 --- a/chromium/components/offline_pages/core/prefetch/import_completed_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_completed_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_COMPLETED_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_COMPLETED_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_COMPLETED_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_COMPLETED_TASK_H_ #include "base/macros.h" #include "base/memory/weak_ptr.h" @@ -31,8 +31,8 @@ class ImportCompletedTask : public Task { void OnStateUpdatedToFinished(bool success); PrefetchDispatcher* prefetch_dispatcher_; // Outlives this class. - PrefetchStore* prefetch_store_; // Outlives this class. - PrefetchImporter* prefetch_importer_; // Outlives this class. + PrefetchStore* prefetch_store_; // Outlives this class. + PrefetchImporter* prefetch_importer_; // Outlives this class. int64_t offline_id_; bool success_; @@ -43,4 +43,4 @@ class ImportCompletedTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_IMPORT_COMPLETED_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_IMPORT_COMPLETED_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/import_completed_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/import_completed_task_unittest.cc index fe8d7912314..9739c79aef2 100644 --- a/chromium/components/offline_pages/core/prefetch/import_completed_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/import_completed_task_unittest.cc @@ -2,11 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/import_completed_task.h" +#include "components/offline_pages/core/prefetch/tasks/import_completed_task.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/test_prefetch_importer.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/mark_operation_done_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/mark_operation_done_task.cc index 7ad289faa3a..26497dca7d7 100644 --- a/chromium/components/offline_pages/core/prefetch/mark_operation_done_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/mark_operation_done_task.cc @@ -2,13 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/mark_operation_done_task.h" +#include "components/offline_pages/core/prefetch/tasks/mark_operation_done_task.h" #include <memory> #include <utility> #include "base/bind.h" #include "base/callback.h" +#include "base/time/clock.h" +#include "components/offline_pages/core/offline_clock.h" +#include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/prefetch_network_request_factory.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" @@ -28,7 +31,7 @@ bool UpdatePrefetchItemsSync(sql::Database* db, sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindInt(0, static_cast<int>(PrefetchItemState::RECEIVED_GCM)); - statement.BindInt64(1, base::Time::Now().ToInternalValue()); + statement.BindInt64(1, store_utils::ToDatabaseTime(OfflineClock()->Now())); statement.BindInt(2, static_cast<int>(PrefetchItemState::AWAITING_GCM)); statement.BindString(3, operation_name); diff --git a/chromium/components/offline_pages/core/prefetch/mark_operation_done_task.h b/chromium/components/offline_pages/core/prefetch/tasks/mark_operation_done_task.h index c5f249ade4c..8c89752820c 100644 --- a/chromium/components/offline_pages/core/prefetch/mark_operation_done_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/mark_operation_done_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_MARK_OPERATION_DONE_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_MARK_OPERATION_DONE_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_MARK_OPERATION_DONE_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_MARK_OPERATION_DONE_TASK_H_ #include <string> #include <vector> @@ -65,4 +65,4 @@ class MarkOperationDoneTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_MARK_OPERATION_DONE_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_MARK_OPERATION_DONE_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/mark_operation_done_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/mark_operation_done_task_unittest.cc index 6a5dd7967a8..3a08148e640 100644 --- a/chromium/components/offline_pages/core/prefetch/mark_operation_done_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/mark_operation_done_task_unittest.cc @@ -2,18 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/mark_operation_done_task.h" +#include "components/offline_pages/core/prefetch/tasks/mark_operation_done_task.h" #include <set> #include <string> #include <vector> #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_utils.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/test_prefetch_gcm_handler.h" #include "components/offline_pages/task/task.h" @@ -33,8 +33,6 @@ class MarkOperationDoneTaskTest : public PrefetchTaskTestBase { MarkOperationDoneTaskTest() = default; ~MarkOperationDoneTaskTest() override = default; - void SetUp() override { PrefetchTaskTestBase::SetUp(); } - int64_t InsertAwaitingGCMOperation(std::string name) { return InsertPrefetchItemInStateWithOperation( name, PrefetchItemState::AWAITING_GCM); diff --git a/chromium/components/offline_pages/core/prefetch/metrics_finalization_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/metrics_finalization_task.cc index 10ef18b1714..60fb7305839 100644 --- a/chromium/components/offline_pages/core/prefetch/metrics_finalization_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/metrics_finalization_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/metrics_finalization_task.h" +#include "components/offline_pages/core/prefetch/tasks/metrics_finalization_task.h" #include <algorithm> #include <memory> @@ -13,6 +13,9 @@ #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" #include "base/metrics/sparse_histogram.h" +#include "base/time/clock.h" +#include "base/time/time.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" @@ -74,8 +77,7 @@ std::vector<PrefetchItemStats> FetchUrlsSync(sql::Database* db) { statement.ColumnInt64(5)), // creation_time static_cast<PrefetchItemErrorCode>( statement.ColumnInt(6)), // error_code - statement.ColumnInt64(7) // file_size - ); + statement.ColumnInt64(7)); // file_size } return urls; @@ -89,7 +91,7 @@ bool MarkUrlAsZombie(sql::Database* db, "offline_id = ?"; sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindInt(0, static_cast<int>(PrefetchItemState::ZOMBIE)); - statement.BindInt(1, store_utils::ToDatabaseTime(freshness_time)); + statement.BindInt64(1, store_utils::ToDatabaseTime(freshness_time)); statement.BindInt64(2, offline_id); return statement.Run(); } @@ -160,7 +162,7 @@ bool ReportMetricsAndFinalizeSync(sql::Database* db) { const std::vector<PrefetchItemStats> urls = FetchUrlsSync(db); - base::Time now = base::Time::Now(); + base::Time now = OfflineClock()->Now(); for (const auto& url : urls) { MarkUrlAsZombie(db, now, url.offline_id); } diff --git a/chromium/components/offline_pages/core/prefetch/metrics_finalization_task.h b/chromium/components/offline_pages/core/prefetch/tasks/metrics_finalization_task.h index df9d961751e..93edc2ddb2b 100644 --- a/chromium/components/offline_pages/core/prefetch/metrics_finalization_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/metrics_finalization_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_METRICS_FINALIZATION_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_METRICS_FINALIZATION_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_METRICS_FINALIZATION_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_METRICS_FINALIZATION_TASK_H_ #include "base/macros.h" #include "base/memory/weak_ptr.h" @@ -12,10 +12,9 @@ namespace offline_pages { class PrefetchStore; -// Prefetching task that takes finished PrefetchItems, records interesting -// metrics about the final status, and marks them as zombies. Zombies are -// cleaned up when suggestions are updated and there are no more -// suggestions at the |requested_url|. +// Prefetching task that takes finished prefetch items, records interesting +// metrics about their final status, and marks them as zombies. Zombies are +// cleaned after a set period of time by the |StaleEntryFinalizerTask|. // NOTE: this task is run periodically as reconciliation task or from some // event handlers. It should not cause 'progress' in pipeline on which other // tasks would depend. It should only move entries to ZOMBIE state. @@ -38,4 +37,4 @@ class MetricsFinalizationTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_METRICS_FINALIZATION_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_METRICS_FINALIZATION_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/metrics_finalization_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/metrics_finalization_task_unittest.cc index 58b9a0f533e..6a7bbe767db 100644 --- a/chromium/components/offline_pages/core/prefetch/metrics_finalization_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/metrics_finalization_task_unittest.cc @@ -2,17 +2,20 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/metrics_finalization_task.h" +#include "components/offline_pages/core/prefetch/tasks/metrics_finalization_task.h" #include <memory> #include <set> +#include <vector> #include "base/test/metrics/histogram_tester.h" +#include "base/time/time.h" #include "components/offline_pages/core/prefetch/mock_prefetch_item_generator.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" +#include "components/offline_pages/core/test_scoped_offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { @@ -78,28 +81,42 @@ TEST_F(MetricsFinalizationTaskTest, LeavesOtherStatesAlone) { } TEST_F(MetricsFinalizationTaskTest, FinalizesMultipleItems) { + base::Time before_insert_time = base::Time::Now(); std::set<PrefetchItem> finished_items = { item_generator()->CreateItem(PrefetchItemState::FINISHED), item_generator()->CreateItem(PrefetchItemState::FINISHED), item_generator()->CreateItem(PrefetchItemState::FINISHED)}; - - PrefetchItem unfinished_item = - item_generator()->CreateItem(PrefetchItemState::NEW_REQUEST); - for (auto& item : finished_items) { ASSERT_TRUE(store_util()->InsertPrefetchItem(item)); + // Confirms that ItemGenerator did set |freshness_time| with Time::Now(). + ASSERT_LE(before_insert_time, item.freshness_time); } + + PrefetchItem unfinished_item = + item_generator()->CreateItem(PrefetchItemState::NEW_REQUEST); ASSERT_TRUE(store_util()->InsertPrefetchItem(unfinished_item)); + // Overrides the offline clock and set a current time in the future. + TestScopedOfflineClock clock; + clock.SetNow(before_insert_time + base::TimeDelta::FromHours(1)); + // Execute the metrics task. RunTask(metrics_finalization_task_.get()); + // The finished ones should all have become zombies and the new request should + // be untouched. std::set<PrefetchItem> all_items; - // The finished ones should be zombies and the new request should be - // untouched. EXPECT_EQ(4U, store_util()->GetAllItems(&all_items)); EXPECT_EQ(0U, FilterByState(all_items, PrefetchItemState::FINISHED).size()); - EXPECT_EQ(3U, FilterByState(all_items, PrefetchItemState::ZOMBIE).size()); + + std::set<PrefetchItem> zombie_items = + FilterByState(all_items, PrefetchItemState::ZOMBIE); + EXPECT_EQ(3U, zombie_items.size()); + for (const PrefetchItem& zombie_item : zombie_items) { + EXPECT_EQ(clock.Now(), zombie_item.freshness_time) + << "Incorrect freshness_time (not updated?) for item " + << zombie_item.client_id; + } std::set<PrefetchItem> items_in_new_request_state = FilterByState(all_items, PrefetchItemState::NEW_REQUEST); diff --git a/chromium/components/offline_pages/core/prefetch/page_bundle_update_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/page_bundle_update_task.cc index e535730849d..b3cf8b27e74 100644 --- a/chromium/components/offline_pages/core/prefetch/page_bundle_update_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/page_bundle_update_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/page_bundle_update_task.h" +#include "components/offline_pages/core/prefetch/tasks/page_bundle_update_task.h" #include <memory> #include <string> diff --git a/chromium/components/offline_pages/core/prefetch/page_bundle_update_task.h b/chromium/components/offline_pages/core/prefetch/tasks/page_bundle_update_task.h index 8cb27b37e2c..8781978f9c9 100644 --- a/chromium/components/offline_pages/core/prefetch/page_bundle_update_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/page_bundle_update_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_PAGE_BUNDLE_UPDATE_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_PAGE_BUNDLE_UPDATE_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_PAGE_BUNDLE_UPDATE_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_PAGE_BUNDLE_UPDATE_TASK_H_ #include <string> #include <vector> @@ -65,4 +65,4 @@ class PageBundleUpdateTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_PAGE_BUNDLE_UPDATE_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_PAGE_BUNDLE_UPDATE_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/page_bundle_update_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/page_bundle_update_task_unittest.cc index 17c4acbd89a..6cff4b9b47d 100644 --- a/chromium/components/offline_pages/core/prefetch/page_bundle_update_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/page_bundle_update_task_unittest.cc @@ -2,16 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/page_bundle_update_task.h" +#include "components/offline_pages/core/prefetch/tasks/page_bundle_update_task.h" #include "base/logging.h" #include "base/test/mock_callback.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_utils.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" #include "components/offline_pages/task/task.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_task_test_base.cc b/chromium/components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.cc index ecc3bae71cb..547a5d49aff 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_task_test_base.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_prefs.h" diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_task_test_base.h b/chromium/components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h index 238a0565fac..bdb5ecc891b 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_task_test_base.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASK_TEST_BASE_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASK_TEST_BASE_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_PREFETCH_TASK_TEST_BASE_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_PREFETCH_TASK_TEST_BASE_H_ #include <memory> #include <set> @@ -82,4 +82,4 @@ class PrefetchTaskTestBase : public TaskTestBase { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASK_TEST_BASE_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_PREFETCH_TASK_TEST_BASE_H_ diff --git a/chromium/components/offline_pages/core/prefetch/prefetch_task_test_base_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/prefetch_task_test_base_unittest.cc index caa28a136e2..029708e6b28 100644 --- a/chromium/components/offline_pages/core/prefetch/prefetch_task_test_base_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/prefetch_task_test_base_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include <algorithm> #include <set> @@ -22,8 +22,9 @@ TEST(PrefetchTaskTestBaseTest, StateEnumIsFullyRepresentedInOrderedArray) { // build error. When the new element is then added here, the test will fail // until it is properly added to |kOrderedPrefetchItemStates|. // Note: this code assumes that the minimum assigned value in the enum is 0 - // and that the maximum is correctly represented by the MAX labeled element. - for (int i = 0; i <= static_cast<int>(PrefetchItemState::MAX); ++i) { + // and that the maximum is correctly represented by the kMaxValue labeled + // element. + for (int i = 0; i <= static_cast<int>(PrefetchItemState::kMaxValue); ++i) { PrefetchItemState maybe_valid_state = static_cast<PrefetchItemState>(i); switch (maybe_valid_state) { case PrefetchItemState::NEW_REQUEST: diff --git a/chromium/components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.cc index 8d37bb2f3c0..122fa3c13fa 100644 --- a/chromium/components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.h" #include <memory> #include <set> diff --git a/chromium/components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.h b/chromium/components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.h index 572874fe665..0bfa93979f5 100644 --- a/chromium/components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_SENT_GET_OPERATION_CLEANUP_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_SENT_GET_OPERATION_CLEANUP_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_SENT_GET_OPERATION_CLEANUP_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_SENT_GET_OPERATION_CLEANUP_TASK_H_ #include "base/macros.h" #include "base/memory/weak_ptr.h" @@ -39,4 +39,4 @@ class SentGetOperationCleanupTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_SENT_GET_OPERATION_CLEANUP_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_SENT_GET_OPERATION_CLEANUP_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/sent_get_operation_cleanup_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task_unittest.cc index d3df58c410c..cc3515dc12f 100644 --- a/chromium/components/offline_pages/core/prefetch/sent_get_operation_cleanup_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/sent_get_operation_cleanup_task.h" +#include "components/offline_pages/core/prefetch/tasks/sent_get_operation_cleanup_task.h" #include <memory> #include <string> @@ -10,10 +10,10 @@ #include "base/time/time.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" #include "components/offline_pages/core/prefetch/prefetch_network_request_factory.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "sql/database.h" #include "sql/statement.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/components/offline_pages/core/prefetch/stale_entry_finalizer_task.cc b/chromium/components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.cc index 6fe1fc22737..1980a5582c7 100644 --- a/chromium/components/offline_pages/core/prefetch/stale_entry_finalizer_task.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.cc @@ -2,12 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/stale_entry_finalizer_task.h" +#include "components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.h" #include <array> #include "base/bind.h" #include "base/metrics/histogram_functions.h" +#include "base/time/clock.h" +#include "base/time/time.h" +#include "components/offline_pages/core/offline_clock.h" #include "components/offline_pages/core/offline_store_utils.h" #include "components/offline_pages/core/prefetch/prefetch_dispatcher.h" #include "components/offline_pages/core/prefetch/prefetch_downloader.h" @@ -23,6 +26,14 @@ using Result = StaleEntryFinalizerTask::Result; namespace { +// Maximum amount of time into the future an item can has its freshness time set +// to after which it will be finalized (or deleted if in the zombie state). +constexpr base::TimeDelta kFutureItemTimeLimit = base::TimeDelta::FromDays(1); + +// Expiration time delay for items entering the zombie state, after which they +// are permanently deleted. +constexpr base::TimeDelta kZombieItemLifetime = base::TimeDelta::FromDays(7); + // If this time changes, we need to update the desciption in histograms.xml // for OfflinePages.Prefetching.StuckItemState. const int kStuckTimeLimitInDays = 7; @@ -122,7 +133,7 @@ bool FinalizeFutureItems(PrefetchItemState state, "UPDATE prefetch_items SET state = ?, error_code = ?" " WHERE state = ? AND freshness_time > ?"; const int64_t future_fresh_db_time_limit = - store_utils::ToDatabaseTime(now + base::TimeDelta::FromDays(1)); + store_utils::ToDatabaseTime(now + kFutureItemTimeLimit); sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); statement.BindInt(0, static_cast<int>(PrefetchItemState::FINISHED)); statement.BindInt( @@ -134,6 +145,22 @@ bool FinalizeFutureItems(PrefetchItemState state, return statement.Run(); } +bool DeleteExpiredAndFutureZombies(base::Time now, sql::Database* db) { + static const char kSql[] = + "DELETE FROM prefetch_items" + " WHERE state = ? " + " AND (freshness_time < ? OR freshness_time > ?)"; + const int64_t earliest_zombie_db_time = + store_utils::ToDatabaseTime(now - kZombieItemLifetime); + const int64_t future_zombie_db_time = + store_utils::ToDatabaseTime(now + kFutureItemTimeLimit); + sql::Statement statement(db->GetCachedStatement(SQL_FROM_HERE, kSql)); + statement.BindInt(0, static_cast<int>(PrefetchItemState::ZOMBIE)); + statement.BindInt64(1, earliest_zombie_db_time); + statement.BindInt64(2, future_zombie_db_time); + return statement.Run(); +} + // If there is a bug in our code, an item might be stuck in the queue waiting // on an event that didn't happen. If so, finalize that item and report it. void ReportAndFinalizeStuckItems(base::Time now, sql::Database* db) { @@ -172,8 +199,7 @@ void ReportAndFinalizeStuckItems(base::Time now, sql::Database* db) { } } -Result FinalizeStaleEntriesSync(StaleEntryFinalizerTask::NowGetter now_getter, - sql::Database* db) { +Result FinalizeStaleEntriesSync(sql::Database* db) { sql::Transaction transaction(db); if (!transaction.Begin()) return Result::NO_MORE_WORK; @@ -189,7 +215,7 @@ Result FinalizeStaleEntriesSync(StaleEntryFinalizerTask::NowGetter now_getter, // Bucket 3. PrefetchItemState::DOWNLOADING, PrefetchItemState::IMPORTING, }}; - base::Time now = now_getter.Run(); + base::Time now = OfflineClock()->Now(); for (PrefetchItemState state : expirable_states) { if (!FinalizeStaleItems(state, now, db)) return Result::NO_MORE_WORK; @@ -198,8 +224,12 @@ Result FinalizeStaleEntriesSync(StaleEntryFinalizerTask::NowGetter now_getter, return Result::NO_MORE_WORK; } + if (!DeleteExpiredAndFutureZombies(now, db)) + return Result::NO_MORE_WORK; + // Items could also be stuck in a non-expirable state due to a bug, report - // them. + // them. This should always be the last step, coming after the regular + // freshness maintenance steps above are done. ReportAndFinalizeStuckItems(now, db); Result result = Result::MORE_WORK_NEEDED; @@ -217,7 +247,6 @@ StaleEntryFinalizerTask::StaleEntryFinalizerTask( PrefetchStore* prefetch_store) : prefetch_dispatcher_(prefetch_dispatcher), prefetch_store_(prefetch_store), - now_getter_(base::BindRepeating(&base::Time::Now)), weak_ptr_factory_(this) { DCHECK(prefetch_dispatcher_); DCHECK(prefetch_store_); @@ -226,15 +255,10 @@ StaleEntryFinalizerTask::StaleEntryFinalizerTask( StaleEntryFinalizerTask::~StaleEntryFinalizerTask() {} void StaleEntryFinalizerTask::Run() { - prefetch_store_->Execute( - base::BindOnce(&FinalizeStaleEntriesSync, now_getter_), - base::BindOnce(&StaleEntryFinalizerTask::OnFinished, - weak_ptr_factory_.GetWeakPtr()), - Result::NO_MORE_WORK); -} - -void StaleEntryFinalizerTask::SetNowGetterForTesting(NowGetter now_getter) { - now_getter_ = now_getter; + prefetch_store_->Execute(base::BindOnce(&FinalizeStaleEntriesSync), + base::BindOnce(&StaleEntryFinalizerTask::OnFinished, + weak_ptr_factory_.GetWeakPtr()), + Result::NO_MORE_WORK); } void StaleEntryFinalizerTask::OnFinished(Result result) { diff --git a/chromium/components/offline_pages/core/prefetch/stale_entry_finalizer_task.h b/chromium/components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.h index c8e62916b10..473cf4cd96a 100644 --- a/chromium/components/offline_pages/core/prefetch/stale_entry_finalizer_task.h +++ b/chromium/components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_STALE_ENTRY_FINALIZER_TASK_H_ -#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_STALE_ENTRY_FINALIZER_TASK_H_ +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_STALE_ENTRY_FINALIZER_TASK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_STALE_ENTRY_FINALIZER_TASK_H_ #include <vector> @@ -21,13 +21,14 @@ class PrefetchStore; // considered stale are moved to the "finished" state and have their error code // column set to the PrefetchItemErrorCode value that identifies the bucket they // were at. +// It also handles items in the the "zombie" state which are deleted once +// considered expired after a set amount of time. // NOTE: This task is run periodically as reconciliation task and from some // event handlers. As such, it must not cause network operations nor cause // 'progress' in the pipeline that would trigger other tasks. class StaleEntryFinalizerTask : public Task { public: enum class Result { NO_MORE_WORK, MORE_WORK_NEEDED }; - using NowGetter = base::RepeatingCallback<base::Time()>; StaleEntryFinalizerTask(PrefetchDispatcher* prefetch_dispatcher, PrefetchStore* prefetch_store); @@ -35,10 +36,6 @@ class StaleEntryFinalizerTask : public Task { void Run() override; - // Allows tests to control the source of current time values used internally - // for freshness checks. - void SetNowGetterForTesting(NowGetter now_getter); - // Will be set to true upon after an error-free run. Result final_status() const { return final_status_; } @@ -51,9 +48,6 @@ class StaleEntryFinalizerTask : public Task { // Prefetch store to execute against. Not owned. PrefetchStore* prefetch_store_; - // Defaults to base::Time::Now upon construction. - NowGetter now_getter_; - Result final_status_ = Result::NO_MORE_WORK; base::WeakPtrFactory<StaleEntryFinalizerTask> weak_ptr_factory_; @@ -62,4 +56,4 @@ class StaleEntryFinalizerTask : public Task { } // namespace offline_pages -#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_STALE_ENTRY_FINALIZER_TASK_H_ +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_TASKS_STALE_ENTRY_FINALIZER_TASK_H_ diff --git a/chromium/components/offline_pages/core/prefetch/stale_entry_finalizer_task_unittest.cc b/chromium/components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task_unittest.cc index 0f6bea4e5e2..1bb956d3ca3 100644 --- a/chromium/components/offline_pages/core/prefetch/stale_entry_finalizer_task_unittest.cc +++ b/chromium/components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task_unittest.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/offline_pages/core/prefetch/stale_entry_finalizer_task.h" +#include "components/offline_pages/core/prefetch/tasks/stale_entry_finalizer_task.h" #include <memory> #include <set> @@ -14,17 +14,19 @@ #include "base/threading/thread_task_runner_handle.h" #include "components/offline_pages/core/prefetch/mock_prefetch_item_generator.h" #include "components/offline_pages/core/prefetch/prefetch_item.h" -#include "components/offline_pages/core/prefetch/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/prefetch_types.h" #include "components/offline_pages/core/prefetch/store/prefetch_store_test_util.h" +#include "components/offline_pages/core/prefetch/tasks/prefetch_task_test_base.h" #include "components/offline_pages/core/prefetch/test_prefetch_dispatcher.h" +#include "components/offline_pages/core/test_scoped_offline_clock.h" #include "testing/gtest/include/gtest/gtest.h" namespace offline_pages { using Result = StaleEntryFinalizerTask::Result; -std::set<PrefetchItem> Filter(const std::set<PrefetchItem>& items, +// Return only the items in the provided |state|. +std::set<PrefetchItem> Select(const std::set<PrefetchItem>& items, PrefetchItemState state) { std::set<PrefetchItem> result; for (const PrefetchItem& item : items) { @@ -34,6 +36,17 @@ std::set<PrefetchItem> Filter(const std::set<PrefetchItem>& items, return result; } +// Return only the items with the provided |error_code|. +std::set<PrefetchItem> Select(const std::set<PrefetchItem>& items, + PrefetchItemErrorCode error_code) { + std::set<PrefetchItem> result; + for (const PrefetchItem& item : items) { + if (item.error_code == error_code) + result.insert(item); + } + return result; +} + class StaleEntryFinalizerTaskTest : public PrefetchTaskTestBase { public: StaleEntryFinalizerTaskTest() = default; @@ -42,24 +55,24 @@ class StaleEntryFinalizerTaskTest : public PrefetchTaskTestBase { void SetUp() override; void TearDown() override; - PrefetchItem CreateAndInsertItem(PrefetchItemState state, - int time_delta_in_hours); + PrefetchItem InsertItemWithFreshnessTime(PrefetchItemState state, + int freshness_delta_in_hours); + PrefetchItem InsertItemWithCreationTime(PrefetchItemState state, + int creation_delta_in_hours); TestPrefetchDispatcher* dispatcher() { return &dispatcher_; } protected: TestPrefetchDispatcher dispatcher_; std::unique_ptr<StaleEntryFinalizerTask> stale_finalizer_task_; - base::Time fake_now_; + TestScopedOfflineClock simple_test_clock_; }; void StaleEntryFinalizerTaskTest::SetUp() { PrefetchTaskTestBase::SetUp(); stale_finalizer_task_ = std::make_unique<StaleEntryFinalizerTask>(dispatcher(), store()); - fake_now_ = base::Time() + base::TimeDelta::FromDays(100); - stale_finalizer_task_->SetNowGetterForTesting(base::BindRepeating( - [](base::Time t) -> base::Time { return t; }, fake_now_)); + simple_test_clock_.SetNow(base::Time() + base::TimeDelta::FromDays(100)); } void StaleEntryFinalizerTaskTest::TearDown() { @@ -67,13 +80,25 @@ void StaleEntryFinalizerTaskTest::TearDown() { PrefetchTaskTestBase::TearDown(); } -PrefetchItem StaleEntryFinalizerTaskTest::CreateAndInsertItem( +PrefetchItem StaleEntryFinalizerTaskTest::InsertItemWithFreshnessTime( + PrefetchItemState state, + int freshness_delta_in_hours) { + PrefetchItem item(item_generator()->CreateItem(state)); + item.freshness_time = simple_test_clock_.Now() + + base::TimeDelta::FromHours(freshness_delta_in_hours); + item.creation_time = simple_test_clock_.Now(); + EXPECT_TRUE(store_util()->InsertPrefetchItem(item)) + << "Failed inserting item with state " << static_cast<int>(state); + return item; +} + +PrefetchItem StaleEntryFinalizerTaskTest::InsertItemWithCreationTime( PrefetchItemState state, - int time_delta_in_hours) { + int creation_delta_in_hours) { PrefetchItem item(item_generator()->CreateItem(state)); - item.freshness_time = - fake_now_ + base::TimeDelta::FromHours(time_delta_in_hours); - item.creation_time = item.freshness_time; + item.creation_time = simple_test_clock_.Now() + + base::TimeDelta::FromHours(creation_delta_in_hours); + item.freshness_time = simple_test_clock_.Now(); EXPECT_TRUE(store_util()->InsertPrefetchItem(item)) << "Failed inserting item with state " << static_cast<int>(state); return item; @@ -101,31 +126,31 @@ TEST_F(StaleEntryFinalizerTaskTest, EmptyRun) { TEST_F(StaleEntryFinalizerTaskTest, HandlesFreshnessTimesCorrectly) { // Insert fresh and stale items for all expirable states from all buckets. PrefetchItem b1_item1_fresh = - CreateAndInsertItem(PrefetchItemState::NEW_REQUEST, -23); + InsertItemWithFreshnessTime(PrefetchItemState::NEW_REQUEST, -23); PrefetchItem b1_item2_stale = - CreateAndInsertItem(PrefetchItemState::NEW_REQUEST, -25); + InsertItemWithFreshnessTime(PrefetchItemState::NEW_REQUEST, -25); PrefetchItem b2_item1_fresh = - CreateAndInsertItem(PrefetchItemState::AWAITING_GCM, -23); + InsertItemWithFreshnessTime(PrefetchItemState::AWAITING_GCM, -23); PrefetchItem b2_item2_stale = - CreateAndInsertItem(PrefetchItemState::AWAITING_GCM, -25); + InsertItemWithFreshnessTime(PrefetchItemState::AWAITING_GCM, -25); PrefetchItem b2_item3_fresh = - CreateAndInsertItem(PrefetchItemState::RECEIVED_GCM, -23); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_GCM, -23); PrefetchItem b2_item4_stale = - CreateAndInsertItem(PrefetchItemState::RECEIVED_GCM, -25); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_GCM, -25); PrefetchItem b2_item5_fresh = - CreateAndInsertItem(PrefetchItemState::RECEIVED_BUNDLE, -23); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_BUNDLE, -23); PrefetchItem b2_item6_stale = - CreateAndInsertItem(PrefetchItemState::RECEIVED_BUNDLE, -25); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_BUNDLE, -25); PrefetchItem b3_item1_fresh = - CreateAndInsertItem(PrefetchItemState::DOWNLOADING, -47); + InsertItemWithFreshnessTime(PrefetchItemState::DOWNLOADING, -47); PrefetchItem b3_item2_stale = - CreateAndInsertItem(PrefetchItemState::DOWNLOADING, -49); + InsertItemWithFreshnessTime(PrefetchItemState::DOWNLOADING, -49); PrefetchItem b3_item3_fresh = - CreateAndInsertItem(PrefetchItemState::IMPORTING, -47); + InsertItemWithFreshnessTime(PrefetchItemState::IMPORTING, -47); PrefetchItem b3_item4_stale = - CreateAndInsertItem(PrefetchItemState::IMPORTING, -49); + InsertItemWithFreshnessTime(PrefetchItemState::IMPORTING, -49); // Check inserted initial items. std::set<PrefetchItem> initial_items = { @@ -175,7 +200,7 @@ TEST_F(StaleEntryFinalizerTaskTest, HandlesStalesInAllStatesCorrectly) { // than the point at which we report items as too old. const int many_hours = -6 * 24; for (PrefetchItemState state : kOrderedPrefetchItemStates) - CreateAndInsertItem(state, many_hours); + InsertItemWithFreshnessTime(state, many_hours); EXPECT_EQ(11, store_util()->CountPrefetchItems()); // Execute the expiration task. @@ -187,19 +212,19 @@ TEST_F(StaleEntryFinalizerTaskTest, HandlesStalesInAllStatesCorrectly) { EXPECT_EQ(11U, post_items.size()); EXPECT_EQ( 1U, - Filter(post_items, PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE).size()); + Select(post_items, PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE).size()); EXPECT_EQ(1U, - Filter(post_items, PrefetchItemState::SENT_GET_OPERATION).size()); - EXPECT_EQ(1U, Filter(post_items, PrefetchItemState::DOWNLOADED).size()); - EXPECT_EQ(7U, Filter(post_items, PrefetchItemState::FINISHED).size()); - EXPECT_EQ(1U, Filter(post_items, PrefetchItemState::ZOMBIE).size()); + Select(post_items, PrefetchItemState::SENT_GET_OPERATION).size()); + EXPECT_EQ(1U, Select(post_items, PrefetchItemState::DOWNLOADED).size()); + EXPECT_EQ(7U, Select(post_items, PrefetchItemState::FINISHED).size()); + EXPECT_EQ(1U, Select(post_items, PrefetchItemState::ZOMBIE).size()); } // Items in states AWAITING_GCM and ZOMBIE should cause the task to finish with // a NO_MORE_WORK result. TEST_F(StaleEntryFinalizerTaskTest, NoWorkInQueue) { - CreateAndInsertItem(PrefetchItemState::AWAITING_GCM, 0); - CreateAndInsertItem(PrefetchItemState::ZOMBIE, 0); + InsertItemWithFreshnessTime(PrefetchItemState::AWAITING_GCM, 0); + InsertItemWithFreshnessTime(PrefetchItemState::ZOMBIE, 0); RunTask(stale_finalizer_task_.get()); EXPECT_EQ(Result::NO_MORE_WORK, stale_finalizer_task_->final_status()); @@ -233,34 +258,34 @@ TEST_F(StaleEntryFinalizerTaskTest, WorkInQueue) { TEST_F(StaleEntryFinalizerTaskTest, HandlesClockSetBackwardsCorrectly) { // Insert fresh and stale items for all expirable states from all buckets. PrefetchItem b1_item1_recent = - CreateAndInsertItem(PrefetchItemState::NEW_REQUEST, 23); + InsertItemWithFreshnessTime(PrefetchItemState::NEW_REQUEST, 23); PrefetchItem b1_item2_future = - CreateAndInsertItem(PrefetchItemState::NEW_REQUEST, 25); + InsertItemWithFreshnessTime(PrefetchItemState::NEW_REQUEST, 25); PrefetchItem b2_item1_recent = - CreateAndInsertItem(PrefetchItemState::AWAITING_GCM, 23); + InsertItemWithFreshnessTime(PrefetchItemState::AWAITING_GCM, 23); PrefetchItem b2_item2_future = - CreateAndInsertItem(PrefetchItemState::AWAITING_GCM, 25); + InsertItemWithFreshnessTime(PrefetchItemState::AWAITING_GCM, 25); PrefetchItem b2_item3_recent = - CreateAndInsertItem(PrefetchItemState::RECEIVED_GCM, 23); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_GCM, 23); PrefetchItem b2_item4_future = - CreateAndInsertItem(PrefetchItemState::RECEIVED_GCM, 25); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_GCM, 25); PrefetchItem b2_item5_recent = - CreateAndInsertItem(PrefetchItemState::RECEIVED_BUNDLE, 23); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_BUNDLE, 23); PrefetchItem b2_item6_future = - CreateAndInsertItem(PrefetchItemState::RECEIVED_BUNDLE, 25); + InsertItemWithFreshnessTime(PrefetchItemState::RECEIVED_BUNDLE, 25); PrefetchItem b3_item1_recent = - CreateAndInsertItem(PrefetchItemState::DOWNLOADING, 23); + InsertItemWithFreshnessTime(PrefetchItemState::DOWNLOADING, 23); PrefetchItem b3_item2_future = - CreateAndInsertItem(PrefetchItemState::DOWNLOADING, 25); + InsertItemWithFreshnessTime(PrefetchItemState::DOWNLOADING, 25); PrefetchItem b3_item3_recent = - CreateAndInsertItem(PrefetchItemState::IMPORTING, 23); + InsertItemWithFreshnessTime(PrefetchItemState::IMPORTING, 23); PrefetchItem b3_item4_future = - CreateAndInsertItem(PrefetchItemState::IMPORTING, 25); + InsertItemWithFreshnessTime(PrefetchItemState::IMPORTING, 25); - PrefetchItem b4_item1_future = - CreateAndInsertItem(PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE, 25); + PrefetchItem b4_item1_future = InsertItemWithFreshnessTime( + PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE, 25); // Check inserted initial items. std::set<PrefetchItem> initial_items = { @@ -320,60 +345,97 @@ TEST_F(StaleEntryFinalizerTaskTest, // Insert "future" items for every state. const int many_hours = 7 * 24; for (PrefetchItemState state : kOrderedPrefetchItemStates) - CreateAndInsertItem(state, many_hours); + InsertItemWithFreshnessTime(state, many_hours); EXPECT_EQ(11, store_util()->CountPrefetchItems()); // Execute the expiration task. RunTask(stale_finalizer_task_.get()); EXPECT_EQ(Result::MORE_WORK_NEEDED, stale_finalizer_task_->final_status()); - // Checks item counts for states expected to still exist. + // Checks item counts for states expected to still exist. The zombie item is + // expected to be deleted. std::set<PrefetchItem> post_items = store_util()->GetAllItems(); - EXPECT_EQ(11U, post_items.size()); + EXPECT_EQ(10U, post_items.size()); EXPECT_EQ( 1U, - Filter(post_items, PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE).size()); + Select(post_items, PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE).size()); EXPECT_EQ(1U, - Filter(post_items, PrefetchItemState::SENT_GET_OPERATION).size()); - EXPECT_EQ(1U, Filter(post_items, PrefetchItemState::DOWNLOADED).size()); - EXPECT_EQ(7U, Filter(post_items, PrefetchItemState::FINISHED).size()); - EXPECT_EQ(1U, Filter(post_items, PrefetchItemState::ZOMBIE).size()); + Select(post_items, PrefetchItemState::SENT_GET_OPERATION).size()); + EXPECT_EQ(1U, Select(post_items, PrefetchItemState::DOWNLOADED).size()); + EXPECT_EQ(7U, Select(post_items, PrefetchItemState::FINISHED).size()); } -// Verifies that only stale, live items are transitioned to 'FINISHED'. TEST_F(StaleEntryFinalizerTaskTest, HandlesStuckItemsCorrectly) { base::HistogramTester histogram_tester; - // Insert fresh and stale items for all expirable states from all buckets. - PrefetchItem item1_recent = - CreateAndInsertItem(PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE, 1); - PrefetchItem item2_stuck = - CreateAndInsertItem(PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE, -170); - PrefetchItem item3_finished = - CreateAndInsertItem(PrefetchItemState::FINISHED, -170); - PrefetchItem item4_zombie = - CreateAndInsertItem(PrefetchItemState::ZOMBIE, -170); + // Insert stuck and non stuck items for all expirable states from all buckets. + // Note that stuck items are determined based on creation time instead of + // freshness. + for (PrefetchItemState state : kOrderedPrefetchItemStates) { + InsertItemWithCreationTime(state, -1); + InsertItemWithCreationTime(state, -170); // 170h is a bit more than a week. + } + EXPECT_EQ(22, store_util()->CountPrefetchItems()); + + // Execute the expiration task. + RunTask(stale_finalizer_task_.get()); + EXPECT_EQ(Result::MORE_WORK_NEEDED, stale_finalizer_task_->final_status()); + + std::set<PrefetchItem> final_items = store_util()->GetAllItems(); + + EXPECT_EQ(22U, final_items.size()); + // Zombie items should still be there. + EXPECT_EQ(2U, Select(final_items, PrefetchItemState::ZOMBIE).size()); + // Stuck entries should have been finalized with appropriate error codes. The + // initially inserter finished items should still be there, with a "success" + // error code. + std::set<PrefetchItem> final_finished_items = + Select(final_items, PrefetchItemState::FINISHED); + EXPECT_EQ(11U, final_finished_items.size()); + EXPECT_EQ(9U, + Select(final_finished_items, PrefetchItemErrorCode::STUCK).size()); + EXPECT_EQ( + 2U, Select(final_finished_items, PrefetchItemErrorCode::SUCCESS).size()); + + // All other non-stuck items should remain as they were. + std::vector<PrefetchItemState> stuck_states = GetAllStatesExcept( + {PrefetchItemState::FINISHED, PrefetchItemState::ZOMBIE}); + for (PrefetchItemState state : stuck_states) + EXPECT_EQ(1U, Select(final_items, state).size()); + + // Check metrics were reported for all stuck entries but not for finished nor + // zombie items. + histogram_tester.ExpectTotalCount("OfflinePages.Prefetching.StuckItemState", + 9); + for (PrefetchItemState state : stuck_states) { + histogram_tester.ExpectBucketCount( + "OfflinePages.Prefetching.StuckItemState", state, 1); + } +} + +TEST_F(StaleEntryFinalizerTaskTest, HandlesZombieFreshnessTimesCorrectly) { + PrefetchItem zombie_item1_fresh = + InsertItemWithFreshnessTime(PrefetchItemState::ZOMBIE, -160); + PrefetchItem zombie_item2_expired = + InsertItemWithFreshnessTime(PrefetchItemState::ZOMBIE, -170); + PrefetchItem zombie_item3_future_fresh = + InsertItemWithFreshnessTime(PrefetchItemState::ZOMBIE, 23); + PrefetchItem zombie_item3_future_expired = + InsertItemWithFreshnessTime(PrefetchItemState::ZOMBIE, 25); // Check inserted initial items. - std::set<PrefetchItem> initial_items = {item1_recent, item2_stuck, - item3_finished, item4_zombie}; + std::set<PrefetchItem> initial_items = { + zombie_item1_fresh, zombie_item2_expired, zombie_item3_future_fresh, + zombie_item3_future_expired}; EXPECT_EQ(initial_items, store_util()->GetAllItems()); // Execute the expiration task. RunTask(stale_finalizer_task_.get()); - EXPECT_EQ(Result::MORE_WORK_NEEDED, stale_finalizer_task_->final_status()); + EXPECT_EQ(Result::NO_MORE_WORK, stale_finalizer_task_->final_status()); - // Only the stuck item is changed. - PrefetchItem want_stuck_item = item2_stuck; - want_stuck_item.state = PrefetchItemState::FINISHED; - want_stuck_item.error_code = PrefetchItemErrorCode::STUCK; - std::set<PrefetchItem> final_items{item1_recent, want_stuck_item, - item3_finished, item4_zombie}; - EXPECT_EQ(final_items, store_util()->GetAllItems()); - // Check that the proper UMA was reported for the stale item, but not the - // fresh item, so there should be exactly one sample. - histogram_tester.ExpectUniqueSample( - "OfflinePages.Prefetching.StuckItemState", - static_cast<int>(PrefetchItemState::SENT_GENERATE_PAGE_BUNDLE), 1); + // Only the unexpired zombie should remain. + std::set<PrefetchItem> expected_final_items = {zombie_item1_fresh, + zombie_item3_future_fresh}; + EXPECT_EQ(expected_final_items, store_util()->GetAllItems()); } } // namespace offline_pages diff --git a/chromium/components/offline_pages/core/prefetch/test_util.cc b/chromium/components/offline_pages/core/prefetch/test_util.cc index 35b0055ebb7..7e70e578f7a 100644 --- a/chromium/components/offline_pages/core/prefetch/test_util.cc +++ b/chromium/components/offline_pages/core/prefetch/test_util.cc @@ -16,6 +16,9 @@ std::string PrefetchItem::ToString() const { << client_id << ", state=" << state << ", url=" << url.possibly_invalid_spec() << ", final_url=" << final_archived_url.possibly_invalid_spec() + << ", thumbnail_url=" << thumbnail_url.possibly_invalid_spec() + << ", favicon_url=" << favicon_url.possibly_invalid_spec() + << ", snippet=" << snippet << ", attribution=" << attribution << ", gb_attempts=" << generate_bundle_attempts << ", get_attempts=" << get_operation_attempts << ", dl_attempts=" << download_initiation_attempts @@ -43,6 +46,9 @@ bool PrefetchItem::operator==(const PrefetchItem& other) const { return offline_id == other.offline_id && guid == other.guid && client_id == other.client_id && state == other.state && url == other.url && final_archived_url == other.final_archived_url && + thumbnail_url == other.thumbnail_url && + favicon_url == other.favicon_url && snippet == other.snippet && + attribution == other.attribution && generate_bundle_attempts == other.generate_bundle_attempts && get_operation_attempts == other.get_operation_attempts && download_initiation_attempts == other.download_initiation_attempts && diff --git a/chromium/components/offline_pages/core/prefetch/thumbnail_fetch_by_url.cc b/chromium/components/offline_pages/core/prefetch/thumbnail_fetch_by_url.cc new file mode 100644 index 00000000000..3874c2759a2 --- /dev/null +++ b/chromium/components/offline_pages/core/prefetch/thumbnail_fetch_by_url.cc @@ -0,0 +1,60 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/prefetch/thumbnail_fetch_by_url.h" + +#include <utility> + +#include "base/bind.h" +#include "components/image_fetcher/core/image_fetcher.h" + +namespace offline_pages { + +namespace { +net::NetworkTrafficAnnotationTag TrafficAnnotation() { + return net::DefineNetworkTrafficAnnotation("prefetch_thumbnail", R"( + semantics { + sender: "Offline Pages Prefetch" + description: + "Chromium fetches suggested articles for offline viewing. This" + " network request is for a thumbnail that matches the article." + trigger: + "Two attempts, directly before and after the article is fetched." + data: + "The requested thumbnail URL." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: NO + setting: + "Users can enable or disable offline prefetch by toggling " + "'Download articles for you' in settings under Downloads or " + "by toggling chrome://flags#offline-prefetch." + chrome_policy { + NTPContentSuggestionsEnabled { + policy_options {mode: MANDATORY} + NTPContentSuggestionsEnabled: false + } + } + })"); +} + +} // namespace + +void FetchThumbnailByURL( + base::OnceCallback<void(const std::string& image_data)> callback, + image_fetcher::ImageFetcher* fetcher, + const GURL thumbnail_url) { + auto forward_callback = + [](base::OnceCallback<void(const std::string& image_data)> callback, + const std::string& image_data, + const image_fetcher::RequestMetadata& request_metadata) { + std::move(callback).Run(image_data); + }; + fetcher->FetchImageData(/*id=*/std::string(), thumbnail_url, + base::BindOnce(forward_callback, std::move(callback)), + TrafficAnnotation()); +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/prefetch/thumbnail_fetch_by_url.h b/chromium/components/offline_pages/core/prefetch/thumbnail_fetch_by_url.h new file mode 100644 index 00000000000..5f0527249cc --- /dev/null +++ b/chromium/components/offline_pages/core/prefetch/thumbnail_fetch_by_url.h @@ -0,0 +1,29 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_THUMBNAIL_FETCH_BY_URL_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_THUMBNAIL_FETCH_BY_URL_H_ + +#include <string> + +#include "base/callback.h" +#include "url/gurl.h" + +namespace image_fetcher { +class ImageFetcher; +} // namespace image_fetcher + +namespace offline_pages { + +// Attempts to fetch a thumbnail, and returns the result to callback. +// |image_data| will be empty if the thumbnail fetch fails. Otherwise, +// |image_data| contains the raw image data (typically JPEG). +void FetchThumbnailByURL( + base::OnceCallback<void(const std::string& image_data)> callback, + image_fetcher::ImageFetcher* fetcher, + const GURL thumbnail_url); + +} // namespace offline_pages + +#endif // COMPONENTS_OFFLINE_PAGES_CORE_PREFETCH_THUMBNAIL_FETCH_BY_URL_H_ diff --git a/chromium/components/offline_pages/core/test_scoped_offline_clock.cc b/chromium/components/offline_pages/core/test_scoped_offline_clock.cc new file mode 100644 index 00000000000..dc9ea6883b9 --- /dev/null +++ b/chromium/components/offline_pages/core/test_scoped_offline_clock.cc @@ -0,0 +1,18 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/offline_pages/core/test_scoped_offline_clock.h" + +#include "components/offline_pages/core/offline_clock.h" + +namespace offline_pages { + +TestScopedOfflineClock::TestScopedOfflineClock() { + SetOfflineClockForTesting(this); +} +TestScopedOfflineClock::~TestScopedOfflineClock() { + SetOfflineClockForTesting(nullptr); +} + +} // namespace offline_pages diff --git a/chromium/components/offline_pages/core/test_scoped_offline_clock.h b/chromium/components/offline_pages/core/test_scoped_offline_clock.h new file mode 100644 index 00000000000..be25a21a33b --- /dev/null +++ b/chromium/components/offline_pages/core/test_scoped_offline_clock.h @@ -0,0 +1,26 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_OFFLINE_PAGES_CORE_TEST_SCOPED_OFFLINE_CLOCK_H_ +#define COMPONENTS_OFFLINE_PAGES_CORE_TEST_SCOPED_OFFLINE_CLOCK_H_ + +#include "base/macros.h" +#include "base/test/simple_test_clock.h" + +namespace offline_pages { + +// Overrides |OfflineClock()| with |this| upon construction. Returns +// |OfflineClock()| to its original state upon destruction. +class TestScopedOfflineClock : public base::SimpleTestClock { + public: + TestScopedOfflineClock(); + ~TestScopedOfflineClock() override; + + private: + DISALLOW_COPY_AND_ASSIGN(TestScopedOfflineClock); +}; + +} // namespace offline_pages + +#endif // COMPONENTS_OFFLINE_PAGES_CORE_TEST_SCOPED_OFFLINE_CLOCK_H_ diff --git a/chromium/components/offline_pages/resources/PRESUBMIT.py b/chromium/components/offline_pages/resources/PRESUBMIT.py deleted file mode 100644 index af7a56f6eec..00000000000 --- a/chromium/components/offline_pages/resources/PRESUBMIT.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -def PostUploadHook(cl, change, output_api): - return output_api.EnsureCQIncludeTrybotsAreAdded( - cl, - [ - 'luci.chromium.try:closure_compilation', - ], - 'Automatically added optional Closure bots to run on CQ.') diff --git a/chromium/components/offline_pages/task/task.cc b/chromium/components/offline_pages/task/task.cc index 78f01ec5cfe..c662a9b436b 100644 --- a/chromium/components/offline_pages/task/task.cc +++ b/chromium/components/offline_pages/task/task.cc @@ -4,8 +4,7 @@ #include "components/offline_pages/task/task.h" -#include "base/bind.h" -#include "base/threading/thread_task_runner_handle.h" +#include <utility> namespace offline_pages { @@ -14,31 +13,21 @@ Task::Task() {} Task::~Task() {} void Task::SetTaskCompletionCallbackForTesting( - scoped_refptr<base::SingleThreadTaskRunner> task_completion_runner, TaskCompletionCallback task_completion_callback) { - SetTaskCompletionCallback(task_completion_runner, - std::move(task_completion_callback)); + SetTaskCompletionCallback(std::move(task_completion_callback)); } void Task::SetTaskCompletionCallback( - scoped_refptr<base::SingleThreadTaskRunner> task_completion_runner, TaskCompletionCallback task_completion_callback) { - // Attempts to enforce that SetTaskCompletionCallback is at most called once - // and enforces that reasonable values are set once that happens. - DCHECK(!task_completion_runner_); - DCHECK(task_completion_runner); + // Attempts to enforce that SetTaskCompletionCallback is at most called once. DCHECK(task_completion_callback_.is_null()); DCHECK(!task_completion_callback.is_null()); - task_completion_runner_ = task_completion_runner; task_completion_callback_ = std::move(task_completion_callback); } void Task::TaskComplete() { - if (task_completion_callback_.is_null() || !task_completion_runner_) - return; - - task_completion_runner_->PostTask( - FROM_HERE, base::BindOnce(std::move(task_completion_callback_), this)); + if (!task_completion_callback_.is_null()) + std::move(task_completion_callback_).Run(this); } } // namespace offline_pages diff --git a/chromium/components/offline_pages/task/task.h b/chromium/components/offline_pages/task/task.h index 696e8d082f1..a071ffa76f0 100644 --- a/chromium/components/offline_pages/task/task.h +++ b/chromium/components/offline_pages/task/task.h @@ -8,7 +8,6 @@ #include "base/callback.h" #include "base/macros.h" #include "base/memory/ref_counted.h" -#include "base/single_thread_task_runner.h" namespace offline_pages { @@ -44,35 +43,29 @@ class Task { // Sets the callback normally used by |TaskQueue| for testing. See // |SetTaskCompletionCallback| for details. void SetTaskCompletionCallbackForTesting( - scoped_refptr<base::SingleThreadTaskRunner> task_completion_runner, TaskCompletionCallback task_completion_callback); protected: - // Call |TaskComplete| as the last call, before the task is terminated. This - // ensures that |TaskQueue| can pick up another task. - // |task_completion_callback_| will be scheduled on the provided - // |task_completion_runner_|, which means task code is no longer going to be - // on stack, when the next call is made. + // Tasks must call |TaskComplete| as their last step. This will cause + // |TaskQueue| to schedule the task's destruction and start another task if + // one is available. void TaskComplete(); private: friend class TaskQueue; - // Allows task queue to Set the |task_completion_callback| and single thread - // task |task_completion_runner| that will be used to inform the |TaskQueue| - // when the task is done. + // Allows |TaskQueue| to set the |task_completion_callback| that will be used + // to inform it when the task is done. If the task is run outside of the + // |TaskQueue| and completion callback is not set, it will also work. // - // If the task is run outside of the |TaskQueue| and completion callback is - // not set, it will also work. + // Note: The callback implementation is responsible for scheduling work with + // the appropriate task runner and not to cause side effects to the |Task| + // calling into |TaskComplete|. void SetTaskCompletionCallback( - scoped_refptr<base::SingleThreadTaskRunner> task_completion_runner, TaskCompletionCallback task_completion_callback); // Completion callback for this task set by |SetTaskCompletionCallback|. TaskCompletionCallback task_completion_callback_; - // Task runner for calling completion callback. Set by - // |SetTaskCompletionCallback|. - scoped_refptr<base::SingleThreadTaskRunner> task_completion_runner_; DISALLOW_COPY_AND_ASSIGN(Task); }; diff --git a/chromium/components/offline_pages/task/task_queue.cc b/chromium/components/offline_pages/task/task_queue.cc index 2fe5986a4c3..2b73e6f2aa1 100644 --- a/chromium/components/offline_pages/task/task_queue.cc +++ b/chromium/components/offline_pages/task/task_queue.cc @@ -4,13 +4,20 @@ #include "components/offline_pages/task/task_queue.h" +#include <utility> + #include "base/bind.h" +#include "base/memory/scoped_refptr.h" +#include "base/memory/weak_ptr.h" +#include "base/single_thread_task_runner.h" #include "base/threading/thread_task_runner_handle.h" namespace offline_pages { TaskQueue::TaskQueue(Delegate* delegate) - : delegate_(delegate), weak_ptr_factory_(this) { + : task_runner_(base::ThreadTaskRunnerHandle::Get()), + delegate_(delegate), + weak_ptr_factory_(this) { DCHECK(delegate_); } @@ -18,10 +25,8 @@ TaskQueue::~TaskQueue() {} void TaskQueue::AddTask(std::unique_ptr<Task> task) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - task->SetTaskCompletionCallback( - base::ThreadTaskRunnerHandle::Get(), - base::BindOnce(&TaskQueue::TaskCompleted, - weak_ptr_factory_.GetWeakPtr())); + task->SetTaskCompletionCallback(base::BindOnce( + &TaskCompletedCallback, task_runner_, weak_ptr_factory_.GetWeakPtr())); tasks_.push(std::move(task)); StartTaskIfAvailable(); } @@ -44,17 +49,32 @@ void TaskQueue::StartTaskIfAvailable() { return; if (!HasPendingTasks()) { - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(&TaskQueue::InformTaskQueueIsIdle, - weak_ptr_factory_.GetWeakPtr())); + task_runner_->PostTask(FROM_HERE, + base::BindOnce(&TaskQueue::InformTaskQueueIsIdle, + weak_ptr_factory_.GetWeakPtr())); return; } current_task_ = std::move(tasks_.front()); tasks_.pop(); + task_runner_->PostTask(FROM_HERE, + base::BindOnce(&TaskQueue::RunCurrentTask, + weak_ptr_factory_.GetWeakPtr())); +} + +void TaskQueue::RunCurrentTask() { current_task_->Run(); } +// static +void TaskQueue::TaskCompletedCallback( + scoped_refptr<base::SingleThreadTaskRunner> task_runner, + base::WeakPtr<TaskQueue> task_queue, + Task* task) { + task_runner->PostTask( + FROM_HERE, base::BindOnce(&TaskQueue::TaskCompleted, task_queue, task)); +} + void TaskQueue::TaskCompleted(Task* task) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK_EQ(task, current_task_.get()); diff --git a/chromium/components/offline_pages/task/task_queue.h b/chromium/components/offline_pages/task/task_queue.h index 4e0e0f925f0..f0952ae964e 100644 --- a/chromium/components/offline_pages/task/task_queue.h +++ b/chromium/components/offline_pages/task/task_queue.h @@ -15,6 +15,10 @@ #include "base/sequence_checker.h" #include "components/offline_pages/task/task.h" +namespace base { +class SingleThreadTaskRunner; +} // namespace base + namespace offline_pages { // Class for coordinating |Task|s in relation to access to a specific resource. @@ -35,7 +39,7 @@ class TaskQueue { public: class Delegate { public: - virtual ~Delegate(){}; + virtual ~Delegate() {} // Invoked once when TaskQueue reached 0 tasks. virtual void OnTaskQueueIsIdle() = 0; @@ -57,11 +61,23 @@ class TaskQueue { // queue. void StartTaskIfAvailable(); - // Callback for informing the queue that a task was completed. + void RunCurrentTask(); + + // Callback for informing the queue that a task was completed. Can be called + // from any thread. + static void TaskCompletedCallback( + scoped_refptr<base::SingleThreadTaskRunner> task_runner, + base::WeakPtr<TaskQueue> task_queue, + Task* task); + void TaskCompleted(Task* task); void InformTaskQueueIsIdle(); + // This TaskQueue's task runner, set on construction using the instance + // assigned to the current thread. + scoped_refptr<base::SingleThreadTaskRunner> task_runner_; + // Owns and outlives this TaskQueue. Delegate* delegate_; diff --git a/chromium/components/offline_pages/task/task_queue_unittest.cc b/chromium/components/offline_pages/task/task_queue_unittest.cc index d8d2f19dea3..bbb95a11481 100644 --- a/chromium/components/offline_pages/task/task_queue_unittest.cc +++ b/chromium/components/offline_pages/task/task_queue_unittest.cc @@ -5,6 +5,7 @@ #include "components/offline_pages/task/task_queue.h" #include <memory> +#include <utility> #include "base/bind.h" #include "base/test/test_simple_task_runner.h" @@ -59,10 +60,11 @@ TEST_F(OfflineTaskQueueTest, AddAndRunSingleTask) { TestTask* task_ptr = task.get(); TaskQueue queue(this); EXPECT_FALSE(on_idle_called()); - EXPECT_EQ(TaskState::NOT_STARTED, task_ptr->state()); queue.AddTask(std::move(task)); EXPECT_TRUE(queue.HasPendingTasks()); EXPECT_TRUE(queue.HasRunningTask()); + EXPECT_EQ(TaskState::NOT_STARTED, task_ptr->state()); + PumpLoop(); // Start running the task. EXPECT_EQ(TaskState::STEP_1, task_ptr->state()); EXPECT_TRUE(resource.HasNextStep()); resource.CompleteStep(); @@ -88,12 +90,13 @@ TEST_F(OfflineTaskQueueTest, AddAndRunMultipleTasks) { TestTask* task_2_ptr = task_2.get(); TaskQueue queue(this); - EXPECT_EQ(TaskState::NOT_STARTED, task_1_ptr->state()); - EXPECT_EQ(TaskState::NOT_STARTED, task_2_ptr->state()); queue.AddTask(std::move(task_1)); queue.AddTask(std::move(task_2)); EXPECT_TRUE(queue.HasPendingTasks()); EXPECT_TRUE(queue.HasRunningTask()); + EXPECT_EQ(TaskState::NOT_STARTED, task_1_ptr->state()); + EXPECT_EQ(TaskState::NOT_STARTED, task_2_ptr->state()); + PumpLoop(); // Start running the task 1. EXPECT_EQ(TaskState::STEP_1, task_1_ptr->state()); EXPECT_EQ(TaskState::NOT_STARTED, task_2_ptr->state()); resource.CompleteStep(); @@ -115,10 +118,11 @@ TEST_F(OfflineTaskQueueTest, LeaveEarly) { new TestTask(&resource, true /* leave early */)); TestTask* task_ptr = task.get(); TaskQueue queue(this); - EXPECT_EQ(TaskState::NOT_STARTED, task_ptr->state()); queue.AddTask(std::move(task)); EXPECT_TRUE(queue.HasPendingTasks()); EXPECT_TRUE(queue.HasRunningTask()); + EXPECT_EQ(TaskState::NOT_STARTED, task_ptr->state()); + PumpLoop(); // Start running the task. EXPECT_EQ(TaskState::STEP_1, task_ptr->state()); EXPECT_TRUE(resource.HasNextStep()); resource.CompleteStep(); diff --git a/chromium/components/offline_pages/task/task_unittest.cc b/chromium/components/offline_pages/task/task_unittest.cc index d4de7df9804..6bdb771a53a 100644 --- a/chromium/components/offline_pages/task/task_unittest.cc +++ b/chromium/components/offline_pages/task/task_unittest.cc @@ -41,14 +41,15 @@ void OfflineTaskTest::PumpLoop() { } void OfflineTaskTest::TaskCompleted(Task* task) { - completed_task_ = task; + auto set_task_callback = [](Task** t_ptr, Task* t) { *t_ptr = t; }; + task_runner_->PostTask( + FROM_HERE, base::BindOnce(set_task_callback, &completed_task_, task)); } TEST_F(OfflineTaskTest, RunTaskStepByStep) { ConsumedResource resource; TestTask task(&resource); task.SetTaskCompletionCallbackForTesting( - base::ThreadTaskRunnerHandle::Get(), base::BindOnce(&OfflineTaskTest::TaskCompleted, base::Unretained(this))); EXPECT_EQ(TaskState::NOT_STARTED, task.state()); diff --git a/chromium/components/offline_pages/task/test_task_runner.cc b/chromium/components/offline_pages/task/test_task_runner.cc index 030ceea138f..caeccae54c2 100644 --- a/chromium/components/offline_pages/task/test_task_runner.cc +++ b/chromium/components/offline_pages/task/test_task_runner.cc @@ -23,14 +23,17 @@ void TestTaskRunner::RunTask(std::unique_ptr<Task> task) { void TestTaskRunner::RunTask(Task* task) { DCHECK(task); Task* completed_task = nullptr; - task->SetTaskCompletionCallbackForTesting( - task_runner_.get(), - base::BindOnce([](Task** completed_task_ptr, - Task* task) { *completed_task_ptr = task; }, - &completed_task)); + task->SetTaskCompletionCallbackForTesting(base::BindOnce( + &TestTaskRunner::TaskComplete, base::Unretained(this), &completed_task)); task->Run(); task_runner_->RunUntilIdle(); EXPECT_EQ(task, completed_task) << "Task did not complete"; } +void TestTaskRunner::TaskComplete(Task** completed_task_ptr, Task* task) { + auto set_task_callback = [](Task** t_ptr, Task* t) { *t_ptr = t; }; + task_runner_->PostTask( + FROM_HERE, base::BindOnce(set_task_callback, completed_task_ptr, task)); +} + } // namespace offline_pages diff --git a/chromium/components/offline_pages/task/test_task_runner.h b/chromium/components/offline_pages/task/test_task_runner.h index be6e2975773..5e0ccde0351 100644 --- a/chromium/components/offline_pages/task/test_task_runner.h +++ b/chromium/components/offline_pages/task/test_task_runner.h @@ -28,6 +28,8 @@ class TestTaskRunner { void RunTask(Task* task); private: + void TaskComplete(Task** completed_task_ptr, Task* task); + // Certainly confusing, but internal task runner, is simply a test version of // a single thread task runner. It is used for running closures. // The difference between that and the encapsulating task runner, is that a |