diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-06 12:48:11 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:33:43 +0000 |
commit | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (patch) | |
tree | fa14ba0ca8d2683ba2efdabd246dc9b18a1229c6 /chromium/components/feed | |
parent | 79b4f909db1049fca459c07cca55af56a9b54fe3 (diff) | |
download | qtwebengine-chromium-7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3.tar.gz |
BASELINE: Update Chromium to 84.0.4147.141
Change-Id: Ib85eb4cfa1cbe2b2b81e5022c8cad5c493969535
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/components/feed')
98 files changed, 6502 insertions, 1292 deletions
diff --git a/chromium/components/feed/DEPS b/chromium/components/feed/DEPS index 7c06e34f28a..2bac831fea2 100644 --- a/chromium/components/feed/DEPS +++ b/chromium/components/feed/DEPS @@ -1,5 +1,6 @@ include_rules = [ "+components/image_fetcher", + "+components/history/core/browser", "+components/keyed_service/core", "+components/leveldb_proto", "+components/offline_pages", @@ -17,6 +18,7 @@ include_rules = [ "+services/network/public/mojom", "+services/network/test", "+third_party/zlib/google", + "+third_party/protobuf/src/google/protobuf/io", "+ui/base/mojom", "+ui/gfx/geometry", "+ui/gfx/image", diff --git a/chromium/components/feed/core/common/pref_names.cc b/chromium/components/feed/core/common/pref_names.cc index 1abd2fc4d1d..a4736acee86 100644 --- a/chromium/components/feed/core/common/pref_names.cc +++ b/chromium/components/feed/core/common/pref_names.cc @@ -4,6 +4,8 @@ #include "components/feed/core/common/pref_names.h" +#include <string> + #include "components/feed/core/common/user_classifier.h" #include "components/prefs/pref_registry_simple.h" @@ -35,6 +37,8 @@ const char kThrottlerRequestCountListPrefName[] = "feedv2.request_throttler.request_counts"; const char kThrottlerLastRequestTime[] = "feedv2.request_throttler.last_request_time"; +const char kDebugStreamData[] = "feedv2.debug_stream_data"; +const char kRequestSchedule[] = "feedv2.request_schedule"; } // namespace prefs @@ -49,6 +53,8 @@ void RegisterProfilePrefs(PrefRegistrySimple* registry) { registry->RegisterListPref(feed::prefs::kThrottlerRequestCountListPrefName); registry->RegisterTimePref(feed::prefs::kThrottlerLastRequestTime, base::Time()); + registry->RegisterStringPref(feed::prefs::kDebugStreamData, std::string()); + registry->RegisterDictionaryPref(feed::prefs::kRequestSchedule); UserClassifier::RegisterProfilePrefs(registry); } diff --git a/chromium/components/feed/core/common/pref_names.h b/chromium/components/feed/core/common/pref_names.h index f3bc8f04ccb..b23c1552bde 100644 --- a/chromium/components/feed/core/common/pref_names.h +++ b/chromium/components/feed/core/common/pref_names.h @@ -48,6 +48,10 @@ extern const char kHostOverrideBlessNonce[]; extern const char kThrottlerRequestCountListPrefName[]; // The pref name for the request throttler's last request time. extern const char kThrottlerLastRequestTime[]; +// The pref name for storing |DebugStreamData|. +extern const char kDebugStreamData[]; +// The pref name for storing the request schedule. +extern const char kRequestSchedule[]; } // namespace prefs diff --git a/chromium/components/feed/core/feed_content_mutation.cc b/chromium/components/feed/core/feed_content_mutation.cc index 9d3d88cd395..57502baf371 100644 --- a/chromium/components/feed/core/feed_content_mutation.cc +++ b/chromium/components/feed/core/feed_content_mutation.cc @@ -6,7 +6,6 @@ #include <utility> -#include "base/logging.h" #include "components/feed/core/feed_content_operation.h" namespace feed { diff --git a/chromium/components/feed/core/feed_content_operation.cc b/chromium/components/feed/core/feed_content_operation.cc index 650600cda05..073de921dbf 100644 --- a/chromium/components/feed/core/feed_content_operation.cc +++ b/chromium/components/feed/core/feed_content_operation.cc @@ -6,7 +6,7 @@ #include <utility> -#include "base/logging.h" +#include "base/check_op.h" namespace feed { diff --git a/chromium/components/feed/core/feed_journal_mutation.cc b/chromium/components/feed/core/feed_journal_mutation.cc index d20c2b20e33..1ff132109cf 100644 --- a/chromium/components/feed/core/feed_journal_mutation.cc +++ b/chromium/components/feed/core/feed_journal_mutation.cc @@ -6,7 +6,7 @@ #include <utility> -#include "base/logging.h" +#include "base/check.h" namespace feed { diff --git a/chromium/components/feed/core/feed_journal_operation.cc b/chromium/components/feed/core/feed_journal_operation.cc index 4e561427818..803114cb56c 100644 --- a/chromium/components/feed/core/feed_journal_operation.cc +++ b/chromium/components/feed/core/feed_journal_operation.cc @@ -6,7 +6,7 @@ #include <utility> -#include "base/logging.h" +#include "base/check_op.h" namespace feed { diff --git a/chromium/components/feed/core/feed_logging_metrics.cc b/chromium/components/feed/core/feed_logging_metrics.cc index 6ba1b9940bf..55fec7919cb 100644 --- a/chromium/components/feed/core/feed_logging_metrics.cc +++ b/chromium/components/feed/core/feed_logging_metrics.cc @@ -603,14 +603,6 @@ void FeedLoggingMetrics::OnSuggestionArticleVisited(base::TimeDelta visit_time, RecordSuggestionPageVisited(return_to_ntp); } -void FeedLoggingMetrics::OnSuggestionOfflinePageVisited( - base::TimeDelta visit_time, - bool return_to_ntp) { - base::UmaHistogramLongTimes( - "NewTabPage.ContentSuggestions.VisitDuration.Downloads", visit_time); - RecordSuggestionPageVisited(return_to_ntp); -} - void FeedLoggingMetrics::OnMoreButtonShown(int position) { // The "more" card can appear in addition to the actual suggestions, so add // one extra bucket to this histogram. diff --git a/chromium/components/feed/core/feed_logging_metrics.h b/chromium/components/feed/core/feed_logging_metrics.h index c0398a710c5..441580c8a44 100644 --- a/chromium/components/feed/core/feed_logging_metrics.h +++ b/chromium/components/feed/core/feed_logging_metrics.h @@ -74,9 +74,6 @@ class FeedLoggingMetrics { void OnSuggestionArticleVisited(base::TimeDelta visit_time, bool return_to_ntp); - void OnSuggestionOfflinePageVisited(base::TimeDelta visit_time, - bool return_to_ntp); - // Should only be called once per NTP for each "more" button. void OnMoreButtonShown(int position); diff --git a/chromium/components/feed/core/proto/BUILD.gn b/chromium/components/feed/core/proto/BUILD.gn index d9da12241c5..bede4090414 100644 --- a/chromium/components/feed/core/proto/BUILD.gn +++ b/chromium/components/feed/core/proto/BUILD.gn @@ -45,10 +45,12 @@ proto_library("proto_v2") { "v2/wire/payload_metadata.proto", "v2/wire/render_data.proto", "v2/wire/request.proto", + "v2/wire/request_schedule.proto", "v2/wire/response.proto", "v2/wire/response_status_code.proto", "v2/wire/stream_structure.proto", "v2/wire/templates.proto", + "v2/wire/there_and_back_again_data.proto", "v2/wire/token.proto", "v2/wire/version.proto", ] diff --git a/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto b/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto index e66a0b818d7..f5e0b2606d3 100644 --- a/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto +++ b/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto @@ -229,6 +229,10 @@ message StreamUploadableAction { optional feedwire1.ActionPayload payload = 6; + // The duration for the action in milliseconds. In case of view actions this + // is the duration for which the content is considered "viewed". + optional int64 duration_ms = 7; + reserved 1, 5; // deprecated fields } diff --git a/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto b/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto index 419ea251146..ad714f035f1 100644 --- a/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto +++ b/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto @@ -30,10 +30,11 @@ message FeedAction { } // Metadata needed by the host to handle the action. -// Next Id: 19 +// Next Id: 11 message FeedActionMetadata { // The type of action, used by the host to perform any custom logic needed for // a specific type of action. + // Next Id: 21 enum Type { UNKNOWN = 0; OPEN_URL = 1; @@ -60,6 +61,8 @@ message FeedActionMetadata { SEE_SUGGESTED_SITES = 16; SEND_FEEDBACK = 17; MANAGE_INTERESTS = 18; + REPORT_VIEW = 19; + BLOCK_CONTENT = 20; reserved 9, 10; // Deprecated } optional Type type = 1; @@ -80,8 +83,15 @@ message FeedActionMetadata { // The data needed by the Stream to render a tooltip. TooltipData tooltip_data = 8; + + // The data needed by the Stream to report a view action. + ViewReportData view_report_data = 10; + + // The data needed by the stream to block content. + BlockContentData block_content_data = 11; } // The type of element this action is bounded on. + // Next Id: 5 enum ElementType { UNKNOWN_ELEMENT_TYPE = 0; CARD_LARGE_IMAGE = 1; @@ -105,6 +115,7 @@ message OpenUrlData { // opening the url. Once this finishes, the client will attach to the url its // latest frequency token as the value of this query param. optional string consistency_token_query_param_name = 2; + // The content ID that was interacted with to cause a URL open. optional feedwire1.ContentId content_id = 3; // Roundtripped server data on a per-action level. @@ -160,6 +171,14 @@ message NotInterestedInData { reserved 4; } +message BlockContentData { + // The data needed by Stream to (locally) dismiss the content. + repeated feedwire1.DataOperation data_operations = 1; + // Roundtripped server data on a per-action level. + // Identifies to the server the content that must be blocked. + optional feedwire1.ActionPayload payload = 2; +} + // Data used by the client to show a confirmation message with an action to // reverse it. message UndoAction { @@ -198,6 +217,21 @@ message Insets { optional int32 bottom = 2; } +// Data used by the client to report a view action on content. +message ViewReportData { + enum Visibility { + UNKNOWN = 0; + SHOW = 1; + HIDE = 2; + } + // Whether we're showing or hiding the content. + optional Visibility visibility = 1; + // The content ID of the view that needs to be reported. + optional feedwire1.ContentId content_id = 2; + // Roundtripped server data on a per-action level. + optional feedwire1.ActionPayload payload = 3; +} + // FeedActionMetadata with a label to show in a context menu. message LabelledFeedActionData { optional string label = 1; diff --git a/chromium/components/feed/core/proto/v2/store.proto b/chromium/components/feed/core/proto/v2/store.proto index 99df5c61363..4adb1deee4f 100644 --- a/chromium/components/feed/core/proto/v2/store.proto +++ b/chromium/components/feed/core/proto/v2/store.proto @@ -17,12 +17,12 @@ option optimize_for = LITE_RUNTIME; // // This is the 'value' in the key/value store. // Keys are defined as: -// 'S/<stream-id>' -> stream_data -// 'T/<stream-id>/<sequence-number>' -> stream_structures -// 'c/<content-id>' -> content -// 'a/<id>' -> action -// 's/<content-id>' -> shared_state -// 'N' -> next_stream_state +// S/<stream-id> -> stream_data +// T/<stream-id>/<sequence-number> -> stream_structures +// c/<content-id> -> content +// a/<id> -> action +// s/<content-id> -> shared_state +// m -> metadata message Record { oneof data { StreamData stream_data = 1; @@ -30,8 +30,7 @@ message Record { Content content = 3; StoredAction local_action = 4; StreamSharedState shared_state = 5; - // The result of a background refresh, to be processed later. - StreamAndContentState next_stream_state = 6; + Metadata metadata = 6; } } @@ -41,14 +40,19 @@ message StreamData { feedwire.ContentId content_id = 1; // Token used to request a 'more' request to the server. bytes next_page_token = 2; - // Token used to read or write to the same storage. - bytes consistency_token = 3; // The unix timestamp in milliseconds that the most recent content was added. int64 last_added_time_millis = 4; - // Next sequential ID to be used for StoredAction.id. - int32 next_action_id = 5; // The content ID of the shared state for this stream. feedwire.ContentId shared_state_id = 6; + reserved 3, 5; +} + +// Data that doesn't belong to a stream. +message Metadata { + // Token used to read or write to the same storage. + bytes consistency_token = 1; + // ID for the next pending action. + int32 next_action_id = 2; } // A set of StreamStructures that should be applied to a stream. @@ -163,11 +167,3 @@ message StoredAction { // The action to upload. feedwire.FeedAction action = 3; } - -// The internal version of the server response. Includes feature tree and -// content. -message StreamAndContentState { - StreamData stream_data = 1; - repeated Content content = 2; - repeated StreamSharedState shared_state = 3; -} diff --git a/chromium/components/feed/core/proto/v2/ui.proto b/chromium/components/feed/core/proto/v2/ui.proto index c1258bc010b..b3e24a7fb81 100644 --- a/chromium/components/feed/core/proto/v2/ui.proto +++ b/chromium/components/feed/core/proto/v2/ui.proto @@ -41,6 +41,7 @@ message Slice { oneof SliceData { XSurfaceSlice xsurface_slice = 1; ZeroStateSlice zero_state_slice = 3; + LoadingSpinnerSlice loading_spinner_slice = 4; } string slice_id = 2; } @@ -57,6 +58,13 @@ message ZeroStateSlice { Type type = 1; } +// An indicator that the feed is loading. +message LoadingSpinnerSlice { + // True if the spinner is at the top of the feed. Otherwise, it is at the + // bottom. + bool is_at_top = 1; +} + message XSurfaceSlice { bytes xsurface_frame = 1; } @@ -66,45 +74,3 @@ message SharedState { string id = 1; bytes xsurface_shared_state = 2; } - -// An event on the UI. -message UiEvent { - enum Type { - UNKNOWN = 0; - CARD_TAPPED = 1; - CARD_VIEWED = 2; - CARD_SWIPED = 3; - MORE_BUTTON_VIEWED = 4; - MORE_BUTTON_CLICKED = 5; - SPINNER_STARTED = 6; - SPINNER_FINISHED = 7; - SPINNER_DESTROYED_WITHOUT_COMPLETING = 8; - PIET_FRAME_RENDERING_EVENT = 9; - SCROLL_STREAM = 10; - } - enum SpinnerType { - UNKNOWN_SPINNER_TYPE = 0; - // Spinner shown on initial load of the Feed. - INITIAL_LOAD = 1; - // Spinner shown when Feed is refreshed. - ZERO_STATE_REFRESH = 2; - // Spinner shown when more button is clicked. - MORE_BUTTON = 3; - // Spinner shown when a synthetic token is consumed. - SYNTHETIC_TOKEN = 4; - // Spinner shown when a spinner is shown for loading the infinite feed. - INFINITE_FEED = 5; - } - // For CARD_* type events. - string chunk_id = 1; - // For MORE_BUTTON_* type events. - int32 more_button_position = 2; - // For SPINNER_* type events. - SpinnerType spinner_type = 3; - // For SPINNER_FINISHED and SPINNER_DESTROYED_WITHOUT_COMPLETING. - int32 spinner_time_shown = 4; - // For PIET_FRAME_RENDERING_EVENT. - repeated int32 piet_error_codes = 5; - // For SCROLL_STREAM. - int32 scroll_distance = 6; -} diff --git a/chromium/components/feed/core/proto/v2/wire/capability.proto b/chromium/components/feed/core/proto/v2/wire/capability.proto index 73b95309fda..f9da90c6a64 100644 --- a/chromium/components/feed/core/proto/v2/wire/capability.proto +++ b/chromium/components/feed/core/proto/v2/wire/capability.proto @@ -35,5 +35,6 @@ enum Capability { INLINE_VIDEO_AUTOPLAY = 18; // Enable the card menu. CARD_MENU = 19; + REQUEST_SCHEDULE = 20; reserved 2 to 4, 6 to 8, 12; } diff --git a/chromium/components/feed/core/proto/v2/wire/client_info.proto b/chromium/components/feed/core/proto/v2/wire/client_info.proto index a4e7753cd34..f21907add5c 100644 --- a/chromium/components/feed/core/proto/v2/wire/client_info.proto +++ b/chromium/components/feed/core/proto/v2/wire/client_info.proto @@ -21,7 +21,10 @@ message ClientInfo { IOS = 2; } - enum AppType { CHROME = 3; } + enum AppType { + TEST_APP = 2; // For use with AGA endpoint for testing. + CHROME = 3; + } // The type of OS that the client is running. optional PlatformType platform_type = 1; diff --git a/chromium/components/feed/core/proto/v2/wire/data_operation.proto b/chromium/components/feed/core/proto/v2/wire/data_operation.proto index 0651eb21797..cdafdb60e6e 100644 --- a/chromium/components/feed/core/proto/v2/wire/data_operation.proto +++ b/chromium/components/feed/core/proto/v2/wire/data_operation.proto @@ -14,6 +14,7 @@ import "components/feed/core/proto/v2/wire/payload_metadata.proto"; import "components/feed/core/proto/v2/wire/render_data.proto"; import "components/feed/core/proto/v2/wire/templates.proto"; import "components/feed/core/proto/v2/wire/token.proto"; +import "components/feed/core/proto/v2/wire/request_schedule.proto"; // An extensible operation to change the state of data on the client. message DataOperation { @@ -51,5 +52,7 @@ message DataOperation { // A collection of templates. Templates templates = 4 [deprecated = true]; + + RequestSchedule request_schedule = 9; } } diff --git a/chromium/components/feed/core/proto/v2/wire/request_schedule.proto b/chromium/components/feed/core/proto/v2/wire/request_schedule.proto new file mode 100644 index 00000000000..9d2ab7aee25 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/request_schedule.proto @@ -0,0 +1,19 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/duration.proto"; + +message RequestSchedule { + message TimeBasedSchedule { + repeated feedwire.Duration refresh_time_from_response_time = 1; + } + + oneof schedule { TimeBasedSchedule time_based_schedule = 1; } +} diff --git a/chromium/components/feed/core/proto/v2/wire/stream_structure.proto b/chromium/components/feed/core/proto/v2/wire/stream_structure.proto index 256aa6c89af..4b267c2266c 100644 --- a/chromium/components/feed/core/proto/v2/wire/stream_structure.proto +++ b/chromium/components/feed/core/proto/v2/wire/stream_structure.proto @@ -24,12 +24,6 @@ message Cluster { // inside or outside a card. Actual data on what to display will be sent on an // extension. message Content { - enum Type { - UNKNOWN_CONTENT = 0; - XSURFACE = 1; - } - optional Type type = 1; - optional bool is_ad = 3; optional XSurfaceContent xsurface_content = 1000; diff --git a/chromium/components/feed/core/proto/v2/wire/there_and_back_again_data.proto b/chromium/components/feed/core/proto/v2/wire/there_and_back_again_data.proto new file mode 100644 index 00000000000..98eaaec997f --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/there_and_back_again_data.proto @@ -0,0 +1,15 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/action_payload.proto"; + +message ThereAndBackAgainData { + optional feedwire.ActionPayload action_payload = 1; +} diff --git a/chromium/components/feed/core/proto/wire/capability.proto b/chromium/components/feed/core/proto/wire/capability.proto index 6bfb7860787..a53a8eaed39 100644 --- a/chromium/components/feed/core/proto/wire/capability.proto +++ b/chromium/components/feed/core/proto/wire/capability.proto @@ -12,7 +12,7 @@ option java_package = "org.chromium.components.feed.core.proto.wire"; option java_outer_classname = "CapabilityProto"; // Feature capability of either the client or the server. -// Next ID: 14. +// Next ID: 16. enum Capability { UNKNOWN_CAPABILITY = 0; BASE_UI = 1; @@ -26,6 +26,8 @@ enum Capability { ELEMENTS = 10; SEND_FEEDBACK = 12; CLICK_ACTION = 13; + VIEW_ACTION = 14; + REPORT_FEED_USER_ACTIONS_NOTICE_CARD = 15; reserved 3, 11; } diff --git a/chromium/components/feed/core/proto/wire/feed_action.proto b/chromium/components/feed/core/proto/wire/feed_action.proto index dce68844591..837ea9e6dbe 100644 --- a/chromium/components/feed/core/proto/wire/feed_action.proto +++ b/chromium/components/feed/core/proto/wire/feed_action.proto @@ -32,5 +32,14 @@ message FeedAction { message ClientData { // When the action was recorded on the client optional int64 timestamp_seconds = 1; + + // A monotonically-increasing sequence number that increments per + // user + device. Used in experiments to measure action loss between client + // and server. + optional int64 sequence_number = 2; + + // The duration for the action in milliseconds. In case of view actions this + // is the duration for which the content is considered "viewed". + optional int64 duration_ms = 3; } } diff --git a/chromium/components/feed/core/proto/wire/feed_action_request.proto b/chromium/components/feed/core/proto/wire/feed_action_request.proto index a82e7ceb962..eae293536d5 100644 --- a/chromium/components/feed/core/proto/wire/feed_action_request.proto +++ b/chromium/components/feed/core/proto/wire/feed_action_request.proto @@ -24,4 +24,14 @@ message FeedActionRequest { repeated FeedAction feed_action = 1; // Token used to write to the same storage. optional ConsistencyToken consistency_token = 2; + + // Diagnostic information about the current state of the client. + optional DiagnosticInfo diagnostic_info = 3; + + // Diagnostics from the client. + message DiagnosticInfo { + // Count of un-uploaded actions remaining on the client at the time + // of a feed content generation request. + optional int32 actions_remaining = 1; + } } diff --git a/chromium/components/feed/core/v2/BUILD.gn b/chromium/components/feed/core/v2/BUILD.gn index 99d8d69a7c1..929a554bd8d 100644 --- a/chromium/components/feed/core/v2/BUILD.gn +++ b/chromium/components/feed/core/v2/BUILD.gn @@ -10,6 +10,8 @@ if (is_android) { source_set("feed_core_v2") { sources = [ + "config.cc", + "config.h", "enums.cc", "enums.h", "feed_network.cc", @@ -20,43 +22,57 @@ source_set("feed_core_v2") { "feed_store.h", "feed_stream.cc", "feed_stream.h", + "metrics_reporter.cc", + "metrics_reporter.h", "prefs.cc", "prefs.h", "proto_util.cc", "proto_util.h", + "protocol_translator.cc", + "protocol_translator.h", "public/feed_service.cc", "public/feed_service.h", + "public/feed_stream_api.cc", "public/feed_stream_api.h", + "public/types.h", "refresh_task_scheduler.h", "request_throttler.cc", "request_throttler.h", "scheduling.cc", "scheduling.h", - "stream_event_metrics.cc", - "stream_event_metrics.h", "stream_model.cc", "stream_model.h", "stream_model/ephemeral_change.cc", "stream_model/ephemeral_change.h", "stream_model/feature_tree.cc", "stream_model/feature_tree.h", - "stream_model_update_request.cc", - "stream_model_update_request.h", + "surface_updater.cc", + "surface_updater.h", + "tasks/clear_all_task.cc", + "tasks/clear_all_task.h", + "tasks/load_more_task.cc", + "tasks/load_more_task.h", "tasks/load_stream_from_store_task.cc", "tasks/load_stream_from_store_task.h", "tasks/load_stream_task.cc", "tasks/load_stream_task.h", + "tasks/upload_actions_task.cc", + "tasks/upload_actions_task.h", "tasks/wait_for_store_initialize_task.cc", "tasks/wait_for_store_initialize_task.h", + "types.cc", + "types.h", ] deps = [ "//components/feed/core:feed_core", "//components/feed/core/common:feed_core_common", + "//components/history/core/browser", "//components/offline_pages/task:task", "//components/prefs", "//components/signin/public/identity_manager", "//components/variations", "//components/variations/net", + "//components/version_info:channel", "//components/web_resource:web_resource", "//net", "//services/network/public/cpp", @@ -77,15 +93,20 @@ source_set("core_unit_tests") { "feed_network_impl_unittest.cc", "feed_store_unittest.cc", "feed_stream_unittest.cc", + "metrics_reporter_unittest.cc", + "proto_util_unittest.cc", + "protocol_translator_unittest.cc", + "public/feed_service_unittest.cc", "request_throttler_unittest.cc", + "scheduling_unittest.cc", "stream_model_unittest.cc", - "stream_model_update_request_unittest.cc", - "test/callback_receiver.h", "test/callback_receiver.h", + "test/callback_receiver_unittest.cc", "test/proto_printer.cc", "test/proto_printer.h", "test/stream_builder.cc", "test/stream_builder.h", + "types_unittest.cc", ] deps = [ @@ -95,10 +116,12 @@ source_set("core_unit_tests") { "//base/test:test_support", "//components/feed/core:feed_core", "//components/feed/core/common:feed_core_common", + "//components/history/core/browser", "//components/leveldb_proto:test_support", "//components/prefs:test_support", "//components/signin/public/identity_manager", "//components/signin/public/identity_manager:test_support", + "//components/version_info:channel", "//net:test_support", "//services/network:test_support", "//services/network/public/cpp", diff --git a/chromium/components/feed/core/v2/config.cc b/chromium/components/feed/core/v2/config.cc new file mode 100644 index 00000000000..7f5279286f5 --- /dev/null +++ b/chromium/components/feed/core/v2/config.cc @@ -0,0 +1,72 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/config.h" + +#include "base/metrics/field_trial_params.h" +#include "components/feed/feed_feature_list.h" + +namespace feed { +namespace { +// A note about the design. +// Soon, we'll add the ability to override configuration values from sources +// other than Finch. Finch serves well for experimentation, but after we're done +// experimenting, we still want to control some of these values. The tentative +// plan is to send configuration down from the server, and store it in prefs. +// The source of a config value would be the following, in order of preference: +// finch, server, default-value. +Config g_config; + +// Override any parameters that may be provided by Finch. +void OverrideWithFinch(Config* config) { + config->max_feed_query_requests_per_day = + base::GetFieldTrialParamByFeatureAsInt( + kInterestFeedV2, "max_feed_query_requests_per_day", + config->max_feed_query_requests_per_day); + + config->max_action_upload_requests_per_day = + base::GetFieldTrialParamByFeatureAsInt( + kInterestFeedV2, "max_action_upload_requests_per_day", + config->max_action_upload_requests_per_day); + + config->stale_content_threshold = + base::TimeDelta::FromSecondsD(base::GetFieldTrialParamByFeatureAsDouble( + kInterestFeedV2, "stale_content_threshold_seconds", + config->stale_content_threshold.InSecondsF())); + + config->default_background_refresh_interval = + base::TimeDelta::FromSecondsD(base::GetFieldTrialParamByFeatureAsDouble( + kInterestFeedV2, "default_background_refresh_interval_seconds", + config->stale_content_threshold.InSecondsF())); + + config->max_action_upload_attempts = base::GetFieldTrialParamByFeatureAsInt( + kInterestFeedV2, "max_action_upload_attempts", + config->max_action_upload_attempts); + + config->max_action_age = + base::TimeDelta::FromSecondsD(base::GetFieldTrialParamByFeatureAsDouble( + kInterestFeedV2, "max_action_age_seconds", + config->max_action_age.InSecondsF())); + + config->max_action_upload_bytes = base::GetFieldTrialParamByFeatureAsInt( + kInterestFeedV2, "max_action_upload_bytes", + config->max_action_upload_bytes); +} + +} // namespace + +const Config& GetFeedConfig() { + static bool initialized = false; + if (!initialized) { + initialized = true; + OverrideWithFinch(&g_config); + } + return g_config; +} + +void SetFeedConfigForTesting(const Config& config) { + g_config = config; +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/config.h b/chromium/components/feed/core/v2/config.h new file mode 100644 index 00000000000..82c60a268ff --- /dev/null +++ b/chromium/components/feed/core/v2/config.h @@ -0,0 +1,40 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_CONFIG_H_ +#define COMPONENTS_FEED_CORE_V2_CONFIG_H_ + +#include "base/time/time.h" + +namespace feed { + +// The Feed configuration. Default values appear below. Always use +// |GetFeedConfig()| to get the current configuration. +struct Config { + // Maximum number of FeedQuery or action upload requests per day. + int max_feed_query_requests_per_day = 20; + int max_action_upload_requests_per_day = 20; + // Content older than this threshold will not be shown to the user. + base::TimeDelta stale_content_threshold = base::TimeDelta::FromHours(48); + // The time between background refresh attempts. Ignored if a server-defined + // fetch schedule has been assigned. + base::TimeDelta default_background_refresh_interval = + base::TimeDelta::FromHours(24); + // Maximum number of times to attempt to upload a pending action before + // deleting it. + int max_action_upload_attempts = 3; + // Maximum age for a pending action. Actions older than this are deleted. + base::TimeDelta max_action_age = base::TimeDelta::FromHours(24); + // Maximum payload size for one action upload batch. + size_t max_action_upload_bytes = 20000; +}; + +// Gets the current configuration. +const Config& GetFeedConfig(); + +void SetFeedConfigForTesting(const Config& config); + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_CONFIG_H_ diff --git a/chromium/components/feed/core/v2/enums.cc b/chromium/components/feed/core/v2/enums.cc index ce8c0ce37ec..bf4a3472b87 100644 --- a/chromium/components/feed/core/v2/enums.cc +++ b/chromium/components/feed/core/v2/enums.cc @@ -43,6 +43,54 @@ std::ostream& operator<<(std::ostream& out, LoadStreamStatus value) { return out << "kLoadNotAllowedEulaNotAccepted"; case LoadStreamStatus::kLoadNotAllowedArticlesListHidden: return out << "kLoadNotAllowedArticlesListHidden"; + case LoadStreamStatus::kCannotParseNetworkResponseBody: + return out << "kCannotParseNetworkResponseBody"; + case LoadStreamStatus::kLoadMoreModelIsNotLoaded: + return out << "kLoadMoreModelIsNotLoaded"; + } +#else + return out << (static_cast<int>(value)); +#endif // ifndef NDEBUG +} + +std::ostream& operator<<(std::ostream& out, UploadActionsStatus value) { +#ifndef NDEBUG + switch (value) { + case UploadActionsStatus::kNoStatus: + return out << "kNoStatus"; + case UploadActionsStatus::kNoPendingActions: + return out << "kNoPendingActions"; + case UploadActionsStatus::kFailedToStorePendingAction: + return out << "kFailedToStorePendingAction"; + case UploadActionsStatus::kStoredPendingAction: + return out << "kStoredPendingAction"; + case UploadActionsStatus::kUpdatedConsistencyToken: + return out << "kUpdatedConsistencyToken"; + case UploadActionsStatus::kFinishedWithoutUpdatingConsistencyToken: + return out << "kFinishedWithoutUpdatingConsistencyToken"; + } +#else + return out << (static_cast<int>(value)); +#endif // ifndef NDEBUG +} + +std::ostream& operator<<(std::ostream& out, UploadActionsBatchStatus value) { +#ifndef NDEBUG + switch (value) { + case UploadActionsBatchStatus::kNoStatus: + return out << "kNoStatus"; + case UploadActionsBatchStatus::kFailedToUpdateStore: + return out << "kFailedToUpdateStore"; + case UploadActionsBatchStatus::kFailedToUpload: + return out << "kFailedToUpload"; + case UploadActionsBatchStatus::kFailedToRemoveUploadedActions: + return out << "kFailedToRemoveUploadedActions"; + case UploadActionsBatchStatus::kExhaustedUploadQuota: + return out << "kExhaustedUploadQuota"; + case UploadActionsBatchStatus::kAllActionsWereStale: + return out << "kAllActionsWereStale"; + case UploadActionsBatchStatus::kSuccessfullyUploadedBatch: + return out << "kSuccessfullyUploadedBatch"; } #else return out << (static_cast<int>(value)); diff --git a/chromium/components/feed/core/v2/enums.h b/chromium/components/feed/core/v2/enums.h index dc6d84e2e96..df2e2861341 100644 --- a/chromium/components/feed/core/v2/enums.h +++ b/chromium/components/feed/core/v2/enums.h @@ -11,11 +11,12 @@ namespace feed { -enum NetworkRequestType : int { +enum class NetworkRequestType : int { kFeedQuery = 0, kUploadActions = 1, }; +// This must be kept in sync with FeedLoadStreamStatus in enums.xml. enum class LoadStreamStatus { // Loading was not attempted. kNoStatus = 0, @@ -36,10 +37,40 @@ enum class LoadStreamStatus { kCannotLoadFromNetworkThrottled = 12, kLoadNotAllowedEulaNotAccepted = 13, kLoadNotAllowedArticlesListHidden = 14, + // TODO(harringtond): Emit this status value. + kCannotParseNetworkResponseBody = 15, + kLoadMoreModelIsNotLoaded = 16, + kMaxValue = kLoadMoreModelIsNotLoaded, }; std::ostream& operator<<(std::ostream& out, LoadStreamStatus value); +// Keep this in sync with FeedUploadActionsStatus in enums.xml. +enum class UploadActionsStatus { + kNoStatus = 0, + kNoPendingActions = 1, + kFailedToStorePendingAction = 2, + kStoredPendingAction = 3, + kUpdatedConsistencyToken = 4, + kFinishedWithoutUpdatingConsistencyToken = 5, + kMaxValue = kFinishedWithoutUpdatingConsistencyToken, +}; + +// Keep this in sync with FeedUploadActionsBatchStatus in enums.xml. +enum class UploadActionsBatchStatus { + kNoStatus = 0, + kFailedToUpdateStore = 1, + kFailedToUpload = 2, + kFailedToRemoveUploadedActions = 3, + kExhaustedUploadQuota = 4, + kAllActionsWereStale = 5, + kSuccessfullyUploadedBatch = 6, + kMaxValue = kSuccessfullyUploadedBatch, +}; + +std::ostream& operator<<(std::ostream& out, UploadActionsStatus value); +std::ostream& operator<<(std::ostream& out, UploadActionsBatchStatus value); + } // namespace feed #endif // COMPONENTS_FEED_CORE_V2_ENUMS_H_ diff --git a/chromium/components/feed/core/v2/feed_network.h b/chromium/components/feed/core/v2/feed_network.h index 8a25a238117..cb09886f662 100644 --- a/chromium/components/feed/core/v2/feed_network.h +++ b/chromium/components/feed/core/v2/feed_network.h @@ -6,11 +6,12 @@ #define COMPONENTS_FEED_CORE_V2_FEED_NETWORK_H_ #include <memory> + #include "base/callback.h" +#include "components/feed/core/v2/public/types.h" namespace feedwire { class ActionRequest; -class FeedActionResponse; class Request; class Response; } // namespace feedwire @@ -25,8 +26,7 @@ class FeedNetwork { ~QueryRequestResult(); QueryRequestResult(QueryRequestResult&&); QueryRequestResult& operator=(QueryRequestResult&&); - // HTTP status code if one was received, 0 otherwise. - int32_t status_code = 0; + NetworkResponseInfo response_info; // Response body if one was received. std::unique_ptr<feedwire::Response> response_body; }; @@ -37,10 +37,9 @@ class FeedNetwork { ~ActionRequestResult(); ActionRequestResult(ActionRequestResult&&); ActionRequestResult& operator=(ActionRequestResult&&); - // HTTP status code if one was received, 0 otherwise. - int32_t status_code = 0; + NetworkResponseInfo response_info; // Response body if one was received. - std::unique_ptr<feedwire::FeedActionResponse> response_body; + std::unique_ptr<feedwire::Response> response_body; }; virtual ~FeedNetwork(); diff --git a/chromium/components/feed/core/v2/feed_network_impl.cc b/chromium/components/feed/core/v2/feed_network_impl.cc index 5231a84277f..df1e4874b27 100644 --- a/chromium/components/feed/core/v2/feed_network_impl.cc +++ b/chromium/components/feed/core/v2/feed_network_impl.cc @@ -3,6 +3,11 @@ // found in the LICENSE file. #include "components/feed/core/v2/feed_network_impl.h" + +#include <memory> +#include <utility> + +#include "base/base64url.h" #include "base/bind.h" #include "base/containers/flat_set.h" #include "base/metrics/histogram_functions.h" @@ -15,6 +20,7 @@ #include "components/feed/core/proto/v2/wire/feed_query.pb.h" #include "components/feed/core/proto/v2/wire/request.pb.h" #include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/feed/core/v2/metrics_reporter.h" #include "components/prefs/pref_service.h" #include "components/signin/public/identity_manager/access_token_info.h" #include "components/signin/public/identity_manager/identity_manager.h" @@ -30,6 +36,7 @@ #include "services/network/public/cpp/resource_request_body.h" #include "services/network/public/cpp/shared_url_loader_factory.h" #include "services/network/public/cpp/simple_url_loader.h" +#include "third_party/protobuf/src/google/protobuf/io/coded_stream.h" #include "third_party/zlib/google/compression_utils.h" namespace feed { @@ -39,37 +46,53 @@ constexpr char kAuthenticationScope[] = constexpr char kApplicationOctetStream[] = "application/octet-stream"; constexpr base::TimeDelta kNetworkTimeout = base::TimeDelta::FromSeconds(30); +// Add URLs for Bling when it is supported. constexpr char kFeedQueryUrl[] = - "https://www.google.com/httpservice/retry/InteractiveDiscoverAgaService/" - "FeedQuery"; + "https://www.google.com/httpservice/retry/TrellisClankService/FeedQuery"; constexpr char kNextPageQueryUrl[] = - "https://www.google.com/httpservice/retry/InteractiveDiscoverAgaService/" + "https://www.google.com/httpservice/retry/TrellisClankService/" "NextPageQuery"; constexpr char kBackgroundQueryUrl[] = - "https://www.google.com/httpservice/noretry/BackgroundDiscoverAgaService/" + "https://www.google.com/httpservice/noretry/TrellisClankService/" "FeedQuery"; +GURL GetUrlWithoutQuery(const GURL& url) { + GURL::Replacements replacements; + replacements.ClearQuery(); + return url.ReplaceComponents(replacements); +} + using RawResponse = FeedNetworkImpl::RawResponse; } // namespace struct FeedNetworkImpl::RawResponse { - // A union of net::Error (if the request failed) and the http - // status code(if the request succeeded in reaching the server). - int32_t status_code; // HTTP response body. std::string response_bytes; + NetworkResponseInfo response_info; }; namespace { -template <typename RESULT> +template <typename RESULT, NetworkRequestType REQUEST_TYPE> void ParseAndForwardResponse(base::OnceCallback<void(RESULT)> result_callback, RawResponse raw_response) { + MetricsReporter::NetworkRequestComplete( + REQUEST_TYPE, raw_response.response_info.status_code); RESULT result; - result.status_code = raw_response.status_code; - if (result.status_code == 200) { + result.response_info = raw_response.response_info; + if (result.response_info.status_code == 200) { auto response_message = std::make_unique<typename decltype( result.response_body)::element_type>(); - if (response_message->ParseFromString(raw_response.response_bytes)) { + + ::google::protobuf::io::CodedInputStream input_stream( + reinterpret_cast<const uint8_t*>(raw_response.response_bytes.data()), + raw_response.response_bytes.size()); + + // The first few bytes of the body are a varint containing the size of the + // message. We need to skip over them. + int message_size; + input_stream.ReadVarintSizeAsInt(&message_size); + + if (response_message->ParseFromCodedStream(&input_stream)) { result.response_body = std::move(response_message); } } @@ -277,7 +300,13 @@ class FeedNetworkImpl::NetworkFetch { } void OnSimpleLoaderComplete(std::unique_ptr<std::string> response) { - int32_t status_code = simple_loader_->NetError(); + NetworkResponseInfo response_info; + response_info.status_code = simple_loader_->NetError(); + response_info.fetch_duration = + tick_clock_->NowTicks() - entire_send_start_ticks_; + response_info.fetch_time = base::Time::Now(); + response_info.base_request_url = GetUrlWithoutQuery(url_); + // If overriding the feed host, try to grab the Bless nonce. This is // strictly informational, and only displayed in snippets-internals. if (host_overridden_ && simple_loader_->ResponseInfo()) { @@ -289,8 +318,7 @@ class FeedNetworkImpl::NetworkFetch { if (pos != std::string::npos) { std::string nonce = value.substr(pos + 7, 16); if (nonce.size() == 16) { - pref_service_->SetString(feed::prefs::kHostOverrideBlessNonce, - nonce); + response_info.bless_nonce = nonce; break; } } @@ -299,10 +327,11 @@ class FeedNetworkImpl::NetworkFetch { std::string response_body; if (response) { - status_code = simple_loader_->ResponseInfo()->headers->response_code(); + response_info.status_code = + simple_loader_->ResponseInfo()->headers->response_code(); response_body = std::move(*response); - if (status_code == net::HTTP_UNAUTHORIZED) { + if (response_info.status_code == net::HTTP_UNAUTHORIZED) { signin::ScopeSet scopes{kAuthenticationScope}; CoreAccountId account_id = identity_manager_->GetPrimaryAccountId(); if (!account_id.empty()) { @@ -312,10 +341,8 @@ class FeedNetworkImpl::NetworkFetch { } } - base::TimeDelta entire_send_duration = - tick_clock_->NowTicks() - entire_send_start_ticks_; UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.Duration", - entire_send_duration); + response_info.fetch_duration); base::TimeDelta loader_only_duration = tick_clock_->NowTicks() - loader_only_start_ticks_; @@ -323,17 +350,17 @@ class FeedNetworkImpl::NetworkFetch { // RemoteSuggestionsFetcherImpl. UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", loader_only_duration); - base::UmaHistogramSparse( - "ContentSuggestions.Feed.Network.RequestStatusCode", status_code); - // The below is true even if there is a protocol error, so this will // record response size as long as the request completed. - if (status_code >= 200) { + if (response_info.status_code >= 200) { UMA_HISTOGRAM_COUNTS_1M("ContentSuggestions.Feed.Network.ResponseSizeKB", static_cast<int>(response_body.size() / 1024)); } - std::move(done_callback_).Run({status_code, std::move(response_body)}); + RawResponse raw_response; + raw_response.response_info = std::move(response_info); + raw_response.response_bytes = std::move(response_body); + std::move(done_callback_).Run(std::move(raw_response)); } private: @@ -380,6 +407,9 @@ void FeedNetworkImpl::SendQueryRequest( base::OnceCallback<void(QueryRequestResult)> callback) { std::string binary_proto; request.SerializeToString(&binary_proto); + std::string base64proto; + base::Base64UrlEncode( + binary_proto, base::Base64UrlEncodePolicy::INCLUDE_PADDING, &base64proto); // TODO(harringtond): Decide how we want to override these URLs for testing. // Should probably add a command-line flag. @@ -400,10 +430,11 @@ void FeedNetworkImpl::SendQueryRequest( return; } - AddMothershipPayloadQueryParams(/*is_post=*/false, binary_proto, + AddMothershipPayloadQueryParams(/*is_post=*/false, base64proto, delegate_->GetLanguageTag(), &url); Send(url, "GET", /*request_body=*/std::string(), - base::BindOnce(&ParseAndForwardResponse<QueryRequestResult>, + base::BindOnce(&ParseAndForwardResponse<QueryRequestResult, + NetworkRequestType::kFeedQuery>, std::move(callback))); } @@ -418,10 +449,11 @@ void FeedNetworkImpl::SendActionRequest( "ClankActionUpload"); AddMothershipPayloadQueryParams(/*is_post=*/true, /*payload=*/std::string(), delegate_->GetLanguageTag(), &url); - Send(url, "POST", std::move(binary_proto), - base::BindOnce(&ParseAndForwardResponse<ActionRequestResult>, - std::move(callback))); + base::BindOnce( + &ParseAndForwardResponse<ActionRequestResult, + NetworkRequestType::kUploadActions>, + std::move(callback))); } void FeedNetworkImpl::CancelRequests() { diff --git a/chromium/components/feed/core/v2/feed_network_impl.h b/chromium/components/feed/core/v2/feed_network_impl.h index b96f25d0b6c..ba6e4dcb96e 100644 --- a/chromium/components/feed/core/v2/feed_network_impl.h +++ b/chromium/components/feed/core/v2/feed_network_impl.h @@ -16,13 +16,13 @@ class PrefService; namespace base { class TickClock; -} +} // namespace base namespace signin { class IdentityManager; -} +} // namespace signin namespace network { class SharedURLLoaderFactory; -} +} // namespace network namespace feed { diff --git a/chromium/components/feed/core/v2/feed_network_impl_unittest.cc b/chromium/components/feed/core/v2/feed_network_impl_unittest.cc index 7170cd5b8f1..c931576ae71 100644 --- a/chromium/components/feed/core/v2/feed_network_impl_unittest.cc +++ b/chromium/components/feed/core/v2/feed_network_impl_unittest.cc @@ -31,6 +31,8 @@ #include "services/network/test/test_url_loader_factory.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" +#include "third_party/protobuf/src/google/protobuf/io/coded_stream.h" +#include "third_party/protobuf/src/google/protobuf/io/zero_copy_stream_impl_lite.h" #include "third_party/zlib/google/compression_utils.h" #include "url/gurl.h" @@ -108,6 +110,8 @@ class FeedNetworkTest : public testing::Test { TestingPrefServiceSimple& profile_prefs() { return profile_prefs_; } + base::HistogramTester& histogram() { return histogram_; } + void Respond(const GURL& url, const std::string& response_string, net::HttpStatusCode code = net::HTTP_OK, @@ -127,6 +131,16 @@ class FeedNetworkTest : public testing::Test { test_factory_.AddResponse(url, std::move(head), response_string, status); } + std::string PrependResponseLength(const std::string& response) { + std::string result; + ::google::protobuf::io::StringOutputStream string_output_stream(&result); + ::google::protobuf::io::CodedOutputStream stream(&string_output_stream); + + stream.WriteVarint32(static_cast<uint32_t>(response.size())); + stream.WriteString(response); + return result; + } + network::ResourceRequest RespondToQueryRequest( const std::string& response_string, net::HttpStatusCode code) { @@ -135,7 +149,8 @@ class FeedNetworkTest : public testing::Test { test_factory()->GetPendingRequest(0); CHECK(pending_request); network::ResourceRequest resource_request = pending_request->request; - Respond(pending_request->request.url, response_string, code); + Respond(pending_request->request.url, + PrependResponseLength(response_string), code); task_environment_.FastForwardUntilNoTasksRemain(); return resource_request; } @@ -167,6 +182,7 @@ class FeedNetworkTest : public testing::Test { scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory_; base::SimpleTestTickClock test_tick_clock_; TestingPrefServiceSimple profile_prefs_; + base::HistogramTester histogram_; }; TEST_F(FeedNetworkTest, SendQueryRequestEmpty) { @@ -175,7 +191,7 @@ TEST_F(FeedNetworkTest, SendQueryRequestEmpty) { ASSERT_TRUE(receiver.GetResult()); const QueryRequestResult& result = *receiver.GetResult(); - EXPECT_EQ(0, result.status_code); + EXPECT_EQ(0, result.response_info.status_code); EXPECT_FALSE(result.response_body); } @@ -186,8 +202,8 @@ TEST_F(FeedNetworkTest, SendQueryRequestSendsValidRequest) { RespondToQueryRequest("", net::HTTP_OK); EXPECT_EQ( - "https://www.google.com/httpservice/retry/InteractiveDiscoverAgaService/" - "FeedQuery?reqpld=%08%01%C2%3E%04%12%02%08%01&fmt=bin&hl=en", + "https://www.google.com/httpservice/retry/TrellisClankService/" + "FeedQuery?reqpld=CAHCPgQSAggB&fmt=bin&hl=en", resource_request.url); EXPECT_EQ("GET", resource_request.method); EXPECT_FALSE(resource_request.headers.HasHeader("content-encoding")); @@ -195,6 +211,8 @@ TEST_F(FeedNetworkTest, SendQueryRequestSendsValidRequest) { EXPECT_TRUE( resource_request.headers.GetHeader("Authorization", &authorization)); EXPECT_EQ(authorization, "Bearer access_token"); + histogram().ExpectBucketCount( + "ContentSuggestions.Feed.Network.ResponseStatus.FeedQuery", 200, 1); } TEST_F(FeedNetworkTest, SendQueryRequestInvalidResponse) { @@ -204,7 +222,7 @@ TEST_F(FeedNetworkTest, SendQueryRequestInvalidResponse) { ASSERT_TRUE(receiver.GetResult()); const QueryRequestResult& result = *receiver.GetResult(); - EXPECT_EQ(net::HTTP_OK, result.status_code); + EXPECT_EQ(net::HTTP_OK, result.response_info.status_code); EXPECT_FALSE(result.response_body); } @@ -215,7 +233,11 @@ TEST_F(FeedNetworkTest, SendQueryRequestReceivesResponse) { ASSERT_TRUE(receiver.GetResult()); const QueryRequestResult& result = *receiver.GetResult(); - EXPECT_EQ(net::HTTP_OK, result.status_code); + EXPECT_EQ(net::HTTP_OK, result.response_info.status_code); + EXPECT_EQ( + "https://www.google.com/httpservice/retry/TrellisClankService/FeedQuery", + result.response_info.base_request_url); + EXPECT_NE(base::Time(), result.response_info.fetch_time); EXPECT_EQ(GetTestFeedResponse().response_version(), result.response_body->response_version()); } @@ -227,8 +249,11 @@ TEST_F(FeedNetworkTest, SendQueryRequestIgnoresBodyForNon200Response) { ASSERT_TRUE(receiver.GetResult()); const QueryRequestResult& result = *receiver.GetResult(); - EXPECT_EQ(net::HTTP_FORBIDDEN, result.status_code); + EXPECT_EQ(net::HTTP_FORBIDDEN, result.response_info.status_code); EXPECT_FALSE(result.response_body); + histogram().ExpectBucketCount( + "ContentSuggestions.Feed.Network.ResponseStatus.FeedQuery", + net::HTTP_FORBIDDEN, 1); } TEST_F(FeedNetworkTest, CancelRequest) { @@ -248,7 +273,7 @@ TEST_F(FeedNetworkTest, RequestTimeout) { ASSERT_TRUE(receiver.GetResult()); const QueryRequestResult& result = *receiver.GetResult(); - EXPECT_EQ(net::ERR_TIMED_OUT, result.status_code); + EXPECT_EQ(net::ERR_TIMED_OUT, result.response_info.status_code); histogram_tester.ExpectTimeBucketCount( "ContentSuggestions.Feed.Network.Duration", TimeDelta::FromSeconds(30), 1); @@ -280,14 +305,15 @@ TEST_F(FeedNetworkTest, ParallelRequests) { EXPECT_TRUE(receiver2.GetResult()); } -TEST_F(FeedNetworkTest, ShouldReportRequestStatusCode) { +TEST_F(FeedNetworkTest, ShouldReportResponseStatusCode) { CallbackReceiver<QueryRequestResult> receiver; base::HistogramTester histogram_tester; feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_FORBIDDEN); + EXPECT_THAT( histogram_tester.GetAllSamples( - "ContentSuggestions.Feed.Network.RequestStatusCode"), + "ContentSuggestions.Feed.Network.ResponseStatus.FeedQuery"), ElementsAre(base::Bucket(/*min=*/net::HTTP_FORBIDDEN, /*count=*/1))); } @@ -346,7 +372,7 @@ TEST_F(FeedNetworkTest, TestDurationHistogram) { } // Verify that the kHostOverrideHost pref overrides the feed host -// and updates the Bless nonce if one sent in the response. +// and returns the Bless nonce if one sent in the response. TEST_F(FeedNetworkTest, TestHostOverrideWithAuthHeader) { CallbackReceiver<QueryRequestResult> receiver; profile_prefs().SetString(feed::prefs::kHostOverrideHost, @@ -359,9 +385,9 @@ TEST_F(FeedNetworkTest, TestHostOverrideWithAuthHeader) { "nonce=\"1234123412341234\"\n\n")); RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_FORBIDDEN); - EXPECT_TRUE(receiver.GetResult()); + ASSERT_TRUE(receiver.GetResult()); EXPECT_EQ("1234123412341234", - profile_prefs().GetString(feed::prefs::kHostOverrideBlessNonce)); + receiver.GetResult()->response_info.bless_nonce); } TEST_F(FeedNetworkTest, SendActionRequest) { @@ -371,8 +397,10 @@ TEST_F(FeedNetworkTest, SendActionRequest) { ASSERT_TRUE(receiver.GetResult()); const ActionRequestResult& result = *receiver.GetResult(); - EXPECT_EQ(net::HTTP_OK, result.status_code); + EXPECT_EQ(net::HTTP_OK, result.response_info.status_code); EXPECT_TRUE(result.response_body); + histogram().ExpectBucketCount( + "ContentSuggestions.Feed.Network.ResponseStatus.UploadActions", 200, 1); } TEST_F(FeedNetworkTest, SendActionRequestSendsValidRequest) { diff --git a/chromium/components/feed/core/v2/feed_store.cc b/chromium/components/feed/core/v2/feed_store.cc index 7321e84fa8f..bb0add49a8e 100644 --- a/chromium/components/feed/core/v2/feed_store.cc +++ b/chromium/components/feed/core/v2/feed_store.cc @@ -10,29 +10,31 @@ #include "base/bind_helpers.h" #include "base/containers/flat_set.h" #include "base/files/file_path.h" -#include "base/logging.h" +#include "base/memory/ptr_util.h" +#include "base/notreached.h" #include "base/strings/strcat.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_piece.h" +#include "base/strings/string_util.h" #include "base/task/post_task.h" #include "base/task/thread_pool.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/leveldb_proto/public/proto_database_provider.h" namespace feed { namespace { // Keys are defined as: -// 'S/<stream-id>' -> stream_data -// 'T/<stream-id>/<sequence-number>' -> stream_structures -// 'c/<content-id>' -> content -// 'a/<id>' -> action -// 's/<content-id>' -> shared_state -// 'N' -> next_stream_state +// S/<stream-id> -> stream_data +// T/<stream-id>/<sequence-number> -> stream_structures +// c/<content-id> -> content +// a/<id> -> action +// s/<content-id> -> shared_state +// m -> metadata constexpr char kMainStreamId[] = "0"; const char kStreamDataKey[] = "S/0"; const char kLocalActionPrefix[] = "a/"; -const char kNextStreamStateKey[] = "N"; +const char kMetadataKey[] = "m"; leveldb::ReadOptions CreateReadOptions() { leveldb::ReadOptions opts; @@ -55,6 +57,26 @@ std::string SharedStateKey(const feedwire::ContentId& content_id) { return KeyForContentId("s/", content_id); } +std::string LocalActionKey(int64_t id) { + return kLocalActionPrefix + base::NumberToString(id); +} + +std::string LocalActionKey(const LocalActionId& id) { + return LocalActionKey(id.GetUnsafeValue()); +} + +// Returns true if the record key is for stream data (stream_data, +// stream_structures, content, shared_state). +bool IsStreamRecordKey(base::StringPiece key) { + return key.size() > 1 && key[1] == '/' && + (key[0] == 'S' || key[0] == 'T' || key[0] == 'c' || key[0] == 's'); +} + +bool IsLocalActionKey(const std::string& key) { + return base::StartsWith(key, kLocalActionPrefix, + base::CompareCase::INSENSITIVE_ASCII); +} + std::string KeyForRecord(const feedstore::Record& record) { switch (record.data_case()) { case feedstore::Record::kStreamData: @@ -66,16 +88,16 @@ std::string KeyForRecord(const feedstore::Record& record) { case feedstore::Record::kContent: return ContentKey(record.content().content_id()); case feedstore::Record::kLocalAction: - return kLocalActionPrefix + - base::NumberToString(record.local_action().id()); + return LocalActionKey(record.local_action().id()); case feedstore::Record::kSharedState: return SharedStateKey(record.shared_state().content_id()); - case feedstore::Record::kNextStreamState: - return kNextStreamStateKey; - case feedstore::Record::DATA_NOT_SET: // fall through - NOTREACHED() << "Invalid record case " << record.data_case(); - return ""; + case feedstore::Record::kMetadata: + return kMetadataKey; + case feedstore::Record::DATA_NOT_SET: + break; } + NOTREACHED() << "Invalid record case " << record.data_case(); + return ""; } bool FilterByKey(const base::flat_set<std::string>& key_set, @@ -108,6 +130,18 @@ feedstore::Record MakeRecord(feedstore::StreamData stream_data) { return record; } +feedstore::Record MakeRecord(feedstore::StoredAction action) { + feedstore::Record record; + *record.mutable_local_action() = std::move(action); + return record; +} + +feedstore::Record MakeRecord(feedstore::Metadata metadata) { + feedstore::Record record; + *record.mutable_metadata() = std::move(metadata); + return record; +} + template <typename T> std::pair<std::string, feedstore::Record> MakeKeyAndRecord(T record_data) { std::pair<std::string, feedstore::Record> result; @@ -116,6 +150,38 @@ std::pair<std::string, feedstore::Record> MakeKeyAndRecord(T record_data) { return result; } +std::unique_ptr<std::vector<std::pair<std::string, feedstore::Record>>> +MakeUpdatesForStreamModelUpdateRequest( + int32_t structure_set_sequence_number, + std::unique_ptr<StreamModelUpdateRequest> update_request) { + auto updates = std::make_unique< + std::vector<std::pair<std::string, feedstore::Record>>>(); + updates->push_back(MakeKeyAndRecord(std::move(update_request->stream_data))); + for (feedstore::Content& content : update_request->content) { + updates->push_back(MakeKeyAndRecord(std::move(content))); + } + for (feedstore::StreamSharedState& shared_state : + update_request->shared_states) { + updates->push_back(MakeKeyAndRecord(std::move(shared_state))); + } + feedstore::StreamStructureSet stream_structure_set; + stream_structure_set.set_stream_id(kMainStreamId); + stream_structure_set.set_sequence_number(structure_set_sequence_number); + for (feedstore::StreamStructure& structure : + update_request->stream_structures) { + *stream_structure_set.add_structures() = std::move(structure); + } + updates->push_back(MakeKeyAndRecord(std::move(stream_structure_set))); + + return updates; +} + +void SortActions(std::vector<feedstore::StoredAction>* actions) { + std::sort(actions->begin(), actions->end(), + [](const feedstore::StoredAction& a, + const feedstore::StoredAction& b) { return a.id() < b.id(); }); +} + } // namespace FeedStore::LoadStreamResult::LoadStreamResult() = default; @@ -137,8 +203,8 @@ void FeedStore::Initialize(base::OnceClosure initialize_complete) { std::move(initialize_complete).Run(); } else { initialize_callback_ = std::move(initialize_complete); - database_->Init(base::BindOnce(&FeedStore::OnDatabaseInitialized, - weak_ptr_factory_.GetWeakPtr())); + database_->Init( + base::BindOnce(&FeedStore::OnDatabaseInitialized, GetWeakPtr())); } } @@ -183,6 +249,15 @@ void FeedStore::ReadMany( /*target_prefix=*/"", std::move(callback)); } +void FeedStore::ClearAll(base::OnceCallback<void(bool)> callback) { + auto filter = [](const std::string& key) { return true; }; + + database_->UpdateEntriesWithRemoveFilter( + std::make_unique< + std::vector<std::pair<std::string, feedstore::Record>>>(), + base::BindRepeating(filter), std::move(callback)); +} + void FeedStore::LoadStream( base::OnceCallback<void(LoadStreamResult)> callback) { if (!IsInitialized()) { @@ -192,13 +267,15 @@ void FeedStore::LoadStream( return; } auto filter = [](const std::string& key) { - return key == "S/0" || (key.size() > 3 && key[0] == 'T' && key[1] == '/' && - key[2] == '0' && key[3] == '/'); + // Read stream data, stream structures, and pending actions. + return key == kStreamDataKey || + base::StartsWith(key, "T/0/", base::CompareCase::SENSITIVE) || + IsLocalActionKey(key); }; database_->LoadEntriesWithFilter( base::BindRepeating(filter), CreateReadOptions(), /*target_prefix=*/"", - base::BindOnce(&FeedStore::OnLoadStreamFinished, base::Unretained(this), + base::BindOnce(&FeedStore::OnLoadStreamFinished, GetWeakPtr(), std::move(callback))); } @@ -211,38 +288,41 @@ void FeedStore::OnLoadStreamFinished( result.read_error = true; } else { for (feedstore::Record& record : *records) { - if (record.has_stream_structures()) { - result.stream_structures.push_back( - std::move(*record.mutable_stream_structures())); - } else if (record.has_stream_data()) { - result.stream_data = std::move(*record.mutable_stream_data()); + switch (record.data_case()) { + case feedstore::Record::kStreamData: + result.stream_data = std::move(*record.mutable_stream_data()); + break; + case feedstore::Record::kStreamStructures: + result.stream_structures.push_back( + std::move(*record.mutable_stream_structures())); + break; + case feedstore::Record::kLocalAction: + result.pending_actions.push_back( + std::move(*record.mutable_local_action())); + break; + default: + break; } } } + + SortActions(&result.pending_actions); std::move(callback).Run(std::move(result)); } -void FeedStore::SaveFullStream( +void FeedStore::OverwriteStream( std::unique_ptr<StreamModelUpdateRequest> update_request, base::OnceCallback<void(bool)> callback) { - auto updates = std::make_unique< - std::vector<std::pair<std::string, feedstore::Record>>>(); - updates->push_back(MakeKeyAndRecord(std::move(update_request->stream_data))); - for (feedstore::Content& content : update_request->content) { - updates->push_back(MakeKeyAndRecord(std::move(content))); - } - for (feedstore::StreamSharedState& shared_state : - update_request->shared_states) { - updates->push_back(MakeKeyAndRecord(std::move(shared_state))); - } - feedstore::StreamStructureSet stream_structure_set; - stream_structure_set.set_stream_id(kMainStreamId); - for (feedstore::StreamStructure& structure : - update_request->stream_structures) { - *stream_structure_set.add_structures() = std::move(structure); - } - updates->push_back(MakeKeyAndRecord(std::move(stream_structure_set))); + std::unique_ptr<std::vector<std::pair<std::string, feedstore::Record>>> + updates = MakeUpdatesForStreamModelUpdateRequest( + /*structure_set_sequence_number=*/0, std::move(update_request)); + UpdateFullStreamData(std::move(updates), std::move(callback)); +} +void FeedStore::UpdateFullStreamData( + std::unique_ptr<std::vector<std::pair<std::string, feedstore::Record>>> + updates, + base::OnceCallback<void(bool)> callback) { // Set up a filter to delete all stream-related data. // But we need to exclude keys being written right now. std::vector<std::string> key_vector(updates->size()); @@ -253,16 +333,32 @@ void FeedStore::SaveFullStream( auto filter = [](const base::flat_set<std::string>& updated_keys, const std::string& key) { - if (key.empty() || updated_keys.contains(key)) - return false; - return key[0] == 'S' || key[0] == 'T' || key[0] == 'c' || key[0] == 's' || - key[0] == 'N'; + return IsStreamRecordKey(key) && !updated_keys.contains(key); }; database_->UpdateEntriesWithRemoveFilter( std::move(updates), base::BindRepeating(filter, std::move(updated_keys)), - base::BindOnce(&FeedStore::OnSaveStreamEntriesUpdated, - base::Unretained(this), std::move(callback))); + base::BindOnce(&FeedStore::OnSaveStreamEntriesUpdated, GetWeakPtr(), + std::move(callback))); +} + +void FeedStore::SaveStreamUpdate( + int32_t structure_set_sequence_number, + std::unique_ptr<StreamModelUpdateRequest> update_request, + base::OnceCallback<void(bool)> callback) { + database_->UpdateEntries( + MakeUpdatesForStreamModelUpdateRequest(structure_set_sequence_number, + std::move(update_request)), + std::make_unique<leveldb_proto::KeyVector>(), + base::BindOnce(&FeedStore::OnSaveStreamEntriesUpdated, GetWeakPtr(), + std::move(callback))); +} + +void FeedStore::ClearStreamData(base::OnceCallback<void(bool)> callback) { + UpdateFullStreamData( + std::make_unique< + std::vector<std::pair<std::string, feedstore::Record>>>(), + std::move(callback)); } void FeedStore::OnSaveStreamEntriesUpdated( @@ -310,8 +406,7 @@ void FeedStore::ReadContent( key_vector.push_back(SharedStateKey(shared_state_id)); ReadMany(base::flat_set<std::string>(std::move(key_vector)), - base::BindOnce(&FeedStore::OnReadContentFinished, - weak_ptr_factory_.GetWeakPtr(), + base::BindOnce(&FeedStore::OnReadContentFinished, GetWeakPtr(), std::move(content_callback))); } @@ -340,27 +435,73 @@ void FeedStore::OnReadContentFinished( std::move(callback).Run(std::move(content), std::move(shared_states)); } -void FeedStore::ReadNextStreamState( - base::OnceCallback<void(std::unique_ptr<feedstore::StreamAndContentState>)> - callback) { - ReadSingle( - kNextStreamStateKey, - base::BindOnce(&FeedStore::OnReadNextStreamStateFinished, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +void FeedStore::ReadActions( + base::OnceCallback<void(std::vector<feedstore::StoredAction>)> callback) { + database_->LoadEntriesWithFilter( + base::BindRepeating(IsLocalActionKey), + base::BindOnce(&FeedStore::OnReadActionsFinished, GetWeakPtr(), + std::move(callback))); } -void FeedStore::OnReadNextStreamStateFinished( - base::OnceCallback<void(std::unique_ptr<feedstore::StreamAndContentState>)> - callback, +void FeedStore::OnReadActionsFinished( + base::OnceCallback<void(std::vector<feedstore::StoredAction>)> callback, bool success, - std::unique_ptr<feedstore::Record> record) { - if (!success || !record) { - std::move(callback).Run(nullptr); + std::unique_ptr<std::vector<feedstore::Record>> records) { + if (!success || !records) { + std::move(callback).Run({}); return; } - std::move(callback).Run( - base::WrapUnique(record->release_next_stream_state())); + std::vector<feedstore::StoredAction> actions; + actions.reserve(records->size()); + for (auto& record : *records) + actions.push_back(std::move(record.local_action())); + + SortActions(&actions); + std::move(callback).Run(std::move(actions)); +} + +void FeedStore::WriteActions(std::vector<feedstore::StoredAction> actions, + base::OnceCallback<void(bool)> callback) { + std::vector<feedstore::Record> records; + records.reserve(actions.size()); + for (auto& action : actions) { + feedstore::Record record; + *record.mutable_local_action() = std::move(action); + records.push_back(record); + } + + Write(std::move(records), std::move(callback)); +} + +void FeedStore::UpdateActions( + std::vector<feedstore::StoredAction> actions_to_update, + std::vector<LocalActionId> ids_to_remove, + base::OnceCallback<void(bool)> callback) { + auto entries_to_save = std::make_unique< + leveldb_proto::ProtoDatabase<feedstore::Record>::KeyEntryVector>(); + for (auto& action : actions_to_update) + entries_to_save->push_back(MakeKeyAndRecord(std::move(action))); + + auto keys_to_remove = std::make_unique<std::vector<std::string>>(); + for (LocalActionId id : ids_to_remove) + keys_to_remove->push_back(LocalActionKey(id)); + + database_->UpdateEntries(std::move(entries_to_save), + std::move(keys_to_remove), std::move(callback)); +} + +void FeedStore::RemoveActions(std::vector<LocalActionId> ids, + base::OnceCallback<void(bool)> callback) { + auto keys = std::make_unique<std::vector<std::string>>(); + keys->reserve(ids.size()); + for (LocalActionId id : ids) + keys->push_back(LocalActionKey(id)); + + database_->UpdateEntries( + /*entries_to_save=*/std::make_unique< + std::vector<std::pair<std::string, feedstore::Record>>>(), + /*key_to_remove=*/std::move(keys), std::move(callback)); } void FeedStore::Write(std::vector<feedstore::Record> records, @@ -376,8 +517,8 @@ void FeedStore::Write(std::vector<feedstore::Record> records, database_->UpdateEntries( std::move(entries_to_save), /*keys_to_remove=*/std::make_unique<leveldb_proto::KeyVector>(), - base::BindOnce(&FeedStore::OnWriteFinished, - weak_ptr_factory_.GetWeakPtr(), std::move(callback))); + base::BindOnce(&FeedStore::OnWriteFinished, GetWeakPtr(), + std::move(callback))); } void FeedStore::OnWriteFinished(base::OnceCallback<void(bool)> callback, @@ -385,4 +526,27 @@ void FeedStore::OnWriteFinished(base::OnceCallback<void(bool)> callback, std::move(callback).Run(success); } +void FeedStore::ReadMetadata( + base::OnceCallback<void(std::unique_ptr<feedstore::Metadata>)> callback) { + ReadSingle(kMetadataKey, base::BindOnce(&FeedStore::OnReadMetadataFinished, + GetWeakPtr(), std::move(callback))); +} + +void FeedStore::OnReadMetadataFinished( + base::OnceCallback<void(std::unique_ptr<feedstore::Metadata>)> callback, + bool read_ok, + std::unique_ptr<feedstore::Record> record) { + if (!record || !read_ok) { + std::move(callback).Run(nullptr); + return; + } + + std::move(callback).Run(base::WrapUnique(record->release_metadata())); +} + +void FeedStore::WriteMetadata(feedstore::Metadata metadata, + base::OnceCallback<void(bool)> callback) { + Write({MakeRecord(std::move(metadata))}, std::move(callback)); +} + } // namespace feed diff --git a/chromium/components/feed/core/v2/feed_store.h b/chromium/components/feed/core/v2/feed_store.h index 3ae93ead5b6..e0363db2cf5 100644 --- a/chromium/components/feed/core/v2/feed_store.h +++ b/chromium/components/feed/core/v2/feed_store.h @@ -13,6 +13,7 @@ #include "base/memory/weak_ptr.h" #include "base/sequenced_task_runner.h" #include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/v2/types.h" #include "components/leveldb_proto/public/proto_database.h" #include "components/leveldb_proto/public/proto_database_provider.h" @@ -26,12 +27,12 @@ class FeedStore { ~LoadStreamResult(); LoadStreamResult(LoadStreamResult&&); LoadStreamResult& operator=(LoadStreamResult&&); - LoadStreamResult(const LoadStreamResult&) = delete; - LoadStreamResult& operator=(const LoadStreamResult&) = delete; bool read_error = false; feedstore::StreamData stream_data; std::vector<feedstore::StreamStructureSet> stream_structures; + // These are sorted by increasing ID. + std::vector<feedstore::StoredAction> pending_actions; }; explicit FeedStore( @@ -43,10 +44,24 @@ class FeedStore { void Initialize(base::OnceClosure initialize_complete); + // Erase all data in the store. + void ClearAll(base::OnceCallback<void(bool)> callback); + void LoadStream(base::OnceCallback<void(LoadStreamResult)> callback); - void SaveFullStream(std::unique_ptr<StreamModelUpdateRequest> update_request, - base::OnceCallback<void(bool)> callback); + // Stores the content of |update_request| in place of any existing stream + // data. + void OverwriteStream(std::unique_ptr<StreamModelUpdateRequest> update_request, + base::OnceCallback<void(bool)> callback); + + // Stores the content of |update_request| as an update to existing stream + // data. + void SaveStreamUpdate( + int32_t structure_set_sequence_number, + std::unique_ptr<StreamModelUpdateRequest> update_request, + base::OnceCallback<void(bool)> callback); + + void ClearStreamData(base::OnceCallback<void(bool)> callback); void WriteOperations(int32_t sequence_number, std::vector<feedstore::DataOperation> operations); @@ -65,11 +80,20 @@ class FeedStore { std::vector<feedstore::StreamSharedState>)> content_callback); - void ReadNextStreamState( - base::OnceCallback< - void(std::unique_ptr<feedstore::StreamAndContentState>)> callback); - - // TODO(iwells): implement reading stored actions + void ReadActions( + base::OnceCallback<void(std::vector<feedstore::StoredAction>)> callback); + void WriteActions(std::vector<feedstore::StoredAction> actions, + base::OnceCallback<void(bool)> callback); + void UpdateActions(std::vector<feedstore::StoredAction> actions_to_update, + std::vector<LocalActionId> ids_to_remove, + base::OnceCallback<void(bool)> callback); + void RemoveActions(std::vector<LocalActionId> ids, + base::OnceCallback<void(bool)> callback); + + void ReadMetadata( + base::OnceCallback<void(std::unique_ptr<feedstore::Metadata>)> callback); + void WriteMetadata(feedstore::Metadata metadata, + base::OnceCallback<void(bool)> callback); // TODO(iwells): implement this // Deletes old records that are no longer needed @@ -77,13 +101,25 @@ class FeedStore { bool IsInitializedForTesting() const; + leveldb_proto::ProtoDatabase<feedstore::Record>* GetDatabaseForTesting() { + return database_.get(); + } + + base::WeakPtr<FeedStore> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + private: void OnDatabaseInitialized(leveldb_proto::Enums::InitStatus status); bool IsInitialized() const; + // Overwrites all stream data with |updates|. + void UpdateFullStreamData( + std::unique_ptr<std::vector<std::pair<std::string, feedstore::Record>>> + updates, + base::OnceCallback<void(bool)> callback); void Write(std::vector<feedstore::Record> records, base::OnceCallback<void(bool)> callback); - void ReadSingle( const std::string& key, base::OnceCallback<void(bool, std::unique_ptr<feedstore::Record>)> @@ -105,13 +141,15 @@ class FeedStore { callback, bool success, std::unique_ptr<std::vector<feedstore::Record>> records); - void OnReadNextStreamStateFinished( - base::OnceCallback< - void(std::unique_ptr<feedstore::StreamAndContentState>)> callback, + void OnReadActionsFinished( + base::OnceCallback<void(std::vector<feedstore::StoredAction>)> callback, bool success, - std::unique_ptr<feedstore::Record> record); - + std::unique_ptr<std::vector<feedstore::Record>> records); void OnWriteFinished(base::OnceCallback<void(bool)> callback, bool success); + void OnReadMetadataFinished( + base::OnceCallback<void(std::unique_ptr<feedstore::Metadata>)> callback, + bool read_ok, + std::unique_ptr<feedstore::Record> record); // TODO(iwells): implement // bool OldRecordFilter(const std::string& key); diff --git a/chromium/components/feed/core/v2/feed_store_unittest.cc b/chromium/components/feed/core/v2/feed_store_unittest.cc index 28e2043fc12..db7473c2d58 100644 --- a/chromium/components/feed/core/v2/feed_store_unittest.cc +++ b/chromium/components/feed/core/v2/feed_store_unittest.cc @@ -14,7 +14,7 @@ #include "base/test/bind_test_util.h" #include "base/test/task_environment.h" #include "components/feed/core/proto/v2/wire/content_id.pb.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/test/callback_receiver.h" #include "components/feed/core/v2/test/proto_printer.h" #include "components/feed/core/v2/test/stream_builder.h" @@ -24,22 +24,8 @@ namespace feed { namespace { -const char kNextPageToken[] = "next page token"; -const char kConsistencyToken[] = "consistency token"; -const int64_t kLastAddedTimeMs = 100; - using LoadStreamResult = FeedStore::LoadStreamResult; -feedstore::StreamData MakeStreamData() { - feedstore::StreamData stream_data; - *stream_data.mutable_content_id() = MakeRootId(); - stream_data.set_next_page_token(kNextPageToken); - stream_data.set_consistency_token(kConsistencyToken); - stream_data.set_last_added_time_millis(kLastAddedTimeMs); - - return stream_data; -} - std::string KeyForContentId(base::StringPiece prefix, const feedwire::ContentId& content_id) { return base::StrCat({prefix, content_id.content_domain(), ",", @@ -59,6 +45,18 @@ feedstore::Record RecordForSharedState(feedstore::StreamSharedState shared) { return record; } +feedstore::Record RecordForAction(feedstore::StoredAction action) { + feedstore::Record record; + *record.mutable_local_action() = std::move(action); + return record; +} + +feedstore::StoredAction MakeAction(int32_t id) { + feedstore::StoredAction action; + action.set_id(id); + return action; +} + } // namespace class FeedStoreTest : public testing::Test { @@ -119,10 +117,10 @@ TEST_F(FeedStoreTest, InitFailure) { EXPECT_FALSE(store->IsInitializedForTesting()); } -TEST_F(FeedStoreTest, SaveFullStream) { +TEST_F(FeedStoreTest, OverwriteStream) { MakeFeedStore({}); CallbackReceiver<bool> receiver; - store_->SaveFullStream(MakeTypicalInitialModelState(), receiver.Bind()); + store_->OverwriteStream(MakeTypicalInitialModelState(), receiver.Bind()); fake_db_->UpdateCallback(true); ASSERT_TRUE(receiver.GetResult()); @@ -132,6 +130,7 @@ TEST_F(FeedStoreTest, SaveFullStream) { content_id { content_domain: "root" } + next_page_token: "page-2" shared_state_id { content_domain: "render_data" } @@ -231,7 +230,7 @@ TEST_F(FeedStoreTest, SaveFullStream) { )"); } -TEST_F(FeedStoreTest, SaveFullStreamOverwritesData) { +TEST_F(FeedStoreTest, OverwriteStreamOverwritesData) { MakeFeedStore({}); // Insert some junk that should be removed. db_entries_["S/0"].mutable_local_action()->set_id(6); @@ -244,7 +243,7 @@ TEST_F(FeedStoreTest, SaveFullStreamOverwritesData) { db_entries_["s/garbage,0,0"].mutable_local_action()->set_id(6); CallbackReceiver<bool> receiver; - store_->SaveFullStream(MakeTypicalInitialModelState(), receiver.Bind()); + store_->OverwriteStream(MakeTypicalInitialModelState(), receiver.Bind()); fake_db_->UpdateCallback(true); ASSERT_TRUE(receiver.GetResult()); @@ -260,13 +259,13 @@ TEST_F(FeedStoreTest, SaveFullStreamOverwritesData) { for (std::string key : StoredKeys()) { EXPECT_FALSE(db_entries_[key].has_local_action()) << "Found local action at key " << key - << ", did SaveFullStream erase everything?"; + << ", did OverwriteStream erase everything?"; } } TEST_F(FeedStoreTest, LoadStreamSuccess) { MakeFeedStore({}); - store_->SaveFullStream(MakeTypicalInitialModelState(), base::DoNothing()); + store_->OverwriteStream(MakeTypicalInitialModelState(), base::DoNothing()); fake_db_->UpdateCallback(true); CallbackReceiver<LoadStreamResult> receiver; @@ -281,7 +280,7 @@ TEST_F(FeedStoreTest, LoadStreamSuccess) { TEST_F(FeedStoreTest, LoadStreamFail) { MakeFeedStore({}); - store_->SaveFullStream(MakeTypicalInitialModelState(), base::DoNothing()); + store_->OverwriteStream(MakeTypicalInitialModelState(), base::DoNothing()); fake_db_->UpdateCallback(true); CallbackReceiver<LoadStreamResult> receiver; @@ -345,19 +344,18 @@ TEST_F(FeedStoreTest, WriteOperations) { TEST_F(FeedStoreTest, ReadNonexistentContentAndSharedStates) { MakeFeedStore({}); + CallbackReceiver<std::vector<feedstore::Content>, + std::vector<feedstore::StreamSharedState>> + cr; - bool did_read = false; - store_->ReadContent( - {MakeContentContentId(0)}, {MakeSharedStateContentId(0)}, - base::BindLambdaForTesting( - [&](std::vector<feedstore::Content> content, - std::vector<feedstore::StreamSharedState> shared_states) { - did_read = true; - EXPECT_EQ(content.size(), 0ul); - EXPECT_EQ(shared_states.size(), 0ul); - })); + store_->ReadContent({MakeContentContentId(0)}, {MakeSharedStateContentId(0)}, + cr.Bind()); fake_db_->LoadCallback(true); - EXPECT_TRUE(did_read); + + ASSERT_NE(cr.GetResult<0>(), base::nullopt); + EXPECT_EQ(cr.GetResult<0>()->size(), 0ul); + ASSERT_NE(cr.GetResult<1>(), base::nullopt); + EXPECT_EQ(cr.GetResult<1>()->size(), 0ul); } TEST_F(FeedStoreTest, ReadContentAndSharedStates) { @@ -380,75 +378,176 @@ TEST_F(FeedStoreTest, ReadContentAndSharedStates) { std::vector<feedwire::ContentId> shared_state_ids = {shared1.content_id(), shared2.content_id()}; + CallbackReceiver<std::vector<feedstore::Content>, + std::vector<feedstore::StreamSharedState>> + cr; + // Successful read - bool did_successful_read = false; - store_->ReadContent( - content_ids, shared_state_ids, - base::BindLambdaForTesting( - [&](std::vector<feedstore::Content> content, - std::vector<feedstore::StreamSharedState> shared_states) { - did_successful_read = true; - ASSERT_EQ(content.size(), 2ul); - EXPECT_EQ(ToTextProto(content[0].content_id()), - ToTextProto(content1.content_id())); - EXPECT_EQ(content[0].frame(), content1.frame()); - - ASSERT_EQ(shared_states.size(), 2ul); - EXPECT_EQ(ToTextProto(shared_states[0].content_id()), - ToTextProto(shared1.content_id())); - EXPECT_EQ(shared_states[0].shared_state_data(), - shared1.shared_state_data()); - })); + store_->ReadContent(content_ids, shared_state_ids, cr.Bind()); fake_db_->LoadCallback(true); - EXPECT_TRUE(did_successful_read); + + ASSERT_NE(cr.GetResult<0>(), base::nullopt); + std::vector<feedstore::Content> content = *cr.GetResult<0>(); + ASSERT_NE(cr.GetResult<1>(), base::nullopt); + std::vector<feedstore::StreamSharedState> shared_states = *cr.GetResult<1>(); + + ASSERT_EQ(content.size(), 2ul); + EXPECT_EQ(ToTextProto(content[0].content_id()), + ToTextProto(content1.content_id())); + EXPECT_EQ(content[0].frame(), content1.frame()); + + ASSERT_EQ(shared_states.size(), 2ul); + EXPECT_EQ(ToTextProto(shared_states[0].content_id()), + ToTextProto(shared1.content_id())); + EXPECT_EQ(shared_states[0].shared_state_data(), shared1.shared_state_data()); // Failed read - bool did_failed_read = false; - store_->ReadContent( - content_ids, shared_state_ids, - base::BindLambdaForTesting( - [&](std::vector<feedstore::Content> content, - std::vector<feedstore::StreamSharedState> shared_states) { - did_failed_read = true; - EXPECT_EQ(content.size(), 0ul); - EXPECT_EQ(shared_states.size(), 0ul); - })); + cr.Clear(); + store_->ReadContent(content_ids, shared_state_ids, cr.Bind()); fake_db_->LoadCallback(false); - EXPECT_TRUE(did_failed_read); -} -TEST_F(FeedStoreTest, ReadNextStreamState) { - feedstore::Record record; - feedstore::StreamAndContentState* next_stream_state = - record.mutable_next_stream_state(); - *next_stream_state->mutable_stream_data() = MakeStreamData(); - *next_stream_state->add_content() = MakeContent(0); - *next_stream_state->add_shared_state() = MakeSharedState(0); + ASSERT_NE(cr.GetResult<0>(), base::nullopt); + EXPECT_EQ(cr.GetResult<0>()->size(), 0ul); + ASSERT_NE(cr.GetResult<1>(), base::nullopt); + EXPECT_EQ(cr.GetResult<1>()->size(), 0ul); +} - MakeFeedStore({{"N", record}}); +TEST_F(FeedStoreTest, ReadActions) { + MakeFeedStore({{"a/0", RecordForAction(MakeAction(0))}, + {"a/1", RecordForAction(MakeAction(1))}, + {"a/2", RecordForAction(MakeAction(2))}}); // Successful read - bool did_successful_read = false; - store_->ReadNextStreamState(base::BindLambdaForTesting( - [&](std::unique_ptr<feedstore::StreamAndContentState> result) { - did_successful_read = true; - ASSERT_TRUE(result); - EXPECT_TRUE(result->has_stream_data()); - EXPECT_EQ(result->content_size(), 1); - EXPECT_EQ(result->shared_state_size(), 1); - })); - fake_db_->GetCallback(true); - EXPECT_TRUE(did_successful_read); + CallbackReceiver<std::vector<feedstore::StoredAction>> receiver; + store_->ReadActions(receiver.Bind()); + fake_db_->LoadCallback(true); + ASSERT_NE(base::nullopt, receiver.GetResult()); + std::vector<feedstore::StoredAction> result = + std::move(*receiver.GetResult()); + + EXPECT_EQ(3ul, result.size()); + EXPECT_EQ(2, result[2].id()); // Failed read - bool did_failed_read = false; - store_->ReadNextStreamState(base::BindLambdaForTesting( - [&](std::unique_ptr<feedstore::StreamAndContentState> result) { - did_failed_read = true; - EXPECT_FALSE(result); - })); + receiver.Clear(); + store_->ReadActions(receiver.Bind()); + fake_db_->LoadCallback(false); + ASSERT_NE(base::nullopt, receiver.GetResult()); + result = std::move(*receiver.GetResult()); + EXPECT_EQ(0ul, result.size()); +} + +TEST_F(FeedStoreTest, WriteActions) { + MakeFeedStore({}); + feedstore::StoredAction action; + + CallbackReceiver<bool> receiver; + store_->WriteActions({action}, receiver.Bind()); + fake_db_->UpdateCallback(true); + ASSERT_TRUE(receiver.GetResult()); + EXPECT_TRUE(*receiver.GetResult()); + + ASSERT_EQ(1ul, db_entries_.size()); + EXPECT_EQ(0, db_entries_["a/0"].local_action().id()); + + receiver.GetResult().reset(); + store_->WriteActions({action}, receiver.Bind()); + fake_db_->UpdateCallback(false); + EXPECT_NE(receiver.GetResult(), base::nullopt); + EXPECT_EQ(receiver.GetResult().value(), false); +} + +TEST_F(FeedStoreTest, RemoveActions) { + MakeFeedStore({{"a/0", RecordForAction(MakeAction(0))}, + {"a/1", RecordForAction(MakeAction(1))}, + {"a/2", RecordForAction(MakeAction(2))}}); + + const std::vector<LocalActionId> ids = {LocalActionId(0), LocalActionId(1), + LocalActionId(2)}; + + CallbackReceiver<bool> receiver; + store_->RemoveActions(ids, receiver.Bind()); + fake_db_->UpdateCallback(true); + EXPECT_EQ(receiver.GetResult().value(), true); + EXPECT_EQ(db_entries_.size(), 0ul); + + receiver.GetResult().reset(); + store_->RemoveActions(ids, receiver.Bind()); + fake_db_->UpdateCallback(false); + EXPECT_NE(receiver.GetResult(), base::nullopt); + EXPECT_EQ(receiver.GetResult().value(), false); +} + +TEST_F(FeedStoreTest, ClearAllSuccess) { + // Write at least one record of each type. + MakeFeedStore({}); + store_->OverwriteStream(MakeTypicalInitialModelState(), base::DoNothing()); + fake_db_->UpdateCallback(true); + store_->WriteActions({MakeAction(0)}, base::DoNothing()); + fake_db_->UpdateCallback(true); + ASSERT_NE("", StoreToString()); + + // ClearAll() and verify the DB is empty. + CallbackReceiver<bool> receiver; + store_->ClearAll(receiver.Bind()); + fake_db_->UpdateCallback(true); + + ASSERT_TRUE(receiver.GetResult()); + EXPECT_TRUE(*receiver.GetResult()); + EXPECT_EQ("", StoreToString()); +} + +TEST_F(FeedStoreTest, ClearAllFail) { + // Just verify that we can handle a storage failure. Note that |FakeDB| will + // actually perform operations even when UpdateCallback(false) is called. + MakeFeedStore({}); + + CallbackReceiver<bool> receiver; + store_->ClearAll(receiver.Bind()); + fake_db_->UpdateCallback(false); + + ASSERT_TRUE(receiver.GetResult()); + EXPECT_FALSE(*receiver.GetResult()); +} + +TEST_F(FeedStoreTest, ReadMetadata) { + feedstore::Record record; + record.mutable_metadata()->set_consistency_token("token"); + record.mutable_metadata()->set_next_action_id(20); + MakeFeedStore({{"m", record}}); + + CallbackReceiver<std::unique_ptr<feedstore::Metadata>> cr; + store_->ReadMetadata(cr.Bind()); + fake_db_->GetCallback(true); + ASSERT_TRUE(cr.GetResult()); + + std::unique_ptr<feedstore::Metadata> metadata = std::move(*cr.GetResult()); + ASSERT_TRUE(metadata); + EXPECT_EQ("token", metadata->consistency_token()); + EXPECT_EQ(20, metadata->next_action_id()); + + store_->ReadMetadata(cr.Bind()); fake_db_->GetCallback(false); - EXPECT_TRUE(did_failed_read); + ASSERT_TRUE(cr.GetResult()); + EXPECT_FALSE(*cr.GetResult()); +} + +TEST_F(FeedStoreTest, WriteMetadata) { + MakeFeedStore({}); + + feedstore::Metadata metadata; + metadata.set_consistency_token("token"); + metadata.set_next_action_id(20); + + CallbackReceiver<bool> cr; + store_->WriteMetadata(metadata, cr.Bind()); + fake_db_->UpdateCallback(true); + ASSERT_TRUE(cr.GetResult()); + EXPECT_TRUE(*cr.GetResult()); + + ASSERT_EQ(1ul, db_entries_.size()); + EXPECT_EQ("token", db_entries_["m"].metadata().consistency_token()); + EXPECT_EQ(20, db_entries_["m"].metadata().next_action_id()); } } // namespace feed diff --git a/chromium/components/feed/core/v2/feed_stream.cc b/chromium/components/feed/core/v2/feed_stream.cc index fe1e13c33f6..4544d65915c 100644 --- a/chromium/components/feed/core/v2/feed_stream.cc +++ b/chromium/components/feed/core/v2/feed_stream.cc @@ -19,203 +19,95 @@ #include "components/feed/core/v2/enums.h" #include "components/feed/core/v2/feed_network.h" #include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/metrics_reporter.h" +#include "components/feed/core/v2/prefs.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/refresh_task_scheduler.h" #include "components/feed/core/v2/scheduling.h" #include "components/feed/core/v2/stream_model.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/surface_updater.h" +#include "components/feed/core/v2/tasks/clear_all_task.h" #include "components/feed/core/v2/tasks/load_stream_task.h" +#include "components/feed/core/v2/tasks/upload_actions_task.h" #include "components/feed/core/v2/tasks/wait_for_store_initialize_task.h" +#include "components/offline_pages/task/closure_task.h" #include "components/prefs/pref_service.h" namespace feed { +namespace { -// Tracks UI changes in |StreamModel| and forwards them to |SurfaceInterface|s. -// TODO(harringtond): implement spinner slice. -class FeedStream::SurfaceUpdater : public StreamModel::Observer { - public: - using ContentRevision = ContentRevision; - explicit SurfaceUpdater(const base::ObserverList<SurfaceInterface>* surfaces) - : surfaces_(surfaces) {} - ~SurfaceUpdater() override = default; - SurfaceUpdater(const SurfaceUpdater&) = delete; - SurfaceUpdater& operator=(const SurfaceUpdater&) = delete; - - void SetModel(StreamModel* model) { - if (model_ == model) - return; - if (model_) - model_->SetObserver(nullptr); - model_ = model; - if (model_) { - model_->SetObserver(this); - - const std::vector<ContentRevision>& content_list = - model_->GetContentList(); - current_content_set_.insert(content_list.begin(), content_list.end()); - for (SurfaceInterface& surface : *surfaces_) { - surface.StreamUpdate(GetUpdateForNewSurface(*model_)); - } - } - } - - // StreamModel::Observer. - void OnUiUpdate(const StreamModel::UiUpdate& update) override { - DCHECK(model_); // The update comes from the model. - - if (!update.content_list_changed) - return; - feedui::StreamUpdate stream_update; - const std::vector<ContentRevision>& content_list = model_->GetContentList(); - for (ContentRevision content_revision : content_list) { - AddSliceUpdate(*model_, content_revision, - current_content_set_.count(content_revision) == 0, - &stream_update); - } - for (const StreamModel::UiUpdate::SharedStateInfo& info : - update.shared_states) { - if (info.updated) - AddSharedState(*model_, info.shared_state_id, &stream_update); - } - - current_content_set_.clear(); - current_content_set_.insert(content_list.begin(), content_list.end()); - - for (SurfaceInterface& surface : *surfaces_) { - surface.StreamUpdate(stream_update); - } - } - - // Sends the initial stream state to a newly connected surface. - void SurfaceAdded(SurfaceInterface* surface) { - if (model_) { - surface->StreamUpdate(GetUpdateForNewSurface(*model_)); - } - } - - void LoadStreamFailed(LoadStreamStatus load_stream_status) { - auto zero_state_type = feedui::ZeroStateSlice::NO_CARDS_AVAILABLE; - switch (load_stream_status) { - case LoadStreamStatus::kProtoTranslationFailed: - case LoadStreamStatus::kNoResponseBody: - case LoadStreamStatus::kCannotLoadFromNetworkOffline: - case LoadStreamStatus::kCannotLoadFromNetworkThrottled: - zero_state_type = feedui::ZeroStateSlice::CANT_REFRESH; - break; - default: - break; - } - // Note that with multiple surface, it's possible that we send a zero-state - // to a single surface multiple times. - for (SurfaceInterface& surface : *surfaces_) { - SendZeroStateUpdate(zero_state_type, &surface); - } - } - - private: - static std::string ToSliceId(ContentRevision content_revision) { - auto integer_value = content_revision.value(); - return std::string(reinterpret_cast<char*>(&integer_value), - sizeof(integer_value)); - } - - static feedui::StreamUpdate GetUpdateForNewSurface(const StreamModel& model) { - feedui::StreamUpdate result; - for (ContentRevision content_revision : model.GetContentList()) { - AddSliceUpdate(model, content_revision, /*is_content_new=*/true, &result); - } - for (std::string& id : model.GetSharedStateIds()) { - AddSharedState(model, id, &result); - } +void PopulateDebugStreamData(const LoadStreamTask::Result& load_result, + PrefService* profile_prefs) { + DebugStreamData debug_data = ::feed::prefs::GetDebugStreamData(profile_prefs); + std::stringstream ss; + ss << "Code: " << load_result.final_status; + debug_data.load_stream_status = ss.str(); + debug_data.fetch_info = load_result.network_response_info; + ::feed::prefs::SetDebugStreamData(debug_data, profile_prefs); +} - return result; - } +} // namespace - static void SendZeroStateUpdate(feedui::ZeroStateSlice::Type zero_state_type, - SurfaceInterface* surface) { - feedui::StreamUpdate update; - feedui::Slice* slice = update.add_updated_slices()->mutable_slice(); - slice->mutable_zero_state_slice()->set_type(zero_state_type); - slice->set_slice_id("zero-state"); - surface->StreamUpdate(update); - } +RefreshResponseData FeedStream::WireResponseTranslator::TranslateWireResponse( + feedwire::Response response, + StreamModelUpdateRequest::Source source, + base::Time current_time) { + return ::feed::TranslateWireResponse(std::move(response), source, + current_time); +} - static void AddSharedState(const StreamModel& model, - const std::string& shared_state_id, - feedui::StreamUpdate* stream_update) { - const std::string* shared_state_data = - model.FindSharedStateData(shared_state_id); - if (!shared_state_data) - return; - feedui::SharedState* added_shared_state = - stream_update->add_new_shared_states(); - added_shared_state->set_id(shared_state_id); - added_shared_state->set_xsurface_shared_state(*shared_state_data); - } +FeedStream::Metadata::Metadata(FeedStore* store) : store_(store) {} +FeedStream::Metadata::~Metadata() = default; - static void AddSliceUpdate(const StreamModel& model, - ContentRevision content_revision, - bool is_content_new, - feedui::StreamUpdate* stream_update) { - if (is_content_new) { - feedui::Slice* slice = - stream_update->add_updated_slices()->mutable_slice(); - slice->set_slice_id(ToSliceId(content_revision)); - const feedstore::Content* content = model.FindContent(content_revision); - DCHECK(content); - slice->mutable_xsurface_slice()->set_xsurface_frame(content->frame()); - } else { - stream_update->add_updated_slices()->set_slice_id( - ToSliceId(content_revision)); - } - } +void FeedStream::Metadata::Populate(feedstore::Metadata metadata) { + metadata_ = std::move(metadata); +} - // Owned by |FeedStream|. - // Warning!: Null when the model is not yet loaded. - StreamModel* model_ = nullptr; - const base::ObserverList<SurfaceInterface>* surfaces_; +std::string FeedStream::Metadata::GetConsistencyToken() const { + return metadata_.consistency_token(); +} - std::set<ContentRevision> current_content_set_; -}; +void FeedStream::Metadata::SetConsistencyToken(std::string consistency_token) { + metadata_.set_consistency_token(std::move(consistency_token)); + store_->WriteMetadata(metadata_, base::DoNothing()); +} -std::unique_ptr<StreamModelUpdateRequest> -FeedStream::WireResponseTranslator::TranslateWireResponse( - feedwire::Response response, - base::TimeDelta response_time, - base::Time current_time) { - return ::feed::TranslateWireResponse(std::move(response), response_time, - current_time); +LocalActionId FeedStream::Metadata::GetNextActionId() { + uint32_t id = metadata_.next_action_id(); + metadata_.set_next_action_id(id + 1); + store_->WriteMetadata(metadata_, base::DoNothing()); + return LocalActionId(id); } -FeedStream::FeedStream( - RefreshTaskScheduler* refresh_task_scheduler, - EventObserver* stream_event_observer, - Delegate* delegate, - PrefService* profile_prefs, - FeedNetwork* feed_network, - FeedStore* feed_store, - const base::Clock* clock, - const base::TickClock* tick_clock, - scoped_refptr<base::SequencedTaskRunner> background_task_runner) +FeedStream::FeedStream(RefreshTaskScheduler* refresh_task_scheduler, + MetricsReporter* metrics_reporter, + Delegate* delegate, + PrefService* profile_prefs, + FeedNetwork* feed_network, + FeedStore* feed_store, + const base::Clock* clock, + const base::TickClock* tick_clock, + const ChromeInfo& chrome_info) : refresh_task_scheduler_(refresh_task_scheduler), - stream_event_observer_(stream_event_observer), + metrics_reporter_(metrics_reporter), delegate_(delegate), profile_prefs_(profile_prefs), feed_network_(feed_network), store_(feed_store), clock_(clock), tick_clock_(tick_clock), - background_task_runner_(background_task_runner), + chrome_info_(chrome_info), task_queue_(this), - user_classifier_(std::make_unique<UserClassifier>(profile_prefs, clock)), - request_throttler_(profile_prefs, clock) { + request_throttler_(profile_prefs, clock), + metadata_(feed_store) { static WireResponseTranslator default_translator; wire_response_translator_ = &default_translator; - surface_updater_ = std::make_unique<SurfaceUpdater>(&surfaces_); + surface_updater_ = std::make_unique<SurfaceUpdater>(metrics_reporter_); // Inserting this task first ensures that |store_| is initialized before // it is used. - task_queue_.AddTask(std::make_unique<WaitForStoreInitializeTask>(store_)); + task_queue_.AddTask(std::make_unique<WaitForStoreInitializeTask>(this)); } void FeedStream::InitializeScheduling() { @@ -223,9 +115,6 @@ void FeedStream::InitializeScheduling() { refresh_task_scheduler_->Cancel(); return; } - - refresh_task_scheduler_->EnsureScheduled( - GetUserClassTriggerThreshold(GetUserClass(), TriggerType::kFixedTimer)); } FeedStream::~FeedStream() = default; @@ -235,45 +124,43 @@ void FeedStream::TriggerStreamLoad() { return; // If we should not load the stream, abort and send a zero-state update. - if (!IsArticlesListVisible()) { - LoadStreamTaskComplete(LoadStreamTask::Result( - LoadStreamStatus::kLoadNotAllowedArticlesListHidden)); - return; - } - if (!delegate_->IsEulaAccepted()) { - LoadStreamTaskComplete(LoadStreamTask::Result( - LoadStreamStatus::kLoadNotAllowedEulaNotAccepted)); + LoadStreamStatus do_not_attempt_reason = ShouldAttemptLoad(); + if (do_not_attempt_reason != LoadStreamStatus::kNoStatus) { + InitialStreamLoadComplete(LoadStreamTask::Result(do_not_attempt_reason)); return; } model_loading_in_progress_ = true; + surface_updater_->LoadStreamStarted(); task_queue_.AddTask(std::make_unique<LoadStreamTask>( - this, base::BindOnce(&FeedStream::LoadStreamTaskComplete, - base::Unretained(this)))); + LoadStreamTask::LoadType::kInitialLoad, this, + base::BindOnce(&FeedStream::InitialStreamLoadComplete, + base::Unretained(this)))); } -void FeedStream::LoadStreamTaskComplete(LoadStreamTask::Result result) { - stream_event_observer_->OnLoadStream(result.load_from_store_status, - result.final_status); - DVLOG(1) << "LoadStreamTaskComplete load_from_store_status=" - << result.load_from_store_status - << " final_status=" << result.final_status; +void FeedStream::InitialStreamLoadComplete(LoadStreamTask::Result result) { + PopulateDebugStreamData(result, profile_prefs_); + metrics_reporter_->OnLoadStream(result.load_from_store_status, + result.final_status); + model_loading_in_progress_ = false; - // If loading failed, update surfaces with an appropriate zero-state error. - if (!model_) { - surface_updater_->LoadStreamFailed(result.final_status); - } + surface_updater_->LoadStreamComplete(model_ != nullptr, result.final_status); +} + +void FeedStream::OnEnterBackground() { + metrics_reporter_->OnEnterBackground(); } void FeedStream::AttachSurface(SurfaceInterface* surface) { - surfaces_.AddObserver(surface); - surface_updater_->SurfaceAdded(surface); + metrics_reporter_->SurfaceOpened(surface->GetSurfaceId()); TriggerStreamLoad(); + surface_updater_->SurfaceAdded(surface); } void FeedStream::DetachSurface(SurfaceInterface* surface) { - surfaces_.RemoveObserver(surface); + metrics_reporter_->SurfaceClosed(surface->GetSurfaceId()); + surface_updater_->SurfaceRemoved(surface); } void FeedStream::SetArticlesListVisible(bool is_visible) { @@ -284,6 +171,37 @@ bool FeedStream::IsArticlesListVisible() { return profile_prefs_->GetBoolean(prefs::kArticlesListVisible); } +void FeedStream::LoadMore(SurfaceId surface_id, + base::OnceCallback<void(bool)> callback) { + metrics_reporter_->OnLoadMoreBegin(surface_id); + if (!model_) { + DLOG(ERROR) << "Ignoring LoadMore() before the model is loaded"; + return std::move(callback).Run(false); + } + surface_updater_->SetLoadingMore(true); + // Have at most one in-flight LoadMore() request. Send the result to all + // requestors. + load_more_complete_callbacks_.push_back(std::move(callback)); + if (load_more_complete_callbacks_.size() == 1) { + task_queue_.AddTask(std::make_unique<LoadMoreTask>( + this, + base::BindOnce(&FeedStream::LoadMoreComplete, base::Unretained(this)))); + } +} + +void FeedStream::LoadMoreComplete(LoadMoreTask::Result result) { + metrics_reporter_->OnLoadMore(result.final_status); + // TODO(harringtond): In the case of failure, do we need to load an error + // message slice? + surface_updater_->SetLoadingMore(false); + std::vector<base::OnceCallback<void(bool)>> moved_callbacks = + std::move(load_more_complete_callbacks_); + bool success = result.final_status == LoadStreamStatus::kLoadedFromNetwork; + for (auto& callback : moved_callbacks) { + std::move(callback).Run(success); + } +} + void FeedStream::ExecuteOperations( std::vector<feedstore::DataOperation> operations) { if (!model_) { @@ -314,8 +232,28 @@ bool FeedStream::RejectEphemeralChange(EphemeralChangeId id) { return model_->RejectEphemeralChange(id); } -UserClass FeedStream::GetUserClass() { - return user_classifier_->GetUserClass(); +DebugStreamData FeedStream::GetDebugStreamData() { + return ::feed::prefs::GetDebugStreamData(profile_prefs_); +} + +void FeedStream::ForceRefreshForDebugging() { + task_queue_.AddTask( + std::make_unique<offline_pages::ClosureTask>(base::BindOnce( + &FeedStream::ForceRefreshForDebuggingTask, base::Unretained(this)))); +} + +void FeedStream::ForceRefreshForDebuggingTask() { + UnloadModel(); + store_->ClearStreamData(base::DoNothing()); + TriggerStreamLoad(); +} + +std::string FeedStream::DumpStateForDebugging() { + std::stringstream ss; + if (model_) { + ss << "model loaded, " << model_->GetContentList().size() << " contents\n"; + } + return ss.str(); } base::Time FeedStream::GetLastFetchTime() { @@ -327,6 +265,10 @@ base::Time FeedStream::GetLastFetchTime() { return fetch_time; } +bool FeedStream::HasSurfaceAttached() const { + return surface_updater_->HasSurfaceAttached(); +} + void FeedStream::LoadModelForTesting(std::unique_ptr<StreamModel> model) { LoadModel(std::move(model)); } @@ -344,22 +286,58 @@ void FeedStream::SetIdleCallbackForTesting( idle_callback_ = idle_callback; } -void FeedStream::SetUserClassifierForTesting( - std::unique_ptr<UserClassifier> user_classifier) { - user_classifier_ = std::move(user_classifier); +void FeedStream::OnStoreChange(StreamModel::StoreUpdate update) { + if (!update.operations.empty()) { + DCHECK(!update.update_request); + store_->WriteOperations(update.sequence_number, update.operations); + } else { + DCHECK(update.update_request); + if (update.overwrite_stream_data) { + DCHECK_EQ(update.sequence_number, 0); + store_->OverwriteStream(std::move(update.update_request), + base::DoNothing()); + } else { + store_->SaveStreamUpdate(update.sequence_number, + std::move(update.update_request), + base::DoNothing()); + } + } } -void FeedStream::OnStoreChange(const StreamModel::StoreUpdate& update) { - store_->WriteOperations(update.sequence_number, update.operations); +LoadStreamStatus FeedStream::ShouldAttemptLoad(bool model_loading) { + // Don't try to load the model if it's already loaded, or in the process of + // being loaded. Because |ShouldAttemptLoad()| is used both before and during + // the load process, we need to ignore this check when |model_loading| is + // true. + if (model_ || (!model_loading && model_loading_in_progress_)) + return LoadStreamStatus::kModelAlreadyLoaded; + + if (!IsArticlesListVisible()) + return LoadStreamStatus::kLoadNotAllowedArticlesListHidden; + + if (!delegate_->IsEulaAccepted()) + return LoadStreamStatus::kLoadNotAllowedEulaNotAccepted; + + return LoadStreamStatus::kNoStatus; } -LoadStreamStatus FeedStream::ShouldMakeFeedQueryRequest() { - // TODO(harringtond): |suppress_refreshes_until_| was historically used for - // privacy purposes after clearing data to make sure sync data made it to the - // server. I'm not sure we need this now. But also, it was documented as not - // affecting manually triggered refreshes, but coded in a way that it does. - // I've tried to keep the same functionality as the old feed code, but we - // should revisit this. +LoadStreamStatus FeedStream::ShouldMakeFeedQueryRequest(bool is_load_more) { + if (!is_load_more) { + // Time has passed since calling |ShouldAttemptLoad()|, call it again to + // confirm we should still attempt loading. + const LoadStreamStatus should_not_attempt_reason = + ShouldAttemptLoad(/*model_loading=*/true); + if (should_not_attempt_reason != LoadStreamStatus::kNoStatus) { + return should_not_attempt_reason; + } + } + + // TODO(harringtond): |suppress_refreshes_until_| was historically used + // for privacy purposes after clearing data to make sure sync data made it + // to the server. I'm not sure we need this now. But also, it was + // documented as not affecting manually triggered refreshes, but coded in + // a way that it does. I've tried to keep the same functionality as the + // old feed code, but we should revisit this. if (tick_clock_->NowTicks() < suppress_refreshes_until_) { return LoadStreamStatus::kCannotLoadFromNetworkSupressedForHistoryDelete; } @@ -375,8 +353,17 @@ LoadStreamStatus FeedStream::ShouldMakeFeedQueryRequest() { return LoadStreamStatus::kNoStatus; } +RequestMetadata FeedStream::GetRequestMetadata() const { + RequestMetadata result; + result.chrome_info = chrome_info_; + result.display_metrics = delegate_->GetDisplayMetrics(); + result.language_tag = delegate_->GetLanguageTag(); + return result; +} + void FeedStream::OnEulaAccepted() { - MaybeTriggerRefresh(TriggerType::kForegrounded); + if (surface_updater_->HasSurfaceAttached()) + TriggerStreamLoad(); } void FeedStream::OnHistoryDeleted() { @@ -400,35 +387,40 @@ void FeedStream::OnSignedOut() { ClearAll(); } -void FeedStream::OnEnterForeground() { - MaybeTriggerRefresh(TriggerType::kForegrounded); -} - void FeedStream::ExecuteRefreshTask() { - if (!IsArticlesListVisible()) { - // While the check and cancel isn't strictly necessary, a long lived session - // could be issuing refreshes due to the background trigger while articles - // are not visible. - refresh_task_scheduler_->Cancel(); + // Schedule the next refresh attempt. If a new refresh schedule is returned + // through this refresh, it will be overwritten. + SetRequestSchedule(feed::prefs::GetRequestSchedule(profile_prefs_)); + + LoadStreamStatus do_not_attempt_reason = ShouldAttemptLoad(); + if (do_not_attempt_reason != LoadStreamStatus::kNoStatus) { + BackgroundRefreshComplete(LoadStreamTask::Result(do_not_attempt_reason)); return; } - MaybeTriggerRefresh(TriggerType::kFixedTimer); + + task_queue_.AddTask(std::make_unique<LoadStreamTask>( + LoadStreamTask::LoadType::kBackgroundRefresh, this, + base::BindOnce(&FeedStream::BackgroundRefreshComplete, + base::Unretained(this)))); +} + +void FeedStream::BackgroundRefreshComplete(LoadStreamTask::Result result) { + metrics_reporter_->OnBackgroundRefresh(result.final_status); + refresh_task_scheduler_->RefreshTaskComplete(); } void FeedStream::ClearAll() { - // TODO(harringtond): How should we handle in-progress tasks. - stream_event_observer_->OnClearAll(clock_->Now() - GetLastFetchTime()); + metrics_reporter_->OnClearAll(clock_->Now() - GetLastFetchTime()); - // TODO(harringtond): This should result in clearing feed data - // and _maybe_ triggering refresh with TriggerType::kNtpShown. - // That work should be embedded in a task. + task_queue_.AddTask(std::make_unique<ClearAllTask>(this)); } -void FeedStream::MaybeTriggerRefresh(TriggerType trigger, - bool clear_all_before_refresh) { - stream_event_observer_->OnMaybeTriggerRefresh(trigger, - clear_all_before_refresh); - // TODO(harringtond): Implement refresh (with LoadStreamTask). +void FeedStream::UploadAction( + feedwire::FeedAction action, + bool upload_now, + base::OnceCallback<void(UploadActionsTask::Result)> callback) { + task_queue_.AddTask(std::make_unique<UploadActionsTask>( + std::move(action), upload_now, this, std::move(callback))); } void FeedStream::LoadModel(std::unique_ptr<StreamModel> model) { @@ -438,11 +430,75 @@ void FeedStream::LoadModel(std::unique_ptr<StreamModel> model) { surface_updater_->SetModel(model_.get()); } +void FeedStream::SetRequestSchedule(RequestSchedule schedule) { + const base::Time now = clock_->Now(); + base::Time run_time = NextScheduledRequestTime(now, &schedule); + if (!run_time.is_null()) { + refresh_task_scheduler_->EnsureScheduled(run_time - now); + } else { + refresh_task_scheduler_->Cancel(); + } + feed::prefs::SetRequestSchedule(schedule, profile_prefs_); +} + void FeedStream::UnloadModel() { + // Note: This should only be called from a running Task, as some tasks assume + // the model remains loaded. if (!model_) return; surface_updater_->SetModel(nullptr); model_.reset(); } +void FeedStream::ReportOpenAction(const std::string& slice_id) { + int index = surface_updater_->GetSliceIndexFromSliceId(slice_id); + if (index >= 0) + metrics_reporter_->OpenAction(index); +} +void FeedStream::ReportOpenInNewTabAction(const std::string& slice_id) { + int index = surface_updater_->GetSliceIndexFromSliceId(slice_id); + if (index >= 0) + metrics_reporter_->OpenInNewTabAction(index); +} +void FeedStream::ReportOpenInNewIncognitoTabAction() { + metrics_reporter_->OpenInNewIncognitoTabAction(); +} +void FeedStream::ReportSliceViewed(SurfaceId surface_id, + const std::string& slice_id) { + int index = surface_updater_->GetSliceIndexFromSliceId(slice_id); + if (index >= 0) + metrics_reporter_->ContentSliceViewed(surface_id, index); +} + +void FeedStream::ReportSendFeedbackAction() { + metrics_reporter_->SendFeedbackAction(); +} +void FeedStream::ReportLearnMoreAction() { + metrics_reporter_->LearnMoreAction(); +} +void FeedStream::ReportDownloadAction() { + metrics_reporter_->DownloadAction(); +} +void FeedStream::ReportNavigationStarted() { + metrics_reporter_->NavigationStarted(); +} +void FeedStream::ReportPageLoaded() { + metrics_reporter_->PageLoaded(); +} +void FeedStream::ReportRemoveAction() { + metrics_reporter_->RemoveAction(); +} +void FeedStream::ReportNotInterestedInAction() { + metrics_reporter_->NotInterestedInAction(); +} +void FeedStream::ReportManageInterestsAction() { + metrics_reporter_->ManageInterestsAction(); +} +void FeedStream::ReportContextMenuOpened() { + metrics_reporter_->ContextMenuOpened(); +} +void FeedStream::ReportStreamScrolled(int distance_dp) { + metrics_reporter_->StreamScrolled(distance_dp); +} + } // namespace feed diff --git a/chromium/components/feed/core/v2/feed_stream.h b/chromium/components/feed/core/v2/feed_stream.h index 5c636822809..d92d889563e 100644 --- a/chromium/components/feed/core/v2/feed_stream.h +++ b/chromium/components/feed/core/v2/feed_stream.h @@ -6,19 +6,24 @@ #define COMPONENTS_FEED_CORE_V2_FEED_STREAM_H_ #include <memory> +#include <string> #include <vector> #include "base/memory/scoped_refptr.h" #include "base/observer_list.h" #include "base/sequenced_task_runner.h" #include "base/task_runner_util.h" +#include "base/version.h" #include "components/feed/core/common/enums.h" #include "components/feed/core/common/user_classifier.h" #include "components/feed/core/proto/v2/wire/response.pb.h" #include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/public/feed_stream_api.h" #include "components/feed/core/v2/request_throttler.h" +#include "components/feed/core/v2/scheduling.h" #include "components/feed/core/v2/stream_model.h" +#include "components/feed/core/v2/tasks/load_more_task.h" #include "components/feed/core/v2/tasks/load_stream_task.h" #include "components/offline_pages/task/task_queue.h" @@ -30,10 +35,12 @@ class TickClock; } // namespace base namespace feed { -class FeedStore; -class StreamModel; class FeedNetwork; +class FeedStore; +class MetricsReporter; class RefreshTaskScheduler; +class StreamModel; +class SurfaceUpdater; struct StreamModelUpdateRequest; // Implements FeedStreamApi. |FeedStream| additionally exposes functionality @@ -49,17 +56,8 @@ class FeedStream : public FeedStreamApi, virtual bool IsEulaAccepted() = 0; // Returns true if the device is offline. virtual bool IsOffline() = 0; - }; - - // An observer of stream events for testing and for tracking metrics. - // Concrete implementation should have no observable effects on the Feed. - class EventObserver { - public: - virtual void OnLoadStream(LoadStreamStatus load_from_store_status, - LoadStreamStatus final_status) = 0; - virtual void OnMaybeTriggerRefresh(TriggerType trigger, - bool clear_all_before_refresh) = 0; - virtual void OnClearAll(base::TimeDelta time_since_last_clear) = 0; + virtual DisplayMetrics GetDisplayMetrics() = 0; + virtual std::string GetLanguageTag() = 0; }; // Forwards to |feed::TranslateWireResponse()| by default. Can be overridden @@ -68,21 +66,38 @@ class FeedStream : public FeedStreamApi, public: WireResponseTranslator() = default; ~WireResponseTranslator() = default; - virtual std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( + virtual RefreshResponseData TranslateWireResponse( feedwire::Response response, - base::TimeDelta response_time, + StreamModelUpdateRequest::Source source, base::Time current_time); }; + class Metadata { + public: + explicit Metadata(FeedStore* store); + ~Metadata(); + + void Populate(feedstore::Metadata metadata); + + std::string GetConsistencyToken() const; + void SetConsistencyToken(std::string consistency_token); + + LocalActionId GetNextActionId(); + + private: + FeedStore* store_; + feedstore::Metadata metadata_; + }; + FeedStream(RefreshTaskScheduler* refresh_task_scheduler, - EventObserver* stream_event_observer, + MetricsReporter* metrics_reporter, Delegate* delegate, PrefService* profile_prefs, FeedNetwork* feed_network, FeedStore* feed_store, const base::Clock* clock, const base::TickClock* tick_clock, - scoped_refptr<base::SequencedTaskRunner> background_task_runner); + const ChromeInfo& chrome_info); ~FeedStream() override; FeedStream(const FeedStream&) = delete; @@ -97,18 +112,40 @@ class FeedStream : public FeedStreamApi, void DetachSurface(SurfaceInterface*) override; void SetArticlesListVisible(bool is_visible) override; bool IsArticlesListVisible() override; + void ExecuteRefreshTask() override; + void LoadMore(SurfaceId surface_id, + base::OnceCallback<void(bool)> callback) override; void ExecuteOperations( std::vector<feedstore::DataOperation> operations) override; EphemeralChangeId CreateEphemeralChange( std::vector<feedstore::DataOperation> operations) override; bool CommitEphemeralChange(EphemeralChangeId id) override; bool RejectEphemeralChange(EphemeralChangeId id) override; + DebugStreamData GetDebugStreamData() override; + void ForceRefreshForDebugging() override; + std::string DumpStateForDebugging() override; + + void ReportSliceViewed(SurfaceId surface_id, + const std::string& slice_id) override; + void ReportNavigationStarted() override; + void ReportPageLoaded() override; + void ReportOpenAction(const std::string& slice_id) override; + void ReportOpenInNewTabAction(const std::string& slice_id) override; + void ReportOpenInNewIncognitoTabAction() override; + void ReportSendFeedbackAction() override; + void ReportLearnMoreAction() override; + void ReportDownloadAction() override; + void ReportRemoveAction() override; + void ReportNotInterestedInAction() override; + void ReportManageInterestsAction() override; + void ReportContextMenuOpened() override; + void ReportStreamScrolled(int distance_dp) override; // offline_pages::TaskQueue::Delegate. void OnTaskQueueIsIdle() override; // StreamModel::StoreObserver. - void OnStoreChange(const StreamModel::StoreUpdate& update) override; + void OnStoreChange(StreamModel::StoreUpdate update) override; // Event indicators. These functions are called from an external source // to indicate an event. @@ -116,8 +153,8 @@ class FeedStream : public FeedStreamApi, // Called when Chrome's EULA has been accepted. This should happen when // Delegate::IsEulaAccepted() changes from false to true. void OnEulaAccepted(); - // Invoked when Chrome is foregrounded. - void OnEnterForeground(); + // Invoked when Chrome is backgrounded. + void OnEnterBackground(); // The user signed in to Chrome. void OnSignedIn(); // The user signed out of Chrome. @@ -126,68 +163,86 @@ class FeedStream : public FeedStreamApi, void OnHistoryDeleted(); // Chrome's cached data was cleared. void OnCacheDataCleared(); - // Invoked by RefreshTaskScheduler's scheduled task. - void ExecuteRefreshTask(); // State shared for the sake of implementing FeedStream. Typically these // functions are used by tasks. void LoadModel(std::unique_ptr<StreamModel> model); + void SetRequestSchedule(RequestSchedule schedule); + + // Store/upload an action and update the consistency token. |callback| is + // called with |true| if the consistency token was written to the store. + void UploadAction( + feedwire::FeedAction action, + bool upload_now, + base::OnceCallback<void(UploadActionsTask::Result)> callback); + FeedNetwork* GetNetwork() { return feed_network_; } FeedStore* GetStore() { return store_; } - - // Returns the computed UserClass for the active user. - UserClass GetUserClass(); + RequestThrottler* GetRequestThrottler() { return &request_throttler_; } + Metadata* GetMetadata() { return &metadata_; } // Returns the time of the last content fetch. base::Time GetLastFetchTime(); + bool HasSurfaceAttached() const; + + // Determines if we should attempt loading the stream or refreshing at all. + // Returns |LoadStreamStatus::kNoStatus| if loading may be attempted. + LoadStreamStatus ShouldAttemptLoad(bool model_loading = false); + // Determines if a FeedQuery request can be made. If successful, // returns |LoadStreamStatus::kNoStatus| and acquires throttler quota. // Otherwise returns the reason. - LoadStreamStatus ShouldMakeFeedQueryRequest(); + LoadStreamStatus ShouldMakeFeedQueryRequest(bool is_load_more = false); - // Loads |model|. Should be used for testing in place of typical model - // loading from network or storage. - void LoadModelForTesting(std::unique_ptr<StreamModel> model); - offline_pages::TaskQueue* GetTaskQueueForTesting(); - void UnloadModelForTesting() { UnloadModel(); } + // Unloads the model. Surfaces are not updated, and will remain frozen until a + // model load is requested. + void UnloadModel(); + + // Triggers a stream load. The load will be aborted if |ShouldAttemptLoad()| + // is not true. + void TriggerStreamLoad(); // Returns the model if it is loaded, or null otherwise. StreamModel* GetModel() { return model_.get(); } - const base::Clock* GetClock() { return clock_; } + const base::Clock* GetClock() const { return clock_; } + const base::TickClock* GetTickClock() const { return tick_clock_; } + RequestMetadata GetRequestMetadata() const; WireResponseTranslator* GetWireResponseTranslator() const { return wire_response_translator_; } + // Testing functionality. + offline_pages::TaskQueue* GetTaskQueueForTesting(); + // Loads |model|. Should be used for testing in place of typical model + // loading from network or storage. + void LoadModelForTesting(std::unique_ptr<StreamModel> model); void SetWireResponseTranslatorForTesting( WireResponseTranslator* wire_response_translator) { wire_response_translator_ = wire_response_translator; } - void SetIdleCallbackForTesting(base::RepeatingClosure idle_callback); - void SetUserClassifierForTesting( - std::unique_ptr<UserClassifier> user_classifier); private: - class SurfaceUpdater; class ModelStoreChangeMonitor; - void MaybeTriggerRefresh(TriggerType trigger, - bool clear_all_before_refresh = false); - void TriggerStreamLoad(); - void UnloadModel(); + // A single function task to delete stored feed data and force a refresh. + // To only be called from within a |Task|. + void ForceRefreshForDebuggingTask(); - void LoadStreamTaskComplete(LoadStreamTask::Result result); + void InitialStreamLoadComplete(LoadStreamTask::Result result); + void LoadMoreComplete(LoadMoreTask::Result result); + void BackgroundRefreshComplete(LoadStreamTask::Result result); void ClearAll(); // Unowned. RefreshTaskScheduler* refresh_task_scheduler_; - EventObserver* stream_event_observer_; + MetricsReporter* metrics_reporter_; Delegate* delegate_; PrefService* profile_prefs_; FeedNetwork* feed_network_; @@ -196,7 +251,7 @@ class FeedStream : public FeedStreamApi, const base::TickClock* tick_clock_; WireResponseTranslator* wire_response_translator_; - scoped_refptr<base::SequencedTaskRunner> background_task_runner_; + ChromeInfo chrome_info_; offline_pages::TaskQueue task_queue_; // Whether the model is being loaded. Used to prevent multiple simultaneous @@ -208,13 +263,11 @@ class FeedStream : public FeedStreamApi, // |UnloadModel()|. std::unique_ptr<StreamModel> model_; - // Set of (unowned) attached surfaces. - base::ObserverList<SurfaceInterface> surfaces_; - // Mutable state. - std::unique_ptr<UserClassifier> user_classifier_; RequestThrottler request_throttler_; base::TimeTicks suppress_refreshes_until_; + std::vector<base::OnceCallback<void(bool)>> load_more_complete_callbacks_; + Metadata metadata_; // To allow tests to wait on task queue idle. base::RepeatingClosure idle_callback_; diff --git a/chromium/components/feed/core/v2/feed_stream_unittest.cc b/chromium/components/feed/core/v2/feed_stream_unittest.cc index a514fd7cc63..d0325fbd69e 100644 --- a/chromium/components/feed/core/v2/feed_stream_unittest.cc +++ b/chromium/components/feed/core/v2/feed_stream_unittest.cc @@ -4,12 +4,19 @@ #include "components/feed/core/v2/feed_stream.h" +#include <map> #include <memory> +#include <sstream> #include <string> #include <utility> +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/logging.h" #include "base/optional.h" +#include "base/path_service.h" #include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" #include "base/test/bind_test_util.h" #include "base/test/scoped_run_loop_timeout.h" #include "base/test/simple_test_clock.h" @@ -19,14 +26,19 @@ #include "components/feed/core/common/pref_names.h" #include "components/feed/core/proto/v2/store.pb.h" #include "components/feed/core/proto/v2/ui.pb.h" +#include "components/feed/core/proto/v2/wire/action_request.pb.h" #include "components/feed/core/proto/v2/wire/request.pb.h" #include "components/feed/core/shared_prefs/pref_names.h" +#include "components/feed/core/v2/config.h" #include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/metrics_reporter.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/refresh_task_scheduler.h" #include "components/feed/core/v2/scheduling.h" #include "components/feed/core/v2/stream_model.h" -#include "components/feed/core/v2/stream_model_update_request.h" #include "components/feed/core/v2/tasks/load_stream_from_store_task.h" +#include "components/feed/core/v2/test/callback_receiver.h" +#include "components/feed/core/v2/test/proto_printer.h" #include "components/feed/core/v2/test/stream_builder.h" #include "components/leveldb_proto/public/proto_database_provider.h" #include "components/prefs/pref_registry_simple.h" @@ -42,8 +54,7 @@ std::unique_ptr<StreamModel> LoadModelFromStore(FeedStore* store) { result = std::move(task_result); }; LoadStreamFromStoreTask load_task( - store, /*clock=*/nullptr, - UserClass::kActiveSuggestionsConsumer, // Has no effect. + LoadStreamFromStoreTask::LoadType::kFullLoad, store, /*clock=*/nullptr, base::BindLambdaForTesting(complete)); // We want to load the data no matter how stale. load_task.IgnoreStalenessForTesting(); @@ -85,6 +96,22 @@ std::string ModelStateFor(FeedStore* store) { return "{Failed to load model from store}"; } +feedwire::FeedAction MakeFeedAction(int64_t id, size_t pad_size = 0) { + feedwire::FeedAction action; + action.mutable_content_id()->set_id(id); + action.mutable_content_id()->set_content_domain(std::string(pad_size, 'a')); + return action; +} + +std::vector<feedstore::StoredAction> ReadStoredActions(FeedStore* store) { + base::RunLoop run_loop; + CallbackReceiver<std::vector<feedstore::StoredAction>> cr(&run_loop); + store->ReadActions(cr.Bind()); + run_loop.Run(); + CHECK(cr.GetResult()); + return std::move(*cr.GetResult()); +} + // This is EXPECT_EQ, but also dumps the string values for ease of reading. #define EXPECT_STRINGS_EQUAL(WANT, GOT) \ { \ @@ -94,12 +121,42 @@ std::string ModelStateFor(FeedStore* store) { class TestSurface : public FeedStream::SurfaceInterface { public: + // Provide some helper functionality to attach/detach the surface. + // This way we can auto-detach in the destructor. + explicit TestSurface(FeedStream* stream = nullptr) { + if (stream) + Attach(stream); + } + + ~TestSurface() override { + if (stream_) + Detach(); + } + + void Attach(FeedStream* stream) { + EXPECT_FALSE(stream_); + stream_ = stream; + stream_->AttachSurface(this); + } + + void Detach() { + EXPECT_TRUE(stream_); + stream_->DetachSurface(this); + stream_ = nullptr; + } + // FeedStream::SurfaceInterface. void StreamUpdate(const feedui::StreamUpdate& stream_update) override { - if (!initial_state) + DVLOG(1) << "StreamUpdate: " << stream_update; + // Some special-case treatment for the loading spinner. We don't count it + // toward |initial_state|. + bool is_initial_loading_spinner = IsInitialLoadSpinnerUpdate(stream_update); + if (!initial_state && !is_initial_loading_spinner) { initial_state = stream_update; + } update = stream_update; - ++update_count_; + + described_updates_.push_back(CurrentState()); } // Test functions. @@ -107,53 +164,67 @@ class TestSurface : public FeedStream::SurfaceInterface { void Clear() { initial_state = base::nullopt; update = base::nullopt; - update_count_ = 0; + described_updates_.clear(); } - // Describe what is shown on the surface in a format that can be easily - // asserted against. - std::string Describe() { - if (!initial_state) - return "empty"; - - if (update->updated_slices().size() == 1 && - update->updated_slices()[0].has_slice() && - update->updated_slices()[0].slice().has_zero_state_slice()) { - return "zero-state"; - } - - std::stringstream ss; - ss << update->updated_slices().size() << " slices"; - // If there's more than one update, we want to know that. - if (update_count_ > 1) { - ss << " " << update_count_ << " updates"; - } - return ss.str(); + // Returns a description of the updates this surface received. Each update + // is separated by ' -> '. Returns only the updates since the last call. + std::string DescribeUpdates() { + std::string result = base::JoinString(described_updates_, " -> "); + described_updates_.clear(); + return result; } + // The initial state of the stream, if it was received. This is nullopt if + // only the loading spinner was seen. base::Optional<feedui::StreamUpdate> initial_state; + // The last stream update received. base::Optional<feedui::StreamUpdate> update; private: - int update_count_ = 0; -}; + std::string CurrentState() { + if (update && IsInitialLoadSpinnerUpdate(*update)) + return "loading"; -class TestUserClassifier : public UserClassifier { - public: - TestUserClassifier(PrefService* pref_service, const base::Clock* clock) - : UserClassifier(pref_service, clock) {} - // UserClassifier. - UserClass GetUserClass() const override { - return overridden_user_class_.value_or(UserClassifier::GetUserClass()); + if (!initial_state) + return "empty"; + + bool has_loading_spinner = false; + for (int i = 0; i < update->updated_slices().size(); ++i) { + const feedui::StreamUpdate_SliceUpdate& slice_update = + update->updated_slices(i); + if (slice_update.has_slice() && + slice_update.slice().has_zero_state_slice()) { + CHECK(update->updated_slices().size() == 1) + << "Zero state with other slices" << *update; + // Returns either "no-cards" or "cant-refresh". + return update->updated_slices()[0].slice().slice_id(); + } + if (slice_update.has_slice() && + slice_update.slice().has_loading_spinner_slice()) { + CHECK_EQ(i, update->updated_slices().size() - 1) + << "Loading spinner in an unexpected place" << *update; + has_loading_spinner = true; + } + } + std::stringstream ss; + if (has_loading_spinner) { + ss << update->updated_slices().size() - 1 << " slices +spinner"; + } else { + ss << update->updated_slices().size() << " slices"; + } + return ss.str(); } - // Test use. - void OverrideUserClass(UserClass user_class) { - overridden_user_class_ = user_class; + bool IsInitialLoadSpinnerUpdate(const feedui::StreamUpdate& update) { + return update.updated_slices().size() == 1 && + update.updated_slices()[0].has_slice() && + update.updated_slices()[0].slice().has_loading_spinner_slice(); } - private: - base::Optional<UserClass> overridden_user_class_; + // The stream if it was attached using the constructor. + FeedStream* stream_ = nullptr; + std::vector<std::string> described_updates_; }; class TestFeedNetwork : public FeedNetwork { @@ -168,79 +239,160 @@ class TestFeedNetwork : public FeedNetwork { // time we want to inject a translated response for ease of test-writing. query_request_sent = request; QueryRequestResult result; - result.status_code = 200; - result.response_body = std::make_unique<feedwire::Response>(); + result.response_info.fetch_duration = base::TimeDelta::FromMilliseconds(42); + if (injected_response_) { + result.response_body = std::make_unique<feedwire::Response>( + std::move(injected_response_.value())); + } else { + result.response_body = std::make_unique<feedwire::Response>(); + } base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, base::BindOnce(std::move(callback), std::move(result))); } void SendActionRequest( const feedwire::ActionRequest& request, base::OnceCallback<void(ActionRequestResult)> callback) override { - NOTIMPLEMENTED(); + action_request_sent = request; + ++action_request_call_count; + + ActionRequestResult result; + if (injected_action_result != base::nullopt) { + result = std::move(*injected_action_result); + } else { + auto response = std::make_unique<feedwire::Response>(); + response->mutable_feed_response() + ->mutable_feed_response() + ->mutable_consistency_token() + ->set_token(consistency_token); + + result.response_body = std::move(response); + } + + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), std::move(result))); } void CancelRequests() override { NOTIMPLEMENTED(); } + void InjectRealResponse() { + base::FilePath response_file_path; + CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &response_file_path)); + response_file_path = response_file_path.AppendASCII( + "components/test/data/feed/response.binarypb"); + std::string response_data; + CHECK(base::ReadFileToString(response_file_path, &response_data)); + + feedwire::Response response; + CHECK(response.ParseFromString(response_data)); + + injected_response_ = response; + } + base::Optional<feedwire::Request> query_request_sent; int send_query_call_count = 0; + + void InjectActionRequestResult(ActionRequestResult result) { + injected_action_result = std::move(result); + } + void InjectEmptyActionRequestResult() { + ActionRequestResult result; + result.response_body = nullptr; + InjectActionRequestResult(std::move(result)); + } + base::Optional<feedwire::ActionRequest> action_request_sent; + int action_request_call_count = 0; + std::string consistency_token; + + private: + base::Optional<feedwire::Response> injected_response_; + base::Optional<ActionRequestResult> injected_action_result; }; // Forwards to |FeedStream::WireResponseTranslator| unless a response is // injected. class TestWireResponseTranslator : public FeedStream::WireResponseTranslator { public: - std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( + RefreshResponseData TranslateWireResponse( feedwire::Response response, - base::TimeDelta response_time, + StreamModelUpdateRequest::Source source, base::Time current_time) override { if (injected_response_) { - return std::move(injected_response_); + if (injected_response_->model_update_request) + injected_response_->model_update_request->source = source; + RefreshResponseData result = std::move(*injected_response_); + injected_response_.reset(); + return result; } return FeedStream::WireResponseTranslator::TranslateWireResponse( - std::move(response), response_time, current_time); + std::move(response), source, current_time); } void InjectResponse(std::unique_ptr<StreamModelUpdateRequest> response) { - injected_response_ = std::move(response); + injected_response_ = RefreshResponseData(); + injected_response_->model_update_request = std::move(response); + } + void InjectResponse(RefreshResponseData response_data) { + injected_response_ = std::move(response_data); } bool InjectedResponseConsumed() const { return !injected_response_; } private: - std::unique_ptr<StreamModelUpdateRequest> injected_response_; + base::Optional<RefreshResponseData> injected_response_; }; class FakeRefreshTaskScheduler : public RefreshTaskScheduler { public: // RefreshTaskScheduler implementation. - void EnsureScheduled(base::TimeDelta period) override { - scheduled_period = period; + void EnsureScheduled(base::TimeDelta run_time) override { + scheduled_run_time = run_time; } void Cancel() override { canceled = true; } void RefreshTaskComplete() override { refresh_task_complete = true; } - base::Optional<base::TimeDelta> scheduled_period; + void Clear() { + scheduled_run_time.reset(); + canceled = false; + refresh_task_complete = false; + } + base::Optional<base::TimeDelta> scheduled_run_time; bool canceled = false; bool refresh_task_complete = false; }; -class TestEventObserver : public FeedStream::EventObserver { +class TestMetricsReporter : public MetricsReporter { public: - // FeedStreamUnittest::StreamEventObserver. + explicit TestMetricsReporter(const base::TickClock* clock) + : MetricsReporter(clock) {} + + // MetricsReporter. + void ContentSliceViewed(SurfaceId surface_id, int index_in_stream) override { + slice_viewed_index = index_in_stream; + MetricsReporter::ContentSliceViewed(surface_id, index_in_stream); + } void OnLoadStream(LoadStreamStatus load_from_store_status, LoadStreamStatus final_status) override { load_stream_status = final_status; LOG(INFO) << "OnLoadStream: " << final_status << " (store status: " << load_from_store_status << ")"; + MetricsReporter::OnLoadStream(load_from_store_status, final_status); + } + void OnLoadMore(LoadStreamStatus final_status) override { + load_more_status = final_status; + MetricsReporter::OnLoadMore(final_status); } - void OnMaybeTriggerRefresh(TriggerType trigger, - bool clear_all_before_refresh) override { - refresh_trigger_type = trigger; + void OnBackgroundRefresh(LoadStreamStatus final_status) override { + background_refresh_status = final_status; + MetricsReporter::OnBackgroundRefresh(final_status); } void OnClearAll(base::TimeDelta time_since_last_clear) override { this->time_since_last_clear = time_since_last_clear; + MetricsReporter::OnClearAll(time_since_last_clear); } // Test access. + base::Optional<int> slice_viewed_index; base::Optional<LoadStreamStatus> load_stream_status; + base::Optional<LoadStreamStatus> load_more_status; + base::Optional<LoadStreamStatus> background_refresh_status; base::Optional<base::TimeDelta> time_since_last_clear; base::Optional<TriggerType> refresh_trigger_type; }; @@ -251,21 +403,8 @@ class FeedStreamTest : public testing::Test, public FeedStream::Delegate { feed::prefs::RegisterFeedSharedProfilePrefs(profile_prefs_.registry()); feed::RegisterProfilePrefs(profile_prefs_.registry()); CHECK_EQ(kTestTimeEpoch, task_environment_.GetMockClock()->Now()); - stream_ = std::make_unique<FeedStream>( - &refresh_scheduler_, &event_observer_, this, &profile_prefs_, &network_, - store_.get(), task_environment_.GetMockClock(), - task_environment_.GetMockTickClock(), - task_environment_.GetMainThreadTaskRunner()); - - // Set the user classifier. - auto user_classifier = std::make_unique<TestUserClassifier>( - &profile_prefs_, task_environment_.GetMockClock()); - user_classifier_ = user_classifier.get(); - stream_->SetUserClassifierForTesting(std::move(user_classifier)); - - WaitForIdleTaskQueue(); // Wait for any initialization. - stream_->SetWireResponseTranslatorForTesting(&response_translator_); + CreateStream(); } void TearDown() override { @@ -280,9 +419,31 @@ class FeedStreamTest : public testing::Test, public FeedStream::Delegate { // FeedStream::Delegate. bool IsEulaAccepted() override { return is_eula_accepted_; } bool IsOffline() override { return is_offline_; } + DisplayMetrics GetDisplayMetrics() override { + DisplayMetrics result; + result.density = 200; + result.height_pixels = 800; + result.width_pixels = 350; + return result; + } + std::string GetLanguageTag() override { return "en-US"; } // For tests. + // Replace stream_. + void CreateStream() { + ChromeInfo chrome_info; + chrome_info.channel = version_info::Channel::STABLE; + chrome_info.version = base::Version({99, 1, 9911, 2}); + stream_ = std::make_unique<FeedStream>( + &refresh_scheduler_, &metrics_reporter_, this, &profile_prefs_, + &network_, store_.get(), task_environment_.GetMockClock(), + task_environment_.GetMockTickClock(), chrome_info); + + WaitForIdleTaskQueue(); // Wait for any initialization. + stream_->SetWireResponseTranslatorForTesting(&response_translator_); + } + bool IsTaskQueueIdle() const { return !stream_->GetTaskQueueForTesting()->HasPendingTasks() && !stream_->GetTaskQueueForTesting()->HasRunningTask(); @@ -300,14 +461,41 @@ class FeedStreamTest : public testing::Test, public FeedStream::Delegate { void UnloadModel() { WaitForIdleTaskQueue(); - stream_->UnloadModelForTesting(); + stream_->UnloadModel(); + } + + // Dumps the state of |FeedStore| to a string for debugging. + std::string DumpStoreState() { + base::RunLoop run_loop; + std::unique_ptr<std::vector<feedstore::Record>> records; + auto callback = + [&](bool, std::unique_ptr<std::vector<feedstore::Record>> result) { + records = std::move(result); + run_loop.Quit(); + }; + store_->GetDatabaseForTesting()->LoadEntries( + base::BindLambdaForTesting(callback)); + + run_loop.Run(); + std::stringstream ss; + for (const feedstore::Record& record : *records) { + ss << record << '\n'; + } + return ss.str(); + } + + void UploadActions(std::vector<feedwire::FeedAction> actions) { + size_t actions_remaining = actions.size(); + for (feedwire::FeedAction& action : actions) { + stream_->UploadAction(action, (--actions_remaining) == 0ul, + base::DoNothing()); + } } protected: base::test::TaskEnvironment task_environment_{ base::test::TaskEnvironment::TimeSource::MOCK_TIME}; - TestUserClassifier* user_classifier_; - TestEventObserver event_observer_; + TestMetricsReporter metrics_reporter_{task_environment_.GetMockTickClock()}; TestingPrefServiceSimple profile_prefs_; TestFeedNetwork network_; TestWireResponseTranslator response_translator_; @@ -335,28 +523,55 @@ TEST_F(FeedStreamTest, SetArticlesListVisible) { EXPECT_TRUE(stream_->IsArticlesListVisible()); } -TEST_F(FeedStreamTest, RefreshIsScheduledOnInitialize) { +TEST_F(FeedStreamTest, DoNotRefreshIfArticlesListIsHidden) { + stream_->SetArticlesListVisible(false); stream_->InitializeScheduling(); - EXPECT_TRUE(refresh_scheduler_.scheduled_period); + EXPECT_TRUE(refresh_scheduler_.canceled); + + stream_->ExecuteRefreshTask(); + EXPECT_TRUE(refresh_scheduler_.refresh_task_complete); + EXPECT_EQ(LoadStreamStatus::kLoadNotAllowedArticlesListHidden, + metrics_reporter_.background_refresh_status); } -TEST_F(FeedStreamTest, ScheduledRefreshTriggersRefresh) { - stream_->InitializeScheduling(); +TEST_F(FeedStreamTest, BackgroundRefreshSuccess) { + // Trigger a background refresh. + response_translator_.InjectResponse(MakeTypicalInitialModelState()); stream_->ExecuteRefreshTask(); + WaitForIdleTaskQueue(); - EXPECT_EQ(TriggerType::kFixedTimer, event_observer_.refresh_trigger_type); - // TODO(harringtond): Once we actually perform the refresh, make sure - // RefreshTaskComplete() is called. - // EXPECT_TRUE(refresh_scheduler_.refresh_task_complete); + // Verify the refresh happened and that we can load a stream without the + // network. + ASSERT_TRUE(refresh_scheduler_.refresh_task_complete); + EXPECT_EQ(LoadStreamStatus::kLoadedFromNetwork, + metrics_reporter_.background_refresh_status); + EXPECT_TRUE(response_translator_.InjectedResponseConsumed()); + EXPECT_FALSE(stream_->GetModel()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + EXPECT_EQ("loading -> 2 slices", surface.DescribeUpdates()); } -TEST_F(FeedStreamTest, DoNotRefreshIfArticlesListIsHidden) { - stream_->SetArticlesListVisible(false); - stream_->InitializeScheduling(); +TEST_F(FeedStreamTest, BackgroundRefreshNotAttemptedWhenModelIsLoading) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); stream_->ExecuteRefreshTask(); + WaitForIdleTaskQueue(); - EXPECT_TRUE(refresh_scheduler_.canceled); - EXPECT_FALSE(event_observer_.refresh_trigger_type); + EXPECT_EQ(metrics_reporter_.background_refresh_status, + LoadStreamStatus::kModelAlreadyLoaded); +} + +TEST_F(FeedStreamTest, BackgroundRefreshNotAttemptedAfterModelIsLoaded) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + stream_->ExecuteRefreshTask(); + WaitForIdleTaskQueue(); + + EXPECT_EQ(metrics_reporter_.background_refresh_status, + LoadStreamStatus::kModelAlreadyLoaded); } TEST_F(FeedStreamTest, SurfaceReceivesInitialContent) { @@ -365,8 +580,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesInitialContent) { model->Update(MakeTypicalInitialModelState()); stream_->LoadModelForTesting(std::move(model)); } - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); ASSERT_TRUE(surface.initial_state); const feedui::StreamUpdate& initial_state = surface.initial_state.value(); ASSERT_EQ(2, initial_state.updated_slices().size()); @@ -386,8 +600,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesInitialContent) { } TEST_F(FeedStreamTest, SurfaceReceivesInitialContentLoadedAfterAttach) { - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); ASSERT_FALSE(surface.initial_state); { auto model = std::make_unique<StreamModel>(); @@ -395,7 +608,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesInitialContentLoadedAfterAttach) { stream_->LoadModelForTesting(std::move(model)); } - ASSERT_EQ("2 slices", surface.Describe()); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); const feedui::StreamUpdate& initial_state = surface.initial_state.value(); EXPECT_NE("", initial_state.updated_slices(0).slice().slice_id()); @@ -419,8 +632,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesUpdatedContent) { model->ExecuteOperations(MakeTypicalStreamOperations()); stream_->LoadModelForTesting(std::move(model)); } - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); // Remove #1, add #2. stream_->ExecuteOperations({ MakeOperation(MakeRemove(MakeClusterId(1))), @@ -432,7 +644,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesUpdatedContent) { const feedui::StreamUpdate& initial_state = surface.initial_state.value(); const feedui::StreamUpdate& update = surface.update.value(); - ASSERT_EQ("2 slices 2 updates", surface.Describe()); + ASSERT_EQ("2 slices -> 2 slices", surface.DescribeUpdates()); // First slice is just an ID that matches the old 1st slice ID. EXPECT_EQ(initial_state.updated_slices(0).slice().slice_id(), update.updated_slices(0).slice_id()); @@ -448,8 +660,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesSecondUpdatedContent) { model->ExecuteOperations(MakeTypicalStreamOperations()); stream_->LoadModelForTesting(std::move(model)); } - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); // Add #2. stream_->ExecuteOperations({ MakeOperation(MakeCluster(2, MakeRootId())), @@ -466,7 +677,7 @@ TEST_F(FeedStreamTest, SurfaceReceivesSecondUpdatedContent) { // The last update should have only one new piece of content. // This verifies the current content set is tracked properly. - ASSERT_EQ("4 slices 3 updates", surface.Describe()); + ASSERT_EQ("2 slices -> 3 slices -> 4 slices", surface.DescribeUpdates()); ASSERT_EQ(4, surface.update->updated_slices().size()); EXPECT_FALSE(surface.update->updated_slices(0).has_slice()); @@ -478,16 +689,29 @@ TEST_F(FeedStreamTest, SurfaceReceivesSecondUpdatedContent) { .xsurface_frame()); } +TEST_F(FeedStreamTest, RemoveAllContentResultsInZeroState) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + // Remove both pieces of content. + stream_->ExecuteOperations({ + MakeOperation(MakeRemove(MakeClusterId(0))), + MakeOperation(MakeRemove(MakeClusterId(1))), + }); + + ASSERT_EQ("loading -> 2 slices -> no-cards", surface.DescribeUpdates()); +} + TEST_F(FeedStreamTest, DetachSurface) { { auto model = std::make_unique<StreamModel>(); model->ExecuteOperations(MakeTypicalStreamOperations()); stream_->LoadModelForTesting(std::move(model)); } - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); EXPECT_TRUE(surface.initial_state); - stream_->DetachSurface(&surface); + surface.Detach(); surface.Clear(); // Arbitrary stream change. Surface should not see the update. @@ -498,40 +722,109 @@ TEST_F(FeedStreamTest, DetachSurface) { } TEST_F(FeedStreamTest, LoadFromNetwork) { + stream_->GetMetadata()->SetConsistencyToken("token"); + // Store is empty, so we should fallback to a network request. response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); - EXPECT_TRUE(network_.query_request_sent); + ASSERT_TRUE(network_.query_request_sent); + EXPECT_EQ( + "token", + network_.query_request_sent->feed_request().consistency_token().token()); EXPECT_TRUE(response_translator_.InjectedResponseConsumed()); - EXPECT_EQ("2 slices", surface.Describe()); + + EXPECT_EQ("loading -> 2 slices", surface.DescribeUpdates()); // Verify the model is filled correctly. EXPECT_STRINGS_EQUAL(ModelStateFor(MakeTypicalInitialModelState()), stream_->GetModel()->DumpStateForTesting()); // Verify the data was written to the store. - EXPECT_STRINGS_EQUAL(ModelStateFor(store_.get()), - ModelStateFor(MakeTypicalInitialModelState())); + EXPECT_STRINGS_EQUAL(ModelStateFor(MakeTypicalInitialModelState()), + ModelStateFor(store_.get())); +} + +TEST_F(FeedStreamTest, ForceRefreshForDebugging) { + // First do a normal load via network that will fail. + is_offline_ = true; + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + // Next, force a refresh that results in a successful load. + is_offline_ = false; + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + stream_->ForceRefreshForDebugging(); + + WaitForIdleTaskQueue(); + EXPECT_EQ("loading -> cant-refresh -> loading -> 2 slices", + surface.DescribeUpdates()); +} + +TEST_F(FeedStreamTest, RefreshScheduleFlow) { + // Inject a typical network response, with a server-defined request schedule. + { + RequestSchedule schedule; + schedule.anchor_time = kTestTimeEpoch; + schedule.refresh_offsets = {base::TimeDelta::FromSeconds(12), + base::TimeDelta::FromSeconds(48)}; + RefreshResponseData response_data; + response_data.model_update_request = MakeTypicalInitialModelState(); + response_data.request_schedule = schedule; + + response_translator_.InjectResponse(std::move(response_data)); + } + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + // Verify the first refresh was scheduled. + EXPECT_EQ(base::TimeDelta::FromSeconds(12), + refresh_scheduler_.scheduled_run_time); + + // Simulate executing the background task. + refresh_scheduler_.Clear(); + task_environment_.AdvanceClock(base::TimeDelta::FromSeconds(12)); + stream_->ExecuteRefreshTask(); + WaitForIdleTaskQueue(); + + // Verify |RefreshTaskComplete()| was called and next refresh was scheduled. + EXPECT_TRUE(refresh_scheduler_.refresh_task_complete); + EXPECT_EQ(base::TimeDelta::FromSeconds(48 - 12), + refresh_scheduler_.scheduled_run_time); + + // Simulate executing the background task again. + refresh_scheduler_.Clear(); + task_environment_.AdvanceClock(base::TimeDelta::FromSeconds(48 - 12)); + stream_->ExecuteRefreshTask(); + WaitForIdleTaskQueue(); + + // Verify |RefreshTaskComplete()| was called and next refresh was scheduled. + EXPECT_TRUE(refresh_scheduler_.refresh_task_complete); + ASSERT_TRUE(refresh_scheduler_.scheduled_run_time); + EXPECT_EQ(GetFeedConfig().default_background_refresh_interval, + *refresh_scheduler_.scheduled_run_time); } TEST_F(FeedStreamTest, LoadFromNetworkBecauseStoreIsStale) { // Fill the store with stream data that is just barely stale, and verify we // fetch new data over the network. - user_classifier_->OverrideUserClass(UserClass::kActiveSuggestionsConsumer); - store_->SaveFullStream(MakeTypicalInitialModelState( - - kTestTimeEpoch - base::TimeDelta::FromHours(12) - - base::TimeDelta::FromMinutes(1)), - base::DoNothing()); + store_->OverwriteStream( + MakeTypicalInitialModelState( + /*first_cluster_id=*/0, kTestTimeEpoch - + GetFeedConfig().stale_content_threshold - + base::TimeDelta::FromMinutes(1)), + base::DoNothing()); + stream_->GetMetadata()->SetConsistencyToken("token-1"); // Store is stale, so we should fallback to a network request. response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); - EXPECT_TRUE(network_.query_request_sent); + ASSERT_TRUE(network_.query_request_sent); + // The stored continutation token should be sent. + EXPECT_EQ( + "token-1", + network_.query_request_sent->feed_request().consistency_token().token()); EXPECT_TRUE(response_translator_.InjectedResponseConsumed()); ASSERT_TRUE(surface.initial_state); } @@ -540,48 +833,60 @@ TEST_F(FeedStreamTest, LoadFromNetworkFailsDueToProtoTranslation) { // No data in the store, so we should fetch from the network. // The network will respond with an empty response, which should fail proto // translation. - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); EXPECT_EQ(LoadStreamStatus::kProtoTranslationFailed, - event_observer_.load_stream_status); + metrics_reporter_.load_stream_status); } TEST_F(FeedStreamTest, DoNotLoadFromNetworkWhenOffline) { is_offline_ = true; response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); EXPECT_EQ(LoadStreamStatus::kCannotLoadFromNetworkOffline, - event_observer_.load_stream_status); - EXPECT_EQ("zero-state", surface.Describe()); + metrics_reporter_.load_stream_status); + EXPECT_EQ("loading -> cant-refresh", surface.DescribeUpdates()); } TEST_F(FeedStreamTest, DoNotLoadStreamWhenArticleListIsHidden) { stream_->SetArticlesListVisible(false); response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); EXPECT_EQ(LoadStreamStatus::kLoadNotAllowedArticlesListHidden, - event_observer_.load_stream_status); - EXPECT_EQ("zero-state", surface.Describe()); + metrics_reporter_.load_stream_status); + EXPECT_EQ("no-cards", surface.DescribeUpdates()); } TEST_F(FeedStreamTest, DoNotLoadStreamWhenEulaIsNotAccepted) { is_eula_accepted_ = false; response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); EXPECT_EQ(LoadStreamStatus::kLoadNotAllowedEulaNotAccepted, - event_observer_.load_stream_status); - EXPECT_EQ("zero-state", surface.Describe()); + metrics_reporter_.load_stream_status); + EXPECT_EQ("no-cards", surface.DescribeUpdates()); +} + +TEST_F(FeedStreamTest, LoadStreamAfterEulaIsAccepted) { + // Connect a surface before the EULA is accepted. + is_eula_accepted_ = false; + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("no-cards", surface.DescribeUpdates()); + + // Accept EULA, our surface should receive data. + is_eula_accepted_ = true; + stream_->OnEulaAccepted(); + WaitForIdleTaskQueue(); + + EXPECT_EQ("loading -> 2 slices", surface.DescribeUpdates()); } TEST_F(FeedStreamTest, DoNotLoadFromNetworkAfterHistoryIsDeleted) { @@ -589,21 +894,21 @@ TEST_F(FeedStreamTest, DoNotLoadFromNetworkAfterHistoryIsDeleted) { task_environment_.FastForwardBy(kSuppressRefreshDuration - base::TimeDelta::FromSeconds(1)); response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); - EXPECT_EQ("zero-state", surface.Describe()); + EXPECT_EQ("loading -> no-cards", surface.DescribeUpdates()); EXPECT_EQ(LoadStreamStatus::kCannotLoadFromNetworkSupressedForHistoryDelete, - event_observer_.load_stream_status); + metrics_reporter_.load_stream_status); - stream_->DetachSurface(&surface); + surface.Detach(); task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(2)); - stream_->AttachSurface(&surface); + surface.Clear(); + surface.Attach(stream_.get()); WaitForIdleTaskQueue(); - EXPECT_EQ("2 slices 2 updates", surface.Describe()); + EXPECT_EQ("loading -> 2 slices", surface.DescribeUpdates()); } TEST_F(FeedStreamTest, ShouldMakeFeedQueryRequestConsumesQuota) { @@ -618,54 +923,59 @@ TEST_F(FeedStreamTest, ShouldMakeFeedQueryRequestConsumesQuota) { TEST_F(FeedStreamTest, LoadStreamFromStore) { // Fill the store with stream data that is just barely fresh, and verify it // loads. - user_classifier_->OverrideUserClass(UserClass::kActiveSuggestionsConsumer); - store_->SaveFullStream(MakeTypicalInitialModelState( - kTestTimeEpoch - base::TimeDelta::FromHours(12) + - base::TimeDelta::FromMinutes(1)), - base::DoNothing()); - TestSurface surface; - stream_->AttachSurface(&surface); + store_->OverwriteStream(MakeTypicalInitialModelState( + /*first_cluster_id=*/0, + kTestTimeEpoch - base::TimeDelta::FromHours(12) + + base::TimeDelta::FromMinutes(1)), + base::DoNothing()); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); - ASSERT_EQ("2 slices", surface.Describe()); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); EXPECT_FALSE(network_.query_request_sent); // Verify the model is filled correctly. EXPECT_STRINGS_EQUAL(ModelStateFor(MakeTypicalInitialModelState()), stream_->GetModel()->DumpStateForTesting()); } +TEST_F(FeedStreamTest, LoadingSpinnerIsSentInitially) { + store_->OverwriteStream(MakeTypicalInitialModelState(), base::DoNothing()); + TestSurface surface(stream_.get()); + + ASSERT_EQ("loading", surface.DescribeUpdates()); +} + TEST_F(FeedStreamTest, DetachSurfaceWhileLoadingModel) { response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - stream_->AttachSurface(&surface); - stream_->DetachSurface(&surface); + TestSurface surface(stream_.get()); + surface.Detach(); WaitForIdleTaskQueue(); - EXPECT_EQ("empty", surface.Describe()); + EXPECT_EQ("loading", surface.DescribeUpdates()); EXPECT_TRUE(network_.query_request_sent); } TEST_F(FeedStreamTest, AttachMultipleSurfacesLoadsModelOnce) { response_translator_.InjectResponse(MakeTypicalInitialModelState()); - TestSurface surface; - TestSurface other_surface; - stream_->AttachSurface(&surface); - stream_->AttachSurface(&other_surface); + TestSurface surface(stream_.get()); + TestSurface other_surface(stream_.get()); WaitForIdleTaskQueue(); ASSERT_EQ(1, network_.send_query_call_count); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + ASSERT_EQ("loading -> 2 slices", other_surface.DescribeUpdates()); - // After load, another surface doesn't trigger any tasks. - TestSurface later_surface; - stream_->AttachSurface(&later_surface); + // After load, another surface doesn't trigger any tasks, + // and immediately has content. + TestSurface later_surface(stream_.get()); + ASSERT_EQ("2 slices", later_surface.DescribeUpdates()); EXPECT_TRUE(IsTaskQueueIdle()); } TEST_F(FeedStreamTest, ModelChangesAreSavedToStorage) { - store_->SaveFullStream(MakeTypicalInitialModelState(), base::DoNothing()); - TestSurface surface; - stream_->AttachSurface(&surface); + store_->OverwriteStream(MakeTypicalInitialModelState(), base::DoNothing()); + TestSurface surface(stream_.get()); WaitForIdleTaskQueue(); ASSERT_TRUE(surface.initial_state); @@ -687,10 +997,10 @@ TEST_F(FeedStreamTest, ModelChangesAreSavedToStorage) { // Unload and reload the model from the store, and verify we can still apply // operations correctly. - stream_->DetachSurface(&surface); + surface.Detach(); surface.Clear(); UnloadModel(); - stream_->AttachSurface(&surface); + surface.Attach(stream_.get()); WaitForIdleTaskQueue(); ASSERT_TRUE(surface.initial_state); @@ -709,5 +1019,411 @@ TEST_F(FeedStreamTest, ModelChangesAreSavedToStorage) { ModelStateFor(store_.get())); } +TEST_F(FeedStreamTest, ReportSliceViewedIdentifiesCorrectIndex) { + store_->OverwriteStream(MakeTypicalInitialModelState(), base::DoNothing()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + stream_->ReportSliceViewed( + surface.GetSurfaceId(), + surface.initial_state->updated_slices(1).slice().slice_id()); + EXPECT_EQ(1, metrics_reporter_.slice_viewed_index); +} + +TEST_F(FeedStreamTest, LoadMoreAppendsContent) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + // Load page 2. + response_translator_.InjectResponse(MakeTypicalNextPageState(2)); + CallbackReceiver<bool> callback; + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + WaitForIdleTaskQueue(); + ASSERT_EQ(base::Optional<bool>(true), callback.GetResult()); + EXPECT_EQ("2 slices +spinner -> 4 slices", surface.DescribeUpdates()); + // Load page 3. + response_translator_.InjectResponse(MakeTypicalNextPageState(3)); + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + + WaitForIdleTaskQueue(); + ASSERT_EQ(base::Optional<bool>(true), callback.GetResult()); + EXPECT_EQ("4 slices +spinner -> 6 slices", surface.DescribeUpdates()); +} + +TEST_F(FeedStreamTest, LoadMorePersistsData) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + + // Load page 2. + response_translator_.InjectResponse(MakeTypicalNextPageState(2)); + CallbackReceiver<bool> callback; + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + + WaitForIdleTaskQueue(); + ASSERT_EQ(base::Optional<bool>(true), callback.GetResult()); + + // Verify stored state is equivalent to in-memory model. + EXPECT_STRINGS_EQUAL(stream_->GetModel()->DumpStateForTesting(), + ModelStateFor(store_.get())); +} + +TEST_F(FeedStreamTest, LoadMorePersistAndLoadMore) { + // Verify we can persist a LoadMore, and then do another LoadMore after + // reloading state. + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + + // Load page 2. + response_translator_.InjectResponse(MakeTypicalNextPageState(2)); + CallbackReceiver<bool> callback; + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + WaitForIdleTaskQueue(); + ASSERT_EQ(base::Optional<bool>(true), callback.GetResult()); + + surface.Detach(); + UnloadModel(); + + // Load page 3. + surface.Attach(stream_.get()); + response_translator_.InjectResponse(MakeTypicalNextPageState(3)); + WaitForIdleTaskQueue(); + callback.Clear(); + surface.Clear(); + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + WaitForIdleTaskQueue(); + + ASSERT_EQ(base::Optional<bool>(true), callback.GetResult()); + ASSERT_EQ("4 slices +spinner -> 6 slices", surface.DescribeUpdates()); + // Verify stored state is equivalent to in-memory model. + EXPECT_STRINGS_EQUAL(stream_->GetModel()->DumpStateForTesting(), + ModelStateFor(store_.get())); +} + +TEST_F(FeedStreamTest, LoadMoreSendsTokens) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + + stream_->GetMetadata()->SetConsistencyToken("token-1"); + response_translator_.InjectResponse(MakeTypicalNextPageState(2)); + CallbackReceiver<bool> callback; + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + + WaitForIdleTaskQueue(); + ASSERT_EQ("2 slices +spinner -> 4 slices", surface.DescribeUpdates()); + + EXPECT_EQ( + "token-1", + network_.query_request_sent->feed_request().consistency_token().token()); + EXPECT_EQ("page-2", network_.query_request_sent->feed_request() + .feed_query() + .next_page_token() + .next_page_token() + .next_page_token()); + + stream_->GetMetadata()->SetConsistencyToken("token-2"); + response_translator_.InjectResponse(MakeTypicalNextPageState(3)); + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + + WaitForIdleTaskQueue(); + ASSERT_EQ("4 slices +spinner -> 6 slices", surface.DescribeUpdates()); + + EXPECT_EQ( + "token-2", + network_.query_request_sent->feed_request().consistency_token().token()); + EXPECT_EQ("page-3", network_.query_request_sent->feed_request() + .feed_query() + .next_page_token() + .next_page_token() + .next_page_token()); +} + +TEST_F(FeedStreamTest, LoadMoreFail) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + + // Don't inject another response, which results in a proto translation + // failure. + CallbackReceiver<bool> callback; + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + WaitForIdleTaskQueue(); + + EXPECT_EQ(base::Optional<bool>(false), callback.GetResult()); + EXPECT_EQ("2 slices +spinner -> 2 slices", surface.DescribeUpdates()); +} + +TEST_F(FeedStreamTest, LoadMoreWithClearAllInResponse) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + ASSERT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + + // Use a different initial state (which includes a CLEAR_ALL). + response_translator_.InjectResponse(MakeTypicalInitialModelState(5)); + CallbackReceiver<bool> callback; + stream_->LoadMore(surface.GetSurfaceId(), callback.Bind()); + + WaitForIdleTaskQueue(); + ASSERT_EQ(base::Optional<bool>(true), callback.GetResult()); + + // Verify stored state is equivalent to in-memory model. + EXPECT_STRINGS_EQUAL(stream_->GetModel()->DumpStateForTesting(), + ModelStateFor(store_.get())); + + // Verify the new state has been pushed to |surface|. + ASSERT_EQ("2 slices +spinner -> 2 slices", surface.DescribeUpdates()); + + const feedui::StreamUpdate& initial_state = surface.update.value(); + ASSERT_EQ(2, initial_state.updated_slices().size()); + EXPECT_NE("", initial_state.updated_slices(0).slice().slice_id()); + EXPECT_EQ("f:5", initial_state.updated_slices(0) + .slice() + .xsurface_slice() + .xsurface_frame()); + EXPECT_NE("", initial_state.updated_slices(1).slice().slice_id()); + EXPECT_EQ("f:6", initial_state.updated_slices(1) + .slice() + .xsurface_slice() + .xsurface_frame()); +} + +TEST_F(FeedStreamTest, LoadMoreBeforeLoad) { + CallbackReceiver<bool> callback; + stream_->LoadMore(SurfaceId(), callback.Bind()); + + EXPECT_EQ(base::Optional<bool>(false), callback.GetResult()); +} + +TEST_F(FeedStreamTest, ReadNetworkResponse) { + network_.InjectRealResponse(); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + ASSERT_EQ("loading -> 10 slices", surface.DescribeUpdates()); +} + +TEST_F(FeedStreamTest, ClearAllAfterLoadResultsInRefresh) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + stream_->OnCacheDataCleared(); // triggers ClearAll(). + + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + WaitForIdleTaskQueue(); + + EXPECT_EQ("loading -> 2 slices -> loading -> 2 slices", + surface.DescribeUpdates()); +} + +TEST_F(FeedStreamTest, ClearAllWithNoSurfacesAttachedDoesNotReload) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + surface.Detach(); + + stream_->OnCacheDataCleared(); // triggers ClearAll(). + WaitForIdleTaskQueue(); + + EXPECT_EQ("loading -> 2 slices", surface.DescribeUpdates()); + // Also check that the storage is cleared. + EXPECT_EQ("", DumpStoreState()); +} + +TEST_F(FeedStreamTest, StorePendingAction) { + stream_->UploadAction(MakeFeedAction(42ul), false, base::DoNothing()); + WaitForIdleTaskQueue(); + + std::vector<feedstore::StoredAction> result = + ReadStoredActions(stream_->GetStore()); + ASSERT_EQ(1ul, result.size()); + EXPECT_EQ(42ul, result[0].action().content_id().id()); +} + +TEST_F(FeedStreamTest, StorePendingActionAndUploadNow) { + network_.consistency_token = "token-11"; + + CallbackReceiver<UploadActionsTask::Result> cr; + stream_->UploadAction(MakeFeedAction(42ul), true, cr.Bind()); + WaitForIdleTaskQueue(); + + ASSERT_TRUE(cr.GetResult()); + EXPECT_EQ(1ul, cr.GetResult()->upload_attempt_count); + EXPECT_EQ(UploadActionsStatus::kUpdatedConsistencyToken, + cr.GetResult()->status); + + std::vector<feedstore::StoredAction> result = + ReadStoredActions(stream_->GetStore()); + ASSERT_EQ(0ul, result.size()); +} + +TEST_F(FeedStreamTest, LoadStreamFromNetworkUploadsActions) { + stream_->UploadAction(MakeFeedAction(99ul), false, base::DoNothing()); + WaitForIdleTaskQueue(); + + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + EXPECT_EQ(1, network_.action_request_call_count); + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); + + // Uploaded action should have been erased from store. + stream_->UploadAction(MakeFeedAction(100ul), true, base::DoNothing()); + WaitForIdleTaskQueue(); + EXPECT_EQ(2, network_.action_request_call_count); + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); +} + +TEST_F(FeedStreamTest, LoadMoreUploadsActions) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface(stream_.get()); + WaitForIdleTaskQueue(); + + stream_->UploadAction(MakeFeedAction(99ul), false, base::DoNothing()); + WaitForIdleTaskQueue(); + + network_.consistency_token = "token-12"; + + stream_->LoadMore(surface.GetSurfaceId(), base::DoNothing()); + WaitForIdleTaskQueue(); + + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); + EXPECT_EQ("token-12", stream_->GetMetadata()->GetConsistencyToken()); + + // Uploaded action should have been erased from the store. + network_.action_request_sent.reset(); + stream_->UploadAction(MakeFeedAction(100ul), true, base::DoNothing()); + WaitForIdleTaskQueue(); + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); + EXPECT_EQ(100ul, network_.action_request_sent->feed_action_request() + .feed_action(0) + .content_id() + .id()); +} + +TEST_F(FeedStreamTest, UploadActionsOneBatch) { + UploadActions( + {MakeFeedAction(97ul), MakeFeedAction(98ul), MakeFeedAction(99ul)}); + WaitForIdleTaskQueue(); + + EXPECT_EQ(1, network_.action_request_call_count); + EXPECT_EQ( + 3, + network_.action_request_sent->feed_action_request().feed_action_size()); + + stream_->UploadAction(MakeFeedAction(99ul), true, base::DoNothing()); + WaitForIdleTaskQueue(); + EXPECT_EQ(2, network_.action_request_call_count); + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); +} + +TEST_F(FeedStreamTest, UploadActionsMultipleBatches) { + UploadActions({ + // Batch 1: One really big action. + MakeFeedAction(100ul, /*pad_size=*/20001ul), + + // Batch 2 + MakeFeedAction(101ul, 10000ul), + MakeFeedAction(102ul, 9000ul), + + // Batch 3. Trigger upload. + MakeFeedAction(103ul, 2000ul), + }); + WaitForIdleTaskQueue(); + + EXPECT_EQ(3, network_.action_request_call_count); + + stream_->UploadAction(MakeFeedAction(99ul), true, base::DoNothing()); + WaitForIdleTaskQueue(); + EXPECT_EQ(4, network_.action_request_call_count); + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); +} + +TEST_F(FeedStreamTest, UploadActionsSkipsStaleActionsByTimestamp) { + stream_->UploadAction(MakeFeedAction(2ul), false, base::DoNothing()); + WaitForIdleTaskQueue(); + task_environment_.FastForwardBy(base::TimeDelta::FromHours(25)); + + // Trigger upload + CallbackReceiver<UploadActionsTask::Result> cr; + stream_->UploadAction(MakeFeedAction(3ul), true, cr.Bind()); + WaitForIdleTaskQueue(); + + // Just one action should have been uploaded. + EXPECT_EQ(1, network_.action_request_call_count); + EXPECT_EQ( + 1, + network_.action_request_sent->feed_action_request().feed_action_size()); + EXPECT_EQ(3ul, network_.action_request_sent->feed_action_request() + .feed_action(0) + .content_id() + .id()); + + ASSERT_TRUE(cr.GetResult()); + EXPECT_EQ(1ul, cr.GetResult()->upload_attempt_count); + EXPECT_EQ(1ul, cr.GetResult()->stale_count); +} + +TEST_F(FeedStreamTest, UploadActionsErasesStaleActionsByAttempts) { + // Three failed uploads, plus one more to cause the first action to be erased. + network_.InjectEmptyActionRequestResult(); + stream_->UploadAction(MakeFeedAction(0ul), true, base::DoNothing()); + network_.InjectEmptyActionRequestResult(); + stream_->UploadAction(MakeFeedAction(1ul), true, base::DoNothing()); + network_.InjectEmptyActionRequestResult(); + stream_->UploadAction(MakeFeedAction(2ul), true, base::DoNothing()); + + CallbackReceiver<UploadActionsTask::Result> cr; + stream_->UploadAction(MakeFeedAction(3ul), true, cr.Bind()); + WaitForIdleTaskQueue(); + + // Four requests, three pending actions in the last request. + EXPECT_EQ(4, network_.action_request_call_count); + EXPECT_EQ( + 3, + network_.action_request_sent->feed_action_request().feed_action_size()); + + // Action 0 should have been erased. + ASSERT_TRUE(cr.GetResult()); + EXPECT_EQ(3ul, cr.GetResult()->upload_attempt_count); + EXPECT_EQ(1ul, cr.GetResult()->stale_count); +} + +TEST_F(FeedStreamTest, MetadataLoadedWhenDatabaseInitialized) { + ASSERT_TRUE(stream_->GetMetadata()); + + // Set the token and increment next action ID. + stream_->GetMetadata()->SetConsistencyToken("token"); + EXPECT_EQ(0, stream_->GetMetadata()->GetNextActionId().GetUnsafeValue()); + + // Creating a stream should load metadata. + CreateStream(); + + ASSERT_TRUE(stream_->GetMetadata()); + EXPECT_EQ("token", stream_->GetMetadata()->GetConsistencyToken()); + EXPECT_EQ(1, stream_->GetMetadata()->GetNextActionId().GetUnsafeValue()); +} + } // namespace } // namespace feed diff --git a/chromium/components/feed/core/v2/metrics_reporter.cc b/chromium/components/feed/core/v2/metrics_reporter.cc new file mode 100644 index 00000000000..975db91a92c --- /dev/null +++ b/chromium/components/feed/core/v2/metrics_reporter.cc @@ -0,0 +1,347 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +#include "components/feed/core/v2/metrics_reporter.h" + +#include <cmath> + +#include "base/metrics/histogram_functions.h" +#include "base/metrics/user_metrics.h" +#include "base/threading/thread_task_runner_handle.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" + +namespace feed { +namespace { +using feed::internal::FeedEngagementType; +using feed::internal::FeedUserActionType; +const int kMaxSuggestionsTotal = 50; +// Maximum time to wait before declaring a load operation failed. +// For both ContentSuggestions.Feed.UserJourney.OpenFeed +// and ContentSuggestions.Feed.UserJourney.GetMore. +constexpr base::TimeDelta kLoadTimeout = base::TimeDelta::FromSeconds(15); +// Maximum time to wait before declaring opening a card a failure. +// For ContentSuggestions.Feed.UserJourney.OpenCard. +constexpr base::TimeDelta kOpenTimeout = base::TimeDelta::FromSeconds(20); + +void ReportEngagementTypeHistogram(FeedEngagementType engagement_type) { + base::UmaHistogramEnumeration("ContentSuggestions.Feed.EngagementType", + engagement_type); +} + +void ReportContentSuggestionsOpened(int index_in_stream) { + base::UmaHistogramExactLinear("NewTabPage.ContentSuggestions.Opened", + index_in_stream, kMaxSuggestionsTotal); +} + +void ReportUserActionHistogram(FeedUserActionType action_type) { + base::UmaHistogramEnumeration("ContentSuggestions.Feed.UserAction", + action_type); +} + +} // namespace + +MetricsReporter::MetricsReporter(const base::TickClock* clock) + : clock_(clock) {} + +MetricsReporter::~MetricsReporter() = default; + +void MetricsReporter::OnEnterBackground() { + FinalizeMetrics(); +} + +// Engagement Tracking. + +void MetricsReporter::RecordInteraction() { + RecordEngagement(/*scroll_distance_dp=*/0, /*interacted=*/true); + ReportEngagementTypeHistogram(FeedEngagementType::kFeedInteracted); +} + +void MetricsReporter::RecordEngagement(int scroll_distance_dp, + bool interacted) { + scroll_distance_dp = std::abs(scroll_distance_dp); + // Determine if this interaction is part of a new 'session'. + auto now = clock_->NowTicks(); + const base::TimeDelta kVisitTimeout = base::TimeDelta::FromMinutes(5); + if (now - visit_start_time_ > kVisitTimeout) { + engaged_reported_ = false; + engaged_simple_reported_ = false; + } + // Reset the last active time for session measurement. + visit_start_time_ = now; + + // Report the user as engaged-simple if they have scrolled any amount or + // interacted with the card, and we have not already reported it for this + // chrome run. + if (!engaged_simple_reported_ && (scroll_distance_dp > 0 || interacted)) { + ReportEngagementTypeHistogram(FeedEngagementType::kFeedEngagedSimple); + engaged_simple_reported_ = true; + } + + // Report the user as engaged if they have scrolled more than the threshold or + // interacted with the card, and we have not already reported it this chrome + // run. + const int kMinScrollThresholdDp = 160; // 1 inch. + if (!engaged_reported_ && + (scroll_distance_dp > kMinScrollThresholdDp || interacted)) { + ReportEngagementTypeHistogram(FeedEngagementType::kFeedEngaged); + engaged_reported_ = true; + } +} + +void MetricsReporter::StreamScrolled(int distance_dp) { + RecordEngagement(distance_dp, /*interacted=*/false); + + if (!scrolled_reported_) { + ReportEngagementTypeHistogram(FeedEngagementType::kFeedScrolled); + scrolled_reported_ = true; + } +} + +void MetricsReporter::ContentSliceViewed(SurfaceId surface_id, + int index_in_stream) { + base::UmaHistogramExactLinear("NewTabPage.ContentSuggestions.Shown", + index_in_stream, kMaxSuggestionsTotal); + + ReportOpenFeedIfNeeded(surface_id, true); +} + +void MetricsReporter::OpenAction(int index_in_stream) { + CardOpenBegin(); + ReportUserActionHistogram(FeedUserActionType::kTappedOnCard); + base::RecordAction( + base::UserMetricsAction("ContentSuggestions.Feed.CardAction.Open")); + ReportContentSuggestionsOpened(index_in_stream); + RecordInteraction(); +} + +void MetricsReporter::OpenInNewTabAction(int index_in_stream) { + CardOpenBegin(); + ReportUserActionHistogram(FeedUserActionType::kTappedOpenInNewTab); + base::RecordAction(base::UserMetricsAction( + "ContentSuggestions.Feed.CardAction.OpenInNewTab")); + ReportContentSuggestionsOpened(index_in_stream); + RecordInteraction(); +} + +void MetricsReporter::OpenInNewIncognitoTabAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedOpenInNewIncognitoTab); + base::RecordAction(base::UserMetricsAction( + "ContentSuggestions.Feed.CardAction.OpenInNewIncognitoTab")); + RecordInteraction(); +} + +void MetricsReporter::SendFeedbackAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedSendFeedback); + base::RecordAction(base::UserMetricsAction( + "ContentSuggestions.Feed.CardAction.SendFeedback")); + RecordInteraction(); +} + +void MetricsReporter::DownloadAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedDownload); + base::RecordAction( + base::UserMetricsAction("ContentSuggestions.Feed.CardAction.Download")); + RecordInteraction(); +} + +void MetricsReporter::LearnMoreAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedLearnMore); + base::RecordAction( + base::UserMetricsAction("ContentSuggestions.Feed.CardAction.LearnMore")); + RecordInteraction(); +} + +void MetricsReporter::NavigationStarted() { + // TODO(harringtond): Use this or remove it. +} + +void MetricsReporter::PageLoaded() { + ReportCardOpenEndIfNeeded(true); +} + +void MetricsReporter::RemoveAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedHideStory); + base::RecordAction( + base::UserMetricsAction("ContentSuggestions.Feed.CardAction.HideStory")); + RecordInteraction(); +} + +void MetricsReporter::NotInterestedInAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedNotInterestedIn); + base::RecordAction(base::UserMetricsAction( + "ContentSuggestions.Feed.CardAction.NotInterestedIn")); + RecordInteraction(); +} + +void MetricsReporter::ManageInterestsAction() { + ReportUserActionHistogram(FeedUserActionType::kTappedManageInterests); + base::RecordAction(base::UserMetricsAction( + "ContentSuggestions.Feed.CardAction.ManageInterests")); + RecordInteraction(); +} + +void MetricsReporter::ContextMenuOpened() { + ReportUserActionHistogram(FeedUserActionType::kOpenedContextMenu); + base::RecordAction(base::UserMetricsAction( + "ContentSuggestions.Feed.CardAction.ContextMenu")); +} + +void MetricsReporter::SurfaceOpened(SurfaceId surface_id) { + surfaces_waiting_for_content_.emplace(surface_id, clock_->NowTicks()); + ReportUserActionHistogram(FeedUserActionType::kOpenedFeedSurface); + base::ThreadTaskRunnerHandle::Get()->PostDelayedTask( + FROM_HERE, + base::BindOnce(&MetricsReporter::ReportOpenFeedIfNeeded, GetWeakPtr(), + surface_id, false), + kLoadTimeout); +} + +void MetricsReporter::SurfaceClosed(SurfaceId surface_id) { + ReportOpenFeedIfNeeded(surface_id, false); + ReportGetMoreIfNeeded(surface_id, false); +} + +void MetricsReporter::FinalizeMetrics() { + ReportCardOpenEndIfNeeded(false); + for (auto iter = surfaces_waiting_for_content_.begin(); + iter != surfaces_waiting_for_content_.end();) { + ReportOpenFeedIfNeeded((iter++)->first, false); + } + for (auto iter = surfaces_waiting_for_more_content_.begin(); + iter != surfaces_waiting_for_more_content_.end();) { + ReportGetMoreIfNeeded((iter++)->first, false); + } +} + +void MetricsReporter::ReportOpenFeedIfNeeded(SurfaceId surface_id, + bool success) { + auto iter = surfaces_waiting_for_content_.find(surface_id); + if (iter == surfaces_waiting_for_content_.end()) + return; + base::TimeDelta latency = clock_->NowTicks() - iter->second; + surfaces_waiting_for_content_.erase(iter); + if (success) { + base::UmaHistogramCustomTimes( + "ContentSuggestions.Feed.UserJourney.OpenFeed.SuccessDuration", latency, + base::TimeDelta::FromMilliseconds(50), kLoadTimeout, 50); + } else { + base::UmaHistogramCustomTimes( + "ContentSuggestions.Feed.UserJourney.OpenFeed.FailureDuration", latency, + base::TimeDelta::FromMilliseconds(50), kLoadTimeout, 50); + } +} + +void MetricsReporter::ReportGetMoreIfNeeded(SurfaceId surface_id, + bool success) { + auto iter = surfaces_waiting_for_more_content_.find(surface_id); + if (iter == surfaces_waiting_for_more_content_.end()) + return; + base::TimeDelta latency = clock_->NowTicks() - iter->second; + surfaces_waiting_for_more_content_.erase(iter); + if (success) { + base::UmaHistogramCustomTimes( + "ContentSuggestions.Feed.UserJourney.GetMore.SuccessDuration", latency, + base::TimeDelta::FromMilliseconds(50), kLoadTimeout, 50); + } else { + base::UmaHistogramCustomTimes( + "ContentSuggestions.Feed.UserJourney.GetMore.FailureDuration", latency, + base::TimeDelta::FromMilliseconds(50), kLoadTimeout, 50); + } +} + +void MetricsReporter::CardOpenBegin() { + ReportCardOpenEndIfNeeded(false); + pending_open_ = clock_->NowTicks(); + base::ThreadTaskRunnerHandle::Get()->PostDelayedTask( + FROM_HERE, + base::BindOnce(&MetricsReporter::CardOpenTimeout, GetWeakPtr(), + *pending_open_), + kOpenTimeout); +} + +void MetricsReporter::CardOpenTimeout(base::TimeTicks start_ticks) { + if (pending_open_ && start_ticks == *pending_open_) + ReportCardOpenEndIfNeeded(false); +} + +void MetricsReporter::ReportCardOpenEndIfNeeded(bool success) { + if (!pending_open_) + return; + base::TimeDelta latency = clock_->NowTicks() - *pending_open_; + pending_open_.reset(); + if (success) { + base::UmaHistogramCustomTimes( + "ContentSuggestions.Feed.UserJourney.OpenCard.SuccessDuration", latency, + base::TimeDelta::FromMilliseconds(100), kOpenTimeout, 50); + } else { + base::UmaHistogramBoolean( + "ContentSuggestions.Feed.UserJourney.OpenCard.Failure", true); + } +} + +void MetricsReporter::NetworkRequestComplete(NetworkRequestType type, + int http_status_code) { + switch (type) { + case NetworkRequestType::kFeedQuery: + base::UmaHistogramSparse( + "ContentSuggestions.Feed.Network.ResponseStatus.FeedQuery", + http_status_code); + return; + case NetworkRequestType::kUploadActions: + base::UmaHistogramSparse( + "ContentSuggestions.Feed.Network.ResponseStatus.UploadActions", + http_status_code); + return; + } +} + +void MetricsReporter::OnLoadStream(LoadStreamStatus load_from_store_status, + LoadStreamStatus final_status) { + DVLOG(1) << "OnLoadStream load_from_store_status=" << load_from_store_status + << " final_status=" << final_status; + base::UmaHistogramEnumeration( + "ContentSuggestions.Feed.LoadStreamStatus.Initial", final_status); + if (load_from_store_status != LoadStreamStatus::kNoStatus) { + base::UmaHistogramEnumeration( + "ContentSuggestions.Feed.LoadStreamStatus.InitialFromStore", + load_from_store_status); + } +} + +void MetricsReporter::OnBackgroundRefresh(LoadStreamStatus final_status) { + base::UmaHistogramEnumeration( + "ContentSuggestions.Feed.LoadStreamStatus.BackgroundRefresh", + final_status); +} + +void MetricsReporter::OnLoadMoreBegin(SurfaceId surface_id) { + ReportGetMoreIfNeeded(surface_id, false); + surfaces_waiting_for_more_content_.emplace(surface_id, clock_->NowTicks()); + + base::ThreadTaskRunnerHandle::Get()->PostDelayedTask( + FROM_HERE, + base::BindOnce(&MetricsReporter::ReportGetMoreIfNeeded, GetWeakPtr(), + surface_id, false), + kLoadTimeout); +} + +void MetricsReporter::OnLoadMore(LoadStreamStatus status) { + DVLOG(1) << "OnLoadMore status=" << status; + base::UmaHistogramEnumeration( + "ContentSuggestions.Feed.LoadStreamStatus.LoadMore", status); +} + +void MetricsReporter::SurfaceReceivedContent(SurfaceId surface_id) { + ReportGetMoreIfNeeded(surface_id, true); +} + +void MetricsReporter::OnClearAll(base::TimeDelta time_since_last_clear) { + base::UmaHistogramCustomTimes( + "ContentSuggestions.Feed.Scheduler.TimeSinceLastFetchOnClear", + time_since_last_clear, base::TimeDelta::FromSeconds(1), + base::TimeDelta::FromDays(7), + /*bucket_count=*/50); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/metrics_reporter.h b/chromium/components/feed/core/v2/metrics_reporter.h new file mode 100644 index 00000000000..d680f1e3916 --- /dev/null +++ b/chromium/components/feed/core/v2/metrics_reporter.h @@ -0,0 +1,139 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_METRICS_REPORTER_H_ +#define COMPONENTS_FEED_CORE_V2_METRICS_REPORTER_H_ + +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/time/time.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_stream.h" + +namespace base { +class TickClock; +} // namespace base +namespace feed { +namespace internal { +// This enum is used for a UMA histogram. Keep in sync with FeedEngagementType +// in enums.xml. +enum class FeedEngagementType { + kFeedEngaged = 0, + kFeedEngagedSimple = 1, + kFeedInteracted = 2, + kFeedScrolled = 3, + kMaxValue = FeedEngagementType::kFeedScrolled, +}; + +// This enum must match FeedUserActionType in enums.xml. +// Note that most of these have a corresponding UserMetricsAction reported here. +// Exceptions are described below. +enum class FeedUserActionType { + kTappedOnCard = 0, + // This is not an actual user action, so there will be no UserMetricsAction + // reported for this. + kShownCard = 1, + kTappedSendFeedback = 2, + kTappedLearnMore = 3, + kTappedHideStory = 4, + kTappedNotInterestedIn = 5, + kTappedManageInterests = 6, + kTappedDownload = 7, + kTappedOpenInNewTab = 8, + kOpenedContextMenu = 9, + // User action not reported here. See Suggestions.SurfaceVisible. + kOpenedFeedSurface = 10, + kTappedOpenInNewIncognitoTab = 11, + kMaxValue = kTappedOpenInNewIncognitoTab, +}; + +} // namespace internal + +// Reports UMA metrics for feed. +// Note this is inherited only for testing. +class MetricsReporter { + public: + explicit MetricsReporter(const base::TickClock* clock); + virtual ~MetricsReporter(); + MetricsReporter(const MetricsReporter&) = delete; + MetricsReporter& operator=(const MetricsReporter&) = delete; + + // User interactions. See |FeedStreamApi| for definitions. + + virtual void ContentSliceViewed(SurfaceId surface_id, int index_in_stream); + void OpenAction(int index_in_stream); + void OpenInNewTabAction(int index_in_stream); + void OpenInNewIncognitoTabAction(); + void SendFeedbackAction(); + void LearnMoreAction(); + void DownloadAction(); + void NavigationStarted(); + void PageLoaded(); + void RemoveAction(); + void NotInterestedInAction(); + void ManageInterestsAction(); + void ContextMenuOpened(); + // Indicates the user scrolled the feed by |distance_dp| and then stopped + // scrolling. + void StreamScrolled(int distance_dp); + + // Called when the Feed surface is opened and closed. + void SurfaceOpened(SurfaceId surface_id); + void SurfaceClosed(SurfaceId surface_id); + + // Network metrics. + + static void NetworkRequestComplete(NetworkRequestType type, + int http_status_code); + + // Stream events. + + virtual void OnLoadStream(LoadStreamStatus load_from_store_status, + LoadStreamStatus final_status); + virtual void OnBackgroundRefresh(LoadStreamStatus final_status); + void OnLoadMoreBegin(SurfaceId surface_id); + virtual void OnLoadMore(LoadStreamStatus final_status); + virtual void OnClearAll(base::TimeDelta time_since_last_clear); + // Called each time the surface receives new content. + void SurfaceReceivedContent(SurfaceId surface_id); + // Called when Chrome is entering the background. + void OnEnterBackground(); + + private: + base::WeakPtr<MetricsReporter> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + void CardOpenBegin(); + void CardOpenTimeout(base::TimeTicks start_ticks); + void ReportCardOpenEndIfNeeded(bool success); + void RecordEngagement(int scroll_distance_dp, bool interacted); + void RecordInteraction(); + void ReportOpenFeedIfNeeded(SurfaceId surface_id, bool success); + void ReportGetMoreIfNeeded(SurfaceId surface_id, bool success); + void FinalizeMetrics(); + + const base::TickClock* clock_; + + base::TimeTicks visit_start_time_; + bool engaged_simple_reported_ = false; + bool engaged_reported_ = false; + bool scrolled_reported_ = false; + // The time a surface was opened, for surfaces still waiting for content. + std::map<SurfaceId, base::TimeTicks> surfaces_waiting_for_content_; + // The time a surface requested more content, for surfaces still waiting for + // more content. + std::map<SurfaceId, base::TimeTicks> surfaces_waiting_for_more_content_; + + // Tracking ContentSuggestions.Feed.UserJourney.OpenCard.*: + // We assume at most one card is opened at a time. The time the card was + // tapped is stored here. Upon timeout, another open attempt, or + // |ChromeStopping()|, the open is considered failed. Otherwise, if the + // loading the page succeeds, the open is considered successful. + base::Optional<base::TimeTicks> pending_open_; + + base::WeakPtrFactory<MetricsReporter> weak_ptr_factory_{this}; +}; +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_METRICS_REPORTER_H_ diff --git a/chromium/components/feed/core/v2/metrics_reporter_unittest.cc b/chromium/components/feed/core/v2/metrics_reporter_unittest.cc new file mode 100644 index 00000000000..7300a16b8fc --- /dev/null +++ b/chromium/components/feed/core/v2/metrics_reporter_unittest.cc @@ -0,0 +1,417 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/metrics_reporter.h" + +#include <map> + +#include "base/test/metrics/histogram_tester.h" +#include "base/test/metrics/user_action_tester.h" +#include "base/test/task_environment.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +using feed::internal::FeedEngagementType; +using feed::internal::FeedUserActionType; +constexpr SurfaceId kSurfaceId = SurfaceId(5); +const base::TimeDelta kEpsilon = base::TimeDelta::FromMilliseconds(1); + +class MetricsReporterTest : public testing::Test { + protected: + std::map<FeedEngagementType, int> ReportedEngagementType() { + std::map<FeedEngagementType, int> result; + for (const auto& bucket : + histogram_.GetAllSamples("ContentSuggestions.Feed.EngagementType")) { + result[static_cast<FeedEngagementType>(bucket.min)] += bucket.count; + } + return result; + } + + protected: + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + MetricsReporter reporter_{task_environment_.GetMockTickClock()}; + base::HistogramTester histogram_; + base::UserActionTester user_actions_; +}; + +TEST_F(MetricsReporterTest, SliceViewedReportsSuggestionShown) { + reporter_.ContentSliceViewed(kSurfaceId, 5); + histogram_.ExpectUniqueSample("NewTabPage.ContentSuggestions.Shown", 5, 1); +} + +TEST_F(MetricsReporterTest, ScrollingSmall) { + reporter_.StreamScrolled(100); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedScrolled, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, ScrollingCanTriggerEngaged) { + reporter_.StreamScrolled(161); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedScrolled, 1}, + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, OpeningContentIsInteracting) { + reporter_.OpenAction(5); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, RemovingContentIsInteracting) { + reporter_.RemoveAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, NotInterestedInIsInteracting) { + reporter_.NotInterestedInAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, ManageInterestsInIsInteracting) { + reporter_.ManageInterestsAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, VisitsCanLastMoreThanFiveMinutes) { + reporter_.StreamScrolled(1); + task_environment_.FastForwardBy(base::TimeDelta::FromMinutes(5) - kEpsilon); + reporter_.OpenAction(0); + task_environment_.FastForwardBy(base::TimeDelta::FromMinutes(5) - kEpsilon); + reporter_.StreamScrolled(1); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedScrolled, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, NewVisitAfterInactivity) { + reporter_.OpenAction(0); + reporter_.StreamScrolled(1); + task_environment_.FastForwardBy(base::TimeDelta::FromMinutes(5) + kEpsilon); + reporter_.OpenAction(0); + reporter_.StreamScrolled(1); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 2}, + {FeedEngagementType::kFeedInteracted, 2}, + {FeedEngagementType::kFeedEngagedSimple, 2}, + {FeedEngagementType::kFeedScrolled, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); +} + +TEST_F(MetricsReporterTest, ReportsLoadStreamStatus) { + reporter_.OnLoadStream(LoadStreamStatus::kDataInStoreIsStale, + LoadStreamStatus::kLoadedFromNetwork); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.LoadStreamStatus.Initial", + LoadStreamStatus::kLoadedFromNetwork, 1); + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.LoadStreamStatus.InitialFromStore", + LoadStreamStatus::kDataInStoreIsStale, 1); +} + +TEST_F(MetricsReporterTest, ReportsLoadStreamStatusIgnoresNoStatusFromStore) { + reporter_.OnLoadStream(LoadStreamStatus::kNoStatus, + LoadStreamStatus::kLoadedFromNetwork); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.LoadStreamStatus.Initial", + LoadStreamStatus::kLoadedFromNetwork, 1); + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.LoadStreamStatus.InitialFromStore", 0); +} + +TEST_F(MetricsReporterTest, ReportsLoadMoreStatus) { + reporter_.OnLoadMore(LoadStreamStatus::kLoadedFromNetwork); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.LoadStreamStatus.LoadMore", + LoadStreamStatus::kLoadedFromNetwork, 1); +} + +TEST_F(MetricsReporterTest, ReportsBackgroundRefreshStatus) { + reporter_.OnBackgroundRefresh(LoadStreamStatus::kLoadedFromNetwork); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.LoadStreamStatus.BackgroundRefresh", + LoadStreamStatus::kLoadedFromNetwork, 1); +} + +TEST_F(MetricsReporterTest, OpenAction) { + reporter_.OpenAction(5); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.Open")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedOnCard, 1); + histogram_.ExpectUniqueSample("NewTabPage.ContentSuggestions.Opened", 5, 1); +} + +TEST_F(MetricsReporterTest, OpenInNewTabAction) { + reporter_.OpenInNewTabAction(5); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.OpenInNewTab")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedOpenInNewTab, 1); + histogram_.ExpectUniqueSample("NewTabPage.ContentSuggestions.Opened", 5, 1); +} + +TEST_F(MetricsReporterTest, OpenInNewIncognitoTabAction) { + reporter_.OpenInNewIncognitoTabAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.OpenInNewIncognitoTab")); + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedOpenInNewIncognitoTab, 1); + histogram_.ExpectTotalCount("NewTabPage.ContentSuggestions.Opened", 0); +} + +TEST_F(MetricsReporterTest, SendFeedbackAction) { + reporter_.SendFeedbackAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.SendFeedback")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedSendFeedback, 1); +} + +TEST_F(MetricsReporterTest, DownloadAction) { + reporter_.DownloadAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.Download")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedDownload, 1); +} + +TEST_F(MetricsReporterTest, LearnMoreAction) { + reporter_.LearnMoreAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.LearnMore")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedLearnMore, 1); +} + +TEST_F(MetricsReporterTest, RemoveAction) { + reporter_.RemoveAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.HideStory")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedHideStory, 1); +} + +TEST_F(MetricsReporterTest, NotInterestedInAction) { + reporter_.NotInterestedInAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.NotInterestedIn")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedNotInterestedIn, 1); +} + +TEST_F(MetricsReporterTest, ManageInterestsAction) { + reporter_.ManageInterestsAction(); + + std::map<FeedEngagementType, int> want({ + {FeedEngagementType::kFeedEngaged, 1}, + {FeedEngagementType::kFeedInteracted, 1}, + {FeedEngagementType::kFeedEngagedSimple, 1}, + }); + EXPECT_EQ(want, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.ManageInterests")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kTappedManageInterests, 1); +} + +TEST_F(MetricsReporterTest, ContextMenuOpened) { + reporter_.ContextMenuOpened(); + + std::map<FeedEngagementType, int> want_empty; + EXPECT_EQ(want_empty, ReportedEngagementType()); + EXPECT_EQ(1, user_actions_.GetActionCount( + "ContentSuggestions.Feed.CardAction.ContextMenu")); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kOpenedContextMenu, 1); +} + +TEST_F(MetricsReporterTest, SurfaceOpened) { + reporter_.SurfaceOpened(kSurfaceId); + + std::map<FeedEngagementType, int> want_empty; + EXPECT_EQ(want_empty, ReportedEngagementType()); + histogram_.ExpectUniqueSample("ContentSuggestions.Feed.UserAction", + FeedUserActionType::kOpenedFeedSurface, 1); +} + +TEST_F(MetricsReporterTest, OpenFeedSuccessDuration) { + reporter_.SurfaceOpened(kSurfaceId); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(9)); + reporter_.ContentSliceViewed(kSurfaceId, 0); + + histogram_.ExpectUniqueTimeSample( + "ContentSuggestions.Feed.UserJourney.OpenFeed.SuccessDuration", + base::TimeDelta::FromSeconds(9), 1); +} + +TEST_F(MetricsReporterTest, OpenFeedLoadTimeout) { + reporter_.SurfaceOpened(kSurfaceId); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(16)); + + histogram_.ExpectUniqueTimeSample( + "ContentSuggestions.Feed.UserJourney.OpenFeed.FailureDuration", + base::TimeDelta::FromSeconds(15), 1); + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.UserJourney.OpenFeed.SuccessDuration", 0); +} + +TEST_F(MetricsReporterTest, OpenFeedCloseBeforeLoad) { + reporter_.SurfaceOpened(kSurfaceId); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(14)); + reporter_.SurfaceClosed(kSurfaceId); + + histogram_.ExpectUniqueTimeSample( + "ContentSuggestions.Feed.UserJourney.OpenFeed.FailureDuration", + base::TimeDelta::FromSeconds(14), 1); + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.UserJourney.OpenFeed.SuccessDuration", 0); +} + +TEST_F(MetricsReporterTest, OpenCardSuccessDuration) { + reporter_.OpenAction(0); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(19)); + reporter_.PageLoaded(); + + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.UserJourney.OpenCard.SuccessDuration", 1); + histogram_.ExpectUniqueTimeSample( + "ContentSuggestions.Feed.UserJourney.OpenCard.SuccessDuration", + base::TimeDelta::FromSeconds(19), 1); +} + +TEST_F(MetricsReporterTest, OpenCardTimeout) { + reporter_.OpenAction(0); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(21)); + reporter_.PageLoaded(); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.UserJourney.OpenCard.Failure", 1, 1); + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.UserJourney.OpenCard.SuccessDuration", 0); +} + +TEST_F(MetricsReporterTest, OpenCardFailureTwiceAndThenSucceed) { + reporter_.OpenAction(0); + reporter_.OpenAction(1); + reporter_.OpenAction(2); + reporter_.PageLoaded(); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.UserJourney.OpenCard.Failure", 1, 2); + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.UserJourney.OpenCard.SuccessDuration", 1); +} + +TEST_F(MetricsReporterTest, OpenCardCloseChromeFailure) { + reporter_.OpenAction(0); + reporter_.OnEnterBackground(); + + histogram_.ExpectUniqueSample( + "ContentSuggestions.Feed.UserJourney.OpenCard.Failure", 1, 1); + histogram_.ExpectTotalCount( + "ContentSuggestions.Feed.UserJourney.OpenCard.SuccessDuration", 0); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/prefs.cc b/chromium/components/feed/core/v2/prefs.cc index d3d5a0b39f2..589e445686d 100644 --- a/chromium/components/feed/core/v2/prefs.cc +++ b/chromium/components/feed/core/v2/prefs.cc @@ -8,18 +8,12 @@ #include "base/value_conversions.h" #include "base/values.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/v2/scheduling.h" #include "components/prefs/pref_registry_simple.h" #include "components/prefs/pref_service.h" namespace feed { -namespace { -const char kThrottlerRequestCountListPrefName[] = - "feedv2.request_throttler.request_counts"; -const char kThrottlerLastRequestTime[] = - "feedv2.request_throttler.last_request_time"; - -} // namespace - namespace prefs { std::vector<int> GetThrottlerRequestCounts(PrefService* pref_service) { @@ -51,6 +45,25 @@ void SetLastRequestTime(base::Time request_time, PrefService* pref_service) { return pref_service->SetTime(kThrottlerLastRequestTime, request_time); } +DebugStreamData GetDebugStreamData(PrefService* pref_service) { + return DeserializeDebugStreamData(pref_service->GetString(kDebugStreamData)) + .value_or(DebugStreamData()); +} + +void SetDebugStreamData(const DebugStreamData& data, + PrefService* pref_service) { + pref_service->SetString(kDebugStreamData, SerializeDebugStreamData(data)); +} + +void SetRequestSchedule(const RequestSchedule& schedule, + PrefService* pref_service) { + pref_service->Set(kRequestSchedule, RequestScheduleToValue(schedule)); +} + +RequestSchedule GetRequestSchedule(PrefService* pref_service) { + return RequestScheduleFromValue(*pref_service->Get(kRequestSchedule)); +} + } // namespace prefs } // namespace feed diff --git a/chromium/components/feed/core/v2/prefs.h b/chromium/components/feed/core/v2/prefs.h index 29da3c9c6cf..334372279ac 100644 --- a/chromium/components/feed/core/v2/prefs.h +++ b/chromium/components/feed/core/v2/prefs.h @@ -8,10 +8,12 @@ #include <vector> #include "base/time/time.h" +#include "components/feed/core/v2/public/types.h" class PrefService; namespace feed { +struct RequestSchedule; namespace prefs { // Functions for accessing prefs. @@ -27,6 +29,13 @@ void SetThrottlerRequestCounts(std::vector<int> request_counts, base::Time GetLastRequestTime(PrefService* pref_service); void SetLastRequestTime(base::Time request_time, PrefService* pref_service); +DebugStreamData GetDebugStreamData(PrefService* pref_service); +void SetDebugStreamData(const DebugStreamData& data, PrefService* pref_service); + +void SetRequestSchedule(const RequestSchedule& schedule, + PrefService* pref_service); +RequestSchedule GetRequestSchedule(PrefService* pref_service); + } // namespace prefs } // namespace feed diff --git a/chromium/components/feed/core/v2/proto_util.cc b/chromium/components/feed/core/v2/proto_util.cc index 90029710307..852666c263a 100644 --- a/chromium/components/feed/core/v2/proto_util.cc +++ b/chromium/components/feed/core/v2/proto_util.cc @@ -5,12 +5,134 @@ #include "components/feed/core/v2/proto_util.h" #include <tuple> +#include <vector> #include "base/strings/strcat.h" #include "base/strings/string_number_conversions.h" +#include "base/system/sys_info.h" +#include "build/build_config.h" #include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/feed_request.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/v2/feed_stream.h" + +#if defined(OS_ANDROID) +#include "base/android/build_info.h" +#endif namespace feed { +namespace { +feedwire::Version::Architecture GetBuildArchitecture() { +#if defined(ARCH_CPU_X86_64) + return feedwire::Version::X86_64; +#elif defined(ARCH_CPU_X86) + return feedwire::Version::X86; +#elif defined(ARCH_CPU_MIPS64) + return feedwire::Version::MIPS64; +#elif defined(ARCH_CPU_MIPS) + return feedwire::Version::MIPS; +#elif defined(ARCH_CPU_ARM64) + return feedwire::Version::ARM64; +#elif defined(ARCH_CPU_ARMEL) + return feedwire::Version::ARM; +#else + return feedwire::Version::UNKNOWN_ARCHITECTURE; +#endif +} + +feedwire::Version::Architecture GetSystemArchitecture() { + // By default, use |GetBuildArchitecture()|. + // In the case of x86 and ARM, the OS might be x86_64 or ARM_64. + feedwire::Version::Architecture build_arch = GetBuildArchitecture(); + if (build_arch == feedwire::Version::X86 && + base::SysInfo::OperatingSystemArchitecture() == "x86_64") { + return feedwire::Version::X86_64; + } + if (feedwire::Version::ARM && + base::SysInfo::OperatingSystemArchitecture() == "arm64") { + return feedwire::Version::ARM64; + } + return build_arch; +} + +feedwire::Version::BuildType GetBuildType(version_info::Channel channel) { + switch (channel) { + case version_info::Channel::CANARY: + return feedwire::Version::ALPHA; + case version_info::Channel::DEV: + return feedwire::Version::DEV; + case version_info::Channel::BETA: + return feedwire::Version::BETA; + case version_info::Channel::STABLE: + return feedwire::Version::RELEASE; + default: + return feedwire::Version::UNKNOWN_BUILD_TYPE; + } +} + +feedwire::Version GetPlatformVersionMessage() { + feedwire::Version result; + result.set_architecture(GetSystemArchitecture()); + result.set_build_type(feedwire::Version::RELEASE); + + int32_t major, minor, revision; + base::SysInfo::OperatingSystemVersionNumbers(&major, &minor, &revision); + result.set_major(major); + result.set_minor(minor); + result.set_revision(revision); +#if defined(OS_ANDROID) + result.set_api_version(base::android::BuildInfo::GetInstance()->sdk_int()); +#endif + return result; +} + +feedwire::Version GetAppVersionMessage(const ChromeInfo& chrome_info) { + feedwire::Version result; + result.set_architecture(GetBuildArchitecture()); + result.set_build_type(GetBuildType(chrome_info.channel)); + // Chrome's version is in the format: MAJOR,MINOR,BUILD,PATCH. + const std::vector<uint32_t>& numbers = chrome_info.version.components(); + if (numbers.size() > 3) { + result.set_major(static_cast<int32_t>(numbers[0])); + result.set_minor(static_cast<int32_t>(numbers[1])); + result.set_build(static_cast<int32_t>(numbers[2])); + result.set_revision(static_cast<int32_t>(numbers[3])); + } + +#if defined(OS_ANDROID) + result.set_api_version(base::android::BuildInfo::GetInstance()->sdk_int()); +#endif + return result; +} + +feedwire::Request CreateFeedQueryRequest( + feedwire::FeedQuery::RequestReason request_reason, + const RequestMetadata& request_metadata, + const std::string& consistency_token, + const std::string& next_page_token) { + feedwire::Request request; + request.set_request_version(feedwire::Request::FEED_QUERY); + + feedwire::FeedRequest& feed_request = *request.mutable_feed_request(); + feed_request.add_client_capability(feedwire::Capability::BASE_UI); + feed_request.add_client_capability(feedwire::Capability::REQUEST_SCHEDULE); + *feed_request.mutable_client_info() = CreateClientInfo(request_metadata); + feedwire::FeedQuery& query = *feed_request.mutable_feed_query(); + query.set_reason(request_reason); + if (!consistency_token.empty()) { + feed_request.mutable_consistency_token()->set_token(consistency_token); + } + if (!next_page_token.empty()) { + DCHECK_EQ(request_reason, feedwire::FeedQuery::NEXT_PAGE_SCROLL); + query.mutable_next_page_token() + ->mutable_next_page_token() + ->set_next_page_token(next_page_token); + feed_request.mutable_consistency_token()->set_token(consistency_token); + } + return request; +} + +} // namespace std::string ContentIdString(const feedwire::ContentId& content_id) { return base::StrCat({content_id.content_domain(), ",", @@ -34,6 +156,50 @@ bool CompareContentId(const feedwire::ContentId& a, std::tie(b.content_domain(), b_id, b_type); } +feedwire::ClientInfo CreateClientInfo(const RequestMetadata& request_metadata) { + feedwire::ClientInfo client_info; + // TODO(harringtond): Fill out client_instance_id. + // TODO(harringtond): Fill out advertising_id. + // TODO(harringtond): Fill out device_country. + + feedwire::DisplayInfo& display_info = *client_info.add_display_info(); + display_info.set_screen_density(request_metadata.display_metrics.density); + display_info.set_screen_width_in_pixels( + request_metadata.display_metrics.width_pixels); + display_info.set_screen_height_in_pixels( + request_metadata.display_metrics.height_pixels); + + client_info.set_locale(request_metadata.language_tag); + +#if defined(OS_ANDROID) + client_info.set_platform_type(feedwire::ClientInfo::ANDROID_ID); +#elif defined(OS_IOS) + client_info.set_platform_type(feedwire::ClientInfo::IOS); +#endif + client_info.set_app_type(feedwire::ClientInfo::TEST_APP); + *client_info.mutable_platform_version() = GetPlatformVersionMessage(); + *client_info.mutable_app_version() = + GetAppVersionMessage(request_metadata.chrome_info); + return client_info; +} + +feedwire::Request CreateFeedQueryRefreshRequest( + feedwire::FeedQuery::RequestReason request_reason, + const RequestMetadata& request_metadata, + const std::string& consistency_token) { + return CreateFeedQueryRequest(request_reason, request_metadata, + consistency_token, std::string()); +} + +feedwire::Request CreateFeedQueryLoadMoreRequest( + const RequestMetadata& request_metadata, + const std::string& consistency_token, + const std::string& next_page_token) { + return CreateFeedQueryRequest(feedwire::FeedQuery::NEXT_PAGE_SCROLL, + request_metadata, consistency_token, + next_page_token); +} + } // namespace feed namespace feedstore { diff --git a/chromium/components/feed/core/v2/proto_util.h b/chromium/components/feed/core/v2/proto_util.h index c7700625eb7..89506b0be82 100644 --- a/chromium/components/feed/core/v2/proto_util.h +++ b/chromium/components/feed/core/v2/proto_util.h @@ -8,16 +8,22 @@ #include <string> #include "base/time/time.h" - +#include "components/feed/core/proto/v2/wire/client_info.pb.h" #include "components/feed/core/proto/v2/wire/content_id.pb.h" +#include "components/feed/core/proto/v2/wire/feed_query.pb.h" +#include "components/feed/core/v2/types.h" +namespace feedwire { +class Request; +} // namespace feedwire namespace feedstore { class StreamData; -} +} // namespace feedstore // Helper functions/classes for dealing with feed proto messages. namespace feed { +using ContentId = feedwire::ContentId; std::string ContentIdString(const feedwire::ContentId&); bool Equal(const feedwire::ContentId& a, const feedwire::ContentId& b); @@ -32,6 +38,18 @@ class ContentIdCompareFunctor { } }; +feedwire::ClientInfo CreateClientInfo(const RequestMetadata& request_metadata); + +feedwire::Request CreateFeedQueryRefreshRequest( + feedwire::FeedQuery::RequestReason request_reason, + const RequestMetadata& request_metadata, + const std::string& consistency_token); + +feedwire::Request CreateFeedQueryLoadMoreRequest( + const RequestMetadata& request_metadata, + const std::string& consistency_token, + const std::string& next_page_token); + } // namespace feed namespace feedstore { diff --git a/chromium/components/feed/core/v2/proto_util_unittest.cc b/chromium/components/feed/core/v2/proto_util_unittest.cc new file mode 100644 index 00000000000..d405ee94d6d --- /dev/null +++ b/chromium/components/feed/core/v2/proto_util_unittest.cc @@ -0,0 +1,45 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/proto_util.h" + +#include "components/feed/core/proto/v2/wire/client_info.pb.h" +#include "components/feed/core/v2/test/proto_printer.h" +#include "components/feed/core/v2/types.h" +#include "components/version_info/channel.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { + +TEST(ProtoUtilTest, CreateClientInfo) { + RequestMetadata request_metadata; + request_metadata.chrome_info.version = base::Version({1, 2, 3, 4}); + request_metadata.chrome_info.channel = version_info::Channel::STABLE; + request_metadata.display_metrics.density = 1; + request_metadata.display_metrics.width_pixels = 2; + request_metadata.display_metrics.height_pixels = 3; + request_metadata.language_tag = "en-US"; + + feedwire::ClientInfo result = CreateClientInfo(request_metadata); + // TODO(harringtond): change back to CHROME when it is supported. + EXPECT_EQ(feedwire::ClientInfo::TEST_APP, result.app_type()); + EXPECT_EQ(feedwire::Version::RELEASE, result.app_version().build_type()); + EXPECT_EQ(1, result.app_version().major()); + EXPECT_EQ(2, result.app_version().minor()); + EXPECT_EQ(3, result.app_version().build()); + EXPECT_EQ(4, result.app_version().revision()); + + EXPECT_EQ(R"({ + screen_density: 1 + screen_width_in_pixels: 2 + screen_height_in_pixels: 3 +} +)", + ToTextProto(result.display_info(0))); + EXPECT_EQ("en-US", result.locale()); +} + +} // namespace +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model_update_request.cc b/chromium/components/feed/core/v2/protocol_translator.cc index 1cd8d8d77fc..0b0b691ba98 100644 --- a/chromium/components/feed/core/v2/stream_model_update_request.cc +++ b/chromium/components/feed/core/v2/protocol_translator.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/protocol_translator.h" #include <utility> @@ -12,6 +12,7 @@ #include "components/feed/core/proto/v2/wire/feature.pb.h" #include "components/feed/core/proto/v2/wire/feed_response.pb.h" #include "components/feed/core/proto/v2/wire/payload_metadata.pb.h" +#include "components/feed/core/proto/v2/wire/request_schedule.pb.h" #include "components/feed/core/proto/v2/wire/stream_structure.pb.h" #include "components/feed/core/proto/v2/wire/token.pb.h" #include "components/feed/core/v2/proto_util.h" @@ -52,13 +53,35 @@ feedstore::StreamStructure::Type TranslateNodeType( } } +base::TimeDelta TranslateDuration(const feedwire::Duration& v) { + return base::TimeDelta::FromSeconds(v.seconds()) + + base::TimeDelta::FromNanoseconds(v.nanos()); +} + +base::Optional<RequestSchedule> TranslateRequestSchedule( + base::Time now, + const feedwire::RequestSchedule& v) { + RequestSchedule schedule; + const feedwire::RequestSchedule_TimeBasedSchedule& time_schedule = + v.time_based_schedule(); + for (const feedwire::Duration& duration : + time_schedule.refresh_time_from_response_time()) { + schedule.refresh_offsets.push_back(TranslateDuration(duration)); + } + schedule.anchor_time = now; + return schedule; +} + +// Fields that should be present at most once in the response. +struct ConvertedGlobalData { + base::Optional<RequestSchedule> request_schedule; +}; + struct ConvertedDataOperation { - bool has_stream_structure = false; feedstore::StreamStructure stream_structure; - bool has_content = false; - feedstore::Content content; - bool has_shared_state = false; - feedstore::StreamSharedState shared_state; + base::Optional<feedstore::Content> content; + base::Optional<feedstore::StreamSharedState> shared_state; + base::Optional<std::string> next_page_token; }; bool TranslateFeature(feedwire::Feature* feature, @@ -70,17 +93,17 @@ bool TranslateFeature(feedwire::Feature* feature, if (type == feedstore::StreamStructure::CONTENT) { feedwire::Content* wire_content = feature->mutable_content_extension(); - if (wire_content->type() != feedwire::Content::XSURFACE) + if (!wire_content->has_xsurface_content()) return false; // TODO(iwells): We still need score, availability_time_seconds, // offline_metadata, and representation_data to populate content_info. - *(result->content.mutable_content_id()) = + result->content.emplace(); + *(result->content->mutable_content_id()) = result->stream_structure.content_id(); - result->content.set_allocated_frame( + result->content->set_allocated_frame( wire_content->mutable_xsurface_content()->release_xsurface_output()); - result->has_content = true; } return true; } @@ -99,7 +122,9 @@ base::Optional<feedstore::StreamSharedState> TranslateSharedState( return shared_state; } -bool TranslatePayload(feedwire::DataOperation operation, +bool TranslatePayload(base::Time now, + feedwire::DataOperation operation, + ConvertedGlobalData* global_data, ConvertedDataOperation* result) { switch (operation.payload_case()) { case feedwire::DataOperation::kFeature: { @@ -114,27 +139,23 @@ bool TranslatePayload(feedwire::DataOperation operation, feedwire::Token* token = operation.mutable_next_page_token(); result->stream_structure.set_allocated_parent_id( token->release_parent_id()); - // TODO(iwells): We should be setting token bytes here. - // result->stream_structure.set_allocated_next_page_token( - // token->MutableExtension( - // components::feed::core::proto::ui - // ::stream::NextPageToken::next_page_token_extension - // )->release_next_page_token()); + result->next_page_token = std::move( + *token->mutable_next_page_token()->mutable_next_page_token()); } break; case feedwire::DataOperation::kRenderData: { - base::Optional<feedstore::StreamSharedState> shared_state = + result->shared_state = TranslateSharedState(result->stream_structure.content_id(), operation.mutable_render_data()); - if (!shared_state) + if (!result->shared_state) return false; - - result->shared_state = std::move(shared_state.value()); - result->has_shared_state = true; } break; - // Fall through - case feedwire::DataOperation::kInPlaceUpdateHandle: - case feedwire::DataOperation::kTemplates: - case feedwire::DataOperation::PAYLOAD_NOT_SET: + case feedwire::DataOperation::kRequestSchedule: { + if (global_data) { + global_data->request_schedule = + TranslateRequestSchedule(now, operation.request_schedule()); + } + } break; + default: return false; } @@ -143,13 +164,14 @@ bool TranslatePayload(feedwire::DataOperation operation, } base::Optional<ConvertedDataOperation> TranslateDataOperationInternal( - feedwire::DataOperation operation) { + base::Time now, + feedwire::DataOperation operation, + ConvertedGlobalData* global_data) { feedstore::StreamStructure::Operation operation_type = TranslateOperationType(operation.operation()); ConvertedDataOperation result; result.stream_structure.set_operation(operation_type); - result.has_stream_structure = true; switch (operation_type) { case feedstore::StreamStructure::CLEAR_ALL: @@ -162,7 +184,7 @@ base::Optional<ConvertedDataOperation> TranslateDataOperationInternal( result.stream_structure.set_allocated_content_id( operation.mutable_metadata()->release_content_id()); - if (!TranslatePayload(std::move(operation), &result)) + if (!TranslatePayload(now, std::move(operation), global_data, &result)) return base::nullopt; break; @@ -191,51 +213,71 @@ StreamModelUpdateRequest::StreamModelUpdateRequest( StreamModelUpdateRequest& StreamModelUpdateRequest::operator=( const StreamModelUpdateRequest&) = default; +RefreshResponseData::RefreshResponseData() = default; +RefreshResponseData::~RefreshResponseData() = default; +RefreshResponseData::RefreshResponseData(RefreshResponseData&&) = default; +RefreshResponseData& RefreshResponseData::operator=(RefreshResponseData&&) = + default; + base::Optional<feedstore::DataOperation> TranslateDataOperation( + base::Time now, feedwire::DataOperation wire_operation) { feedstore::DataOperation store_operation; + // Note: This function is used when executing operations in response to + // actions embedded in the server protobuf. Some data in data operations + // aren't supported by this function, which is why we're passing in + // global_data=nullptr. base::Optional<ConvertedDataOperation> converted = - TranslateDataOperationInternal(std::move(wire_operation)); + TranslateDataOperationInternal(now, std::move(wire_operation), nullptr); if (!converted) return base::nullopt; - if (!converted->has_stream_structure && !converted->has_content) + // We only support translating StreamSharedStates when they will be attached + // to StreamModelUpdateRequests. + if (converted->shared_state) return base::nullopt; *store_operation.mutable_structure() = std::move(converted->stream_structure); - *store_operation.mutable_content() = std::move(converted->content); + if (converted->content) + *store_operation.mutable_content() = std::move(*converted->content); + return store_operation; } -std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( +RefreshResponseData TranslateWireResponse( feedwire::Response response, - base::TimeDelta response_time, + StreamModelUpdateRequest::Source source, base::Time current_time) { if (response.response_version() != feedwire::Response::FEED_RESPONSE) - return nullptr; + return {}; auto result = std::make_unique<StreamModelUpdateRequest>(); + result->source = source; + ConvertedGlobalData global_data; feedwire::FeedResponse* feed_response = response.mutable_feed_response(); for (auto& wire_data_operation : *feed_response->mutable_data_operation()) { if (!wire_data_operation.has_operation()) continue; base::Optional<ConvertedDataOperation> operation = - TranslateDataOperationInternal(std::move(wire_data_operation)); + TranslateDataOperationInternal( + current_time, std::move(wire_data_operation), &global_data); if (!operation) continue; - if (operation->has_stream_structure) { - result->stream_structures.push_back( - std::move(operation->stream_structure)); - } + result->stream_structures.push_back(std::move(operation->stream_structure)); - if (operation->has_content) - result->content.push_back(std::move(operation.value().content)); + if (operation->content) + result->content.push_back(std::move(*operation->content)); - if (operation->has_shared_state) - result->shared_states.push_back(std::move(operation->shared_state)); + if (operation->shared_state) + result->shared_states.push_back(std::move(*operation->shared_state)); + + if (operation->next_page_token) { + result->stream_data.set_next_page_token( + std::move(*operation->next_page_token)); + } } // TODO(harringtond): If there's more than one shared state, record some @@ -245,11 +287,12 @@ std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( result->shared_states.front().content_id(); } feedstore::SetLastAddedTime(current_time, &result->stream_data); - result->server_response_time = - feed_response->feed_response_metadata().response_time_ms(); - result->response_time = response_time; - return result; + RefreshResponseData response_data; + response_data.model_update_request = std::move(result); + response_data.request_schedule = std::move(global_data.request_schedule); + + return response_data; } } // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model_update_request.h b/chromium/components/feed/core/v2/protocol_translator.h index 6aea209a782..fdaaa60368c 100644 --- a/chromium/components/feed/core/v2/stream_model_update_request.h +++ b/chromium/components/feed/core/v2/protocol_translator.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_ -#define COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_ +#ifndef COMPONENTS_FEED_CORE_V2_PROTOCOL_TRANSLATOR_H_ +#define COMPONENTS_FEED_CORE_V2_PROTOCOL_TRANSLATOR_H_ #include <memory> #include <vector> @@ -13,6 +13,7 @@ #include "components/feed/core/proto/v2/store.pb.h" #include "components/feed/core/proto/v2/wire/data_operation.pb.h" #include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/feed/core/v2/scheduling.h" namespace feed { @@ -23,6 +24,7 @@ struct StreamModelUpdateRequest { enum class Source { kNetworkUpdate, kInitialLoadFromStore, + kNetworkLoadMore, }; StreamModelUpdateRequest(); @@ -48,27 +50,30 @@ struct StreamModelUpdateRequest { std::vector<feedstore::StreamStructure> stream_structures; - // If this data originates from the network, this is the server-reported time - // at which the request was fulfilled. - // TODO(harringtond): Use this or remove it. - int64_t server_response_time = 0; + int32_t max_structure_sequence_number = 0; +}; - // If this data originates from the network, this is the time taken by the - // server to produce the response. - // TODO(harringtond): Use this or remove it. - base::TimeDelta response_time; +struct RefreshResponseData { + RefreshResponseData(); + ~RefreshResponseData(); + RefreshResponseData(RefreshResponseData&&); + RefreshResponseData& operator=(RefreshResponseData&&); - int32_t max_structure_sequence_number = 0; + std::unique_ptr<StreamModelUpdateRequest> model_update_request; + + // Server-defined request schedule, if provided. + base::Optional<RequestSchedule> request_schedule; }; base::Optional<feedstore::DataOperation> TranslateDataOperation( + base::Time current_time, feedwire::DataOperation wire_operation); -std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( +RefreshResponseData TranslateWireResponse( feedwire::Response response, - base::TimeDelta response_time, + StreamModelUpdateRequest::Source source, base::Time current_time); } // namespace feed -#endif // COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_ +#endif // COMPONENTS_FEED_CORE_V2_PROTOCOL_TRANSLATOR_H_ diff --git a/chromium/components/feed/core/v2/protocol_translator_unittest.cc b/chromium/components/feed/core/v2/protocol_translator_unittest.cc new file mode 100644 index 00000000000..99a93a3d478 --- /dev/null +++ b/chromium/components/feed/core/v2/protocol_translator_unittest.cc @@ -0,0 +1,603 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/protocol_translator.h" + +#include <sstream> +#include <string> + +#include "base/base_paths.h" +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/path_service.h" +#include "base/strings/string_number_conversions.h" +#include "base/time/time.h" +#include "components/feed/core/proto/v2/wire/feed_response.pb.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/feed/core/v2/proto_util.h" +#include "components/feed/core/v2/test/proto_printer.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { + +const char kResponsePbPath[] = "components/test/data/feed/response.binarypb"; +const base::Time kCurrentTime = + base::Time::UnixEpoch() + base::TimeDelta::FromDays(123); + +feedwire::Response TestWireResponse() { + // Read and parse response.binarypb. + base::FilePath response_file_path; + CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &response_file_path)); + response_file_path = response_file_path.AppendASCII(kResponsePbPath); + + CHECK(base::PathExists(response_file_path)); + + std::string response_data; + CHECK(base::ReadFileToString(response_file_path, &response_data)); + + feedwire::Response response; + CHECK(response.ParseFromString(response_data)); + return response; +} + +feedwire::Response EmptyWireResponse() { + feedwire::Response response; + response.set_response_version(feedwire::Response::FEED_RESPONSE); + return response; +} + +feedwire::DataOperation MakeDataOperation( + feedwire::DataOperation::Operation operation) { + feedwire::DataOperation result; + result.set_operation(operation); + result.mutable_metadata()->mutable_content_id()->set_id(42); + return result; +} + +feedwire::DataOperation MakeDataOperationWithContent( + feedwire::DataOperation::Operation operation, + std::string xsurface_content = "content") { + feedwire::DataOperation result = MakeDataOperation(operation); + result.mutable_feature()->set_renderable_unit(feedwire::Feature::CONTENT); + result.mutable_feature() + ->mutable_content_extension() + ->mutable_xsurface_content() + ->set_xsurface_output(std::move(xsurface_content)); + return result; +} + +feedwire::DataOperation MakeDataOperationWithRenderData( + feedwire::DataOperation::Operation operation, + std::string xsurface_render_data = "renderdata") { + feedwire::DataOperation result = MakeDataOperation(operation); + result.mutable_render_data()->set_render_data_type( + feedwire::RenderData::XSURFACE); + result.mutable_render_data()->mutable_xsurface_container()->set_render_data( + std::move(xsurface_render_data)); + return result; +} + +// Helpers to add some common params. +RefreshResponseData TranslateWireResponse(feedwire::Response response) { + return TranslateWireResponse( + response, StreamModelUpdateRequest::Source::kNetworkUpdate, kCurrentTime); +} + +base::Optional<feedstore::DataOperation> TranslateDataOperation( + feedwire::DataOperation operation) { + return ::feed::TranslateDataOperation(base::Time(), std::move(operation)); +} + +} // namespace + +TEST(ProtocolTranslatorTest, NextPageToken) { + feedwire::Response response = EmptyWireResponse(); + feedwire::DataOperation* operation = + response.mutable_feed_response()->add_data_operation(); + operation->set_operation(feedwire::DataOperation::UPDATE_OR_APPEND); + operation->mutable_metadata()->mutable_content_id()->set_id(1); + operation->mutable_next_page_token() + ->mutable_next_page_token() + ->set_next_page_token("token"); + + RefreshResponseData translated = TranslateWireResponse(response); + ASSERT_TRUE(translated.model_update_request); + EXPECT_EQ(1ul, translated.model_update_request->stream_structures.size()); + EXPECT_EQ("token", + translated.model_update_request->stream_data.next_page_token()); +} + +TEST(ProtocolTranslatorTest, EmptyResponse) { + feedwire::Response response = EmptyWireResponse(); + EXPECT_TRUE(TranslateWireResponse(response).model_update_request); +} + +TEST(ProtocolTranslatorTest, MissingResponseVersion) { + feedwire::Response response = EmptyWireResponse(); + response.set_response_version(feedwire::Response::UNKNOWN_RESPONSE_VERSION); + EXPECT_FALSE(TranslateWireResponse(response).model_update_request); +} + +TEST(ProtocolTranslatorTest, TranslateContent) { + feedwire::DataOperation wire_operation = + MakeDataOperationWithContent(feedwire::DataOperation::UPDATE_OR_APPEND); + base::Optional<feedstore::DataOperation> translated = + TranslateDataOperation(wire_operation); + EXPECT_TRUE(translated); + EXPECT_EQ("content", translated->content().frame()); +} + +TEST(ProtocolTranslatorTest, TranslateContentFailsWhenMissingContent) { + feedwire::DataOperation wire_operation = + MakeDataOperationWithContent(feedwire::DataOperation::UPDATE_OR_APPEND); + wire_operation.mutable_feature()->clear_content_extension(); + EXPECT_FALSE(TranslateDataOperation(wire_operation)); +} + +TEST(ProtocolTranslatorTest, TranslateRenderData) { + feedwire::Response wire_response = EmptyWireResponse(); + *wire_response.mutable_feed_response()->add_data_operation() = + MakeDataOperationWithRenderData( + feedwire::DataOperation::UPDATE_OR_APPEND); + RefreshResponseData translated = TranslateWireResponse(wire_response); + EXPECT_TRUE(translated.model_update_request); + ASSERT_EQ(1ul, translated.model_update_request->shared_states.size()); + EXPECT_EQ( + "renderdata", + translated.model_update_request->shared_states[0].shared_state_data()); +} + +TEST(ProtocolTranslatorTest, TranslateRenderDataFailsWithUnknownType) { + feedwire::Response wire_response = EmptyWireResponse(); + feedwire::DataOperation wire_operation = MakeDataOperationWithRenderData( + feedwire::DataOperation::UPDATE_OR_APPEND); + wire_operation.mutable_render_data()->clear_render_data_type(); + *wire_response.mutable_feed_response()->add_data_operation() = + std::move(wire_operation); + + RefreshResponseData translated = TranslateWireResponse(wire_response); + EXPECT_TRUE(translated.model_update_request); + ASSERT_EQ(0ul, translated.model_update_request->shared_states.size()); +} + +TEST(ProtocolTranslatorTest, RenderDataOperationCanOnlyComeFromFullResponse) { + EXPECT_FALSE(TranslateDataOperation(MakeDataOperationWithRenderData( + feedwire::DataOperation::UPDATE_OR_APPEND))); +} + +TEST(ProtocolTranslatorTest, TranslateOperationFailsWithNoPayload) { + feedwire::DataOperation wire_operation = + MakeDataOperationWithContent(feedwire::DataOperation::UPDATE_OR_APPEND); + wire_operation.clear_feature(); + EXPECT_FALSE(TranslateDataOperation(wire_operation)); +} + +TEST(ProtocolTranslatorTest, TranslateOperationWithoutContentId) { + feedwire::DataOperation update_operation = + MakeDataOperationWithContent(feedwire::DataOperation::UPDATE_OR_APPEND); + update_operation.clear_metadata(); + EXPECT_FALSE(TranslateDataOperation(update_operation)); + + feedwire::DataOperation remove_operation = + MakeDataOperationWithContent(feedwire::DataOperation::REMOVE); + remove_operation.clear_metadata(); + EXPECT_FALSE(TranslateDataOperation(remove_operation)); + + // CLEAR_ALL doesn't need a content ID. + feedwire::DataOperation clear_operation = + MakeDataOperation(feedwire::DataOperation::CLEAR_ALL); + EXPECT_TRUE(TranslateDataOperation(clear_operation)); +} + +TEST(ProtocolTranslatorTest, TranslateOperationFailsWithUnknownOperation) { + feedwire::DataOperation wire_operation = + MakeDataOperation(feedwire::DataOperation::UNKNOWN_OPERATION); + EXPECT_FALSE(TranslateDataOperation(wire_operation)); +} + +TEST(ProtocolTranslatorTest, TranslateRealResponse) { + // Tests how proto translation works on a real response from the server. + // + // The response will periodically need to be updated as changes are made to + // the server. Update testdata/response.textproto and then run + // tools/generate_test_response_binarypb.sh. + + feedwire::Response response = TestWireResponse(); + + RefreshResponseData translated = TranslateWireResponse(response); + + ASSERT_TRUE(translated.model_update_request); + ASSERT_TRUE(translated.request_schedule); + EXPECT_EQ(kCurrentTime, translated.request_schedule->anchor_time); + EXPECT_EQ(std::vector<base::TimeDelta>( + {base::TimeDelta::FromSeconds(86308) + + base::TimeDelta::FromNanoseconds(822963644)}), + translated.request_schedule->refresh_offsets); + + std::stringstream ss; + ss << *translated.model_update_request; + + const std::string want = R"(source: 0 +stream_data: { + last_added_time_millis: 10627200000 + shared_state_id { + content_domain: "render_data" + } +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 3328940074512586021 + } + frame: "data2" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 8191455549164721606 + } + frame: "data3" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 10337142060535577025 + } + frame: "data4" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 9467333465122011616 + } + frame: "data5" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 10024917518268143371 + } + frame: "data6" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 14956621708214864803 + } + frame: "data7" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 2741853109953412745 + } + frame: "data8" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 586433679892097787 + } + frame: "data9" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 790985792726953756 + } + frame: "data10" +} +content: { + content_id { + content_domain: "stories.f" + type: 1 + id: 7324025093440047528 + } + frame: "data11" +} +shared_state: { + content_id { + content_domain: "render_data" + } + shared_state_data: "data1" +} +stream_structure: { + operation: 1 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "root" + } + type: 1 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "render_data" + } +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 3328940074512586021 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 14679492703605464401 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 14679492703605464401 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 8191455549164721606 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 16663153735812675251 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 16663153735812675251 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 10337142060535577025 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 15532023010474785878 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 15532023010474785878 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 9467333465122011616 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 10111267591181086437 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 10111267591181086437 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 10024917518268143371 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 6703713839373923610 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 6703713839373923610 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 14956621708214864803 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 12592500096310265284 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 12592500096310265284 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 2741853109953412745 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 1016582787945881825 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 1016582787945881825 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 586433679892097787 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 9506447424580769257 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 9506447424580769257 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 790985792726953756 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 17612738377810195843 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 17612738377810195843 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "stories.f" + type: 1 + id: 7324025093440047528 + } + parent_id { + content_domain: "content.f" + type: 3 + id: 5093490247022575399 + } + type: 3 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "content.f" + type: 3 + id: 5093490247022575399 + } + parent_id { + content_domain: "root" + } + type: 4 +} +stream_structure: { + operation: 2 + content_id { + content_domain: "request_schedule" + id: 300842786 + } +} +max_structure_sequence_number: 0 +)"; + EXPECT_EQ(want, ss.str()); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/public/feed_service.cc b/chromium/components/feed/core/v2/public/feed_service.cc index d1fb36f1fd9..67202a0f3dc 100644 --- a/chromium/components/feed/core/v2/public/feed_service.cc +++ b/chromium/components/feed/core/v2/public/feed_service.cc @@ -8,16 +8,20 @@ #include "base/time/default_clock.h" #include "base/time/default_tick_clock.h" +#include "build/build_config.h" #include "components/feed/core/v2/feed_network_impl.h" #include "components/feed/core/v2/feed_store.h" #include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/metrics_reporter.h" #include "components/feed/core/v2/refresh_task_scheduler.h" +#include "components/history/core/browser/history_service.h" +#include "components/history/core/browser/history_service_observer.h" +#include "components/history/core/browser/history_types.h" #include "net/base/network_change_notifier.h" #include "services/network/public/cpp/shared_url_loader_factory.h" namespace feed { namespace { - class EulaObserver : public web_resource::EulaAcceptedNotifier::Observer { public: explicit EulaObserver(FeedStream* feed_stream) : feed_stream_(feed_stream) {} @@ -33,6 +37,41 @@ class EulaObserver : public web_resource::EulaAcceptedNotifier::Observer { } // namespace +namespace internal { +bool ShouldClearFeed(const history::DeletionInfo& deletion_info) { + // We ignore expirations since they're not user-initiated. + if (deletion_info.is_from_expiration()) + return false; + + // If a user deletes a single URL, we don't consider this a clear user + // intent to clear our data. + return deletion_info.IsAllHistory() || + deletion_info.deleted_rows().size() > 1; +} +} // namespace internal + +class FeedService::HistoryObserverImpl + : public history::HistoryServiceObserver { + public: + HistoryObserverImpl(history::HistoryService* history_service, + FeedStream* feed_stream) + : feed_stream_(feed_stream) { + history_service->AddObserver(this); + } + HistoryObserverImpl(const HistoryObserverImpl&) = delete; + HistoryObserverImpl& operator=(const HistoryObserverImpl&) = delete; + + // history::HistoryServiceObserver. + void OnURLsDeleted(history::HistoryService* history_service, + const history::DeletionInfo& deletion_info) override { + if (internal::ShouldClearFeed(deletion_info)) + feed_stream_->OnHistoryDeleted(); + } + + private: + FeedStream* feed_stream_; +}; + class FeedService::NetworkDelegateImpl : public FeedNetworkImpl::Delegate { public: explicit NetworkDelegateImpl(FeedService::Delegate* service_delegate) @@ -51,8 +90,9 @@ class FeedService::NetworkDelegateImpl : public FeedNetworkImpl::Delegate { class FeedService::StreamDelegateImpl : public FeedStream::Delegate { public: - explicit StreamDelegateImpl(PrefService* local_state) - : eula_notifier_(local_state) {} + StreamDelegateImpl(PrefService* local_state, + FeedService::Delegate* service_delegate) + : service_delegate_(service_delegate), eula_notifier_(local_state) {} StreamDelegateImpl(const StreamDelegateImpl&) = delete; StreamDelegateImpl& operator=(const StreamDelegateImpl&) = delete; @@ -64,13 +104,21 @@ class FeedService::StreamDelegateImpl : public FeedStream::Delegate { // FeedStream::Delegate. bool IsEulaAccepted() override { return eula_notifier_.IsEulaAccepted(); } bool IsOffline() override { return net::NetworkChangeNotifier::IsOffline(); } + DisplayMetrics GetDisplayMetrics() override { + return service_delegate_->GetDisplayMetrics(); + } + std::string GetLanguageTag() override { + return service_delegate_->GetLanguageTag(); + } private: + FeedService::Delegate* service_delegate_; web_resource::EulaAcceptedNotifier eula_notifier_; std::unique_ptr<EulaObserver> eula_observer_; + std::unique_ptr<HistoryObserverImpl> history_observer_; }; -FeedService::FeedService(std::unique_ptr<FeedStreamApi> stream) +FeedService::FeedService(std::unique_ptr<FeedStream> stream) : stream_(std::move(stream)) {} FeedService::FeedService( @@ -80,34 +128,66 @@ FeedService::FeedService( PrefService* local_state, std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> database, signin::IdentityManager* identity_manager, + history::HistoryService* history_service, scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory, scoped_refptr<base::SequencedTaskRunner> background_task_runner, - const std::string& api_key) + const std::string& api_key, + const ChromeInfo& chrome_info) : delegate_(std::move(delegate)), refresh_task_scheduler_(std::move(refresh_task_scheduler)) { - stream_delegate_ = std::make_unique<StreamDelegateImpl>(local_state); + stream_delegate_ = + std::make_unique<StreamDelegateImpl>(local_state, delegate_.get()); network_delegate_ = std::make_unique<NetworkDelegateImpl>(delegate_.get()); + metrics_reporter_ = + std::make_unique<MetricsReporter>(base::DefaultTickClock::GetInstance()); feed_network_ = std::make_unique<FeedNetworkImpl>( network_delegate_.get(), identity_manager, api_key, url_loader_factory, base::DefaultTickClock::GetInstance(), profile_prefs); store_ = std::make_unique<FeedStore>(std::move(database)); stream_ = std::make_unique<FeedStream>( - refresh_task_scheduler_.get(), - nullptr, // TODO(harringtond): Implement EventObserver. + refresh_task_scheduler_.get(), metrics_reporter_.get(), stream_delegate_.get(), profile_prefs, feed_network_.get(), store_.get(), base::DefaultClock::GetInstance(), base::DefaultTickClock::GetInstance(), - background_task_runner); + chrome_info); + history_observer_ = std::make_unique<HistoryObserverImpl>( + history_service, static_cast<FeedStream*>(stream_.get())); stream_delegate_->Initialize(static_cast<FeedStream*>(stream_.get())); - // TODO(harringtond): Call FeedStream::OnSignedIn() - // TODO(harringtond): Call FeedStream::OnSignedOut() - // TODO(harringtond): Call FeedStream::OnHistoryDeleted() - // TODO(harringtond): Call FeedStream::OnCacheDataCleared() - // TODO(harringtond): Call FeedStream::OnEnterForeground() +// TODO(harringtond): Call FeedStream::OnSignedIn() +// TODO(harringtond): Call FeedStream::OnSignedOut() +#if defined(OS_ANDROID) + application_status_listener_ = + base::android::ApplicationStatusListener::New(base::BindRepeating( + &FeedService::OnApplicationStateChange, base::Unretained(this))); +#endif } FeedService::~FeedService() = default; +FeedStreamApi* FeedService::GetStream() { + return stream_.get(); +} + +void FeedService::ClearCachedData() { + stream_->OnCacheDataCleared(); +} + +#if defined(OS_ANDROID) +void FeedService::OnApplicationStateChange( + base::android::ApplicationState state) { + if (state == base::android::APPLICATION_STATE_HAS_RUNNING_ACTIVITIES) { + // If we want to trigger an OnEnterForeground event, we'll need to be + // careful about the initial state of foregrounded_. + foregrounded_ = true; + } + if (foregrounded_ && + state == base::android::APPLICATION_STATE_HAS_PAUSED_ACTIVITIES) { + foregrounded_ = false; + stream_->OnEnterBackground(); + } +} +#endif + } // namespace feed diff --git a/chromium/components/feed/core/v2/public/feed_service.h b/chromium/components/feed/core/v2/public/feed_service.h index 2d24ad0d0f6..45fc8848abd 100644 --- a/chromium/components/feed/core/v2/public/feed_service.h +++ b/chromium/components/feed/core/v2/public/feed_service.h @@ -10,28 +10,44 @@ #include "base/files/file_path.h" #include "base/memory/scoped_refptr.h" +#include "build/build_config.h" #include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/public/types.h" #include "components/keyed_service/core/keyed_service.h" #include "components/leveldb_proto/public/proto_database.h" #include "components/web_resource/eula_accepted_notifier.h" +#if defined(OS_ANDROID) +#include "base/android/application_status_listener.h" +#endif + namespace base { class SequencedTaskRunner; -} +} // namespace base +namespace history { +class HistoryService; +class DeletionInfo; +} // namespace history namespace feedstore { class Record; -} +} // namespace feedstore namespace network { class SharedURLLoaderFactory; -} +} // namespace network namespace signin { class IdentityManager; -} +} // namespace signin namespace feed { class RefreshTaskScheduler; +class MetricsReporter; class FeedNetwork; class FeedStore; +class FeedStream; + +namespace internal { +bool ShouldClearFeed(const history::DeletionInfo& deletion_info); +} // namespace internal class FeedService : public KeyedService { public: @@ -41,11 +57,13 @@ class FeedService : public KeyedService { // Returns a string which represents the top locale and region of the // device. virtual std::string GetLanguageTag() = 0; + // Returns display metrics for the device. + virtual DisplayMetrics GetDisplayMetrics() = 0; }; - // Construct a FeedService given an already constructed FeedStreamApi. + // Construct a FeedService given an already constructed FeedStream. // Used for testing only. - explicit FeedService(std::unique_ptr<FeedStreamApi> stream); + explicit FeedService(std::unique_ptr<FeedStream> stream); // Construct a new FeedStreamApi along with FeedService. FeedService( @@ -55,29 +73,47 @@ class FeedService : public KeyedService { PrefService* local_state, std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> database, signin::IdentityManager* identity_manager, + history::HistoryService* history_service, scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory, scoped_refptr<base::SequencedTaskRunner> background_task_runner, - const std::string& api_key); + const std::string& api_key, + const ChromeInfo& chrome_info); ~FeedService() override; FeedService(const FeedService&) = delete; FeedService& operator=(const FeedService&) = delete; - FeedStreamApi* GetStream() { return stream_.get(); } + FeedStreamApi* GetStream(); + + void ClearCachedData(); + + RefreshTaskScheduler* GetRefreshTaskScheduler() { + return refresh_task_scheduler_.get(); + } private: class StreamDelegateImpl; class NetworkDelegateImpl; + class HistoryObserverImpl; +#if defined(OS_ANDROID) + void OnApplicationStateChange(base::android::ApplicationState state); +#endif // These components are owned for construction of |FeedStreamApi|. These will // be null if |FeedStreamApi| is created externally. std::unique_ptr<Delegate> delegate_; std::unique_ptr<StreamDelegateImpl> stream_delegate_; + std::unique_ptr<MetricsReporter> metrics_reporter_; std::unique_ptr<NetworkDelegateImpl> network_delegate_; std::unique_ptr<FeedNetwork> feed_network_; std::unique_ptr<FeedStore> store_; std::unique_ptr<RefreshTaskScheduler> refresh_task_scheduler_; - - std::unique_ptr<FeedStreamApi> stream_; + std::unique_ptr<HistoryObserverImpl> history_observer_; +#if defined(OS_ANDROID) + bool foregrounded_ = true; + std::unique_ptr<base::android::ApplicationStatusListener> + application_status_listener_; +#endif + std::unique_ptr<FeedStream> stream_; }; } // namespace feed diff --git a/chromium/components/feed/core/v2/public/feed_service_unittest.cc b/chromium/components/feed/core/v2/public/feed_service_unittest.cc new file mode 100644 index 00000000000..50fa7e2776c --- /dev/null +++ b/chromium/components/feed/core/v2/public/feed_service_unittest.cc @@ -0,0 +1,34 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/public/feed_service.h" +#include "components/history/core/browser/history_types.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "url/gurl.h" + +namespace feed { +namespace internal { +namespace { +using history::DeletionInfo; + +TEST(ShouldClearFeed, ShouldClearFeed) { + EXPECT_TRUE(ShouldClearFeed(DeletionInfo::ForAllHistory())); + EXPECT_TRUE(ShouldClearFeed(DeletionInfo::ForUrls( + { + history::URLRow(GURL("http://url1")), + history::URLRow(GURL("http://url2")), + }, + /*favicon_urls=*/{}))); + + EXPECT_FALSE(ShouldClearFeed(DeletionInfo::ForUrls( + { + history::URLRow(GURL("http://url1")), + }, + /*favicon_urls=*/{}))); +} + +} // namespace + +} // namespace internal +} // namespace feed diff --git a/chromium/components/feed/core/v2/public/feed_stream_api.cc b/chromium/components/feed/core/v2/public/feed_stream_api.cc new file mode 100644 index 00000000000..87c23e7bb52 --- /dev/null +++ b/chromium/components/feed/core/v2/public/feed_stream_api.cc @@ -0,0 +1,23 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/public/feed_stream_api.h" + +namespace feed { + +FeedStreamApi::FeedStreamApi() = default; +FeedStreamApi::~FeedStreamApi() = default; + +FeedStreamApi::SurfaceInterface::SurfaceInterface() { + static SurfaceId::Generator id_generator; + surface_id_ = id_generator.GenerateNextId(); +} + +FeedStreamApi::SurfaceInterface::~SurfaceInterface() = default; + +SurfaceId FeedStreamApi::SurfaceInterface::GetSurfaceId() const { + return surface_id_; +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/public/feed_stream_api.h b/chromium/components/feed/core/v2/public/feed_stream_api.h index 46a025c71fe..6699db20184 100644 --- a/chromium/components/feed/core/v2/public/feed_stream_api.h +++ b/chromium/components/feed/core/v2/public/feed_stream_api.h @@ -5,26 +5,20 @@ #ifndef COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_STREAM_API_H_ #define COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_STREAM_API_H_ +#include <string> #include <vector> #include "base/observer_list_types.h" -#include "base/util/type_safety/id_type.h" -#include "components/feed/core/proto/v2/wire/content_id.pb.h" +#include "components/feed/core/v2/public/types.h" namespace feedui { class StreamUpdate; -} +} // namespace feedui namespace feedstore { class DataOperation; -} +} // namespace feedstore namespace feed { -using ContentId = feedwire::ContentId; -// Uniquely identifies a revision of a |feedstore::Content|. If Content changes, -// it is assigned a new revision number. -using ContentRevision = util::IdTypeU32<class ContentRevisionClass>; -// A unique ID for an ephemeral change. -using EphemeralChangeId = util::IdTypeU32<class EphemeralChangeIdClass>; // This is the public access point for interacting with the Feed stream // contents. @@ -32,13 +26,23 @@ class FeedStreamApi { public: class SurfaceInterface : public base::CheckedObserver { public: + SurfaceInterface(); + ~SurfaceInterface() override; // Called after registering the observer to provide the full stream state. // Also called whenever the stream changes. virtual void StreamUpdate(const feedui::StreamUpdate&) = 0; + // Returns a unique ID for the surface. The ID will not be reused until + // after the Chrome process is closed. + SurfaceId GetSurfaceId() const; + + private: + SurfaceId surface_id_; }; - FeedStreamApi() = default; - virtual ~FeedStreamApi() = default; + FeedStreamApi(); + virtual ~FeedStreamApi(); + FeedStreamApi(const FeedStreamApi&) = delete; + FeedStreamApi& operator=(const FeedStreamApi&) = delete; virtual void AttachSurface(SurfaceInterface*) = 0; virtual void DetachSurface(SurfaceInterface*) = 0; @@ -46,6 +50,16 @@ class FeedStreamApi { virtual void SetArticlesListVisible(bool is_visible) = 0; virtual bool IsArticlesListVisible() = 0; + // Invoked by RefreshTaskScheduler's scheduled task. + virtual void ExecuteRefreshTask() = 0; + + // Request to load additional content at the end of the stream. + // Calls |callback| when complete. If no content could be added, the parameter + // is false, and the caller should expect |LoadMore| to fail if called + // further. + virtual void LoadMore(SurfaceId surface, + base::OnceCallback<void(bool)> callback) = 0; + // Apply |operations| to the stream model. Does nothing if the model is not // yet loaded. virtual void ExecuteOperations( @@ -59,6 +73,50 @@ class FeedStreamApi { virtual bool CommitEphemeralChange(EphemeralChangeId id) = 0; // Rejects a change. Returns false if the change does not exist. virtual bool RejectEphemeralChange(EphemeralChangeId id) = 0; + + // User interaction reporting. These should have no side-effects other than + // reporting metrics. + + // A slice was viewed (2/3rds of it is in the viewport). Should be called + // once for each viewed slice in the stream. + virtual void ReportSliceViewed(SurfaceId surface_id, + const std::string& slice_id) = 0; + // Navigation was started in response to a link in the Feed. This event + // eventually leads to |ReportPageLoaded()| if a page is loaded successfully. + virtual void ReportNavigationStarted() = 0; + // A web page was loaded in response to opening a link from the Feed. + virtual void ReportPageLoaded() = 0; + // The user triggered the default open action, usually by tapping the card. + virtual void ReportOpenAction(const std::string& slice_id) = 0; + // The user triggered the 'open in new tab' action. + virtual void ReportOpenInNewTabAction(const std::string& slice_id) = 0; + // The user triggered the 'open in new incognito tab' action. + virtual void ReportOpenInNewIncognitoTabAction() = 0; + // The user pressed the 'send feedback' context menu option, but may have not + // completed the feedback process. + virtual void ReportSendFeedbackAction() = 0; + // The user selected the 'learn more' option on the context menu. + virtual void ReportLearnMoreAction() = 0; + // The user selected the 'download' option on the context menu. + virtual void ReportDownloadAction() = 0; + // A piece of content was removed or dismissed explicitly by the user. + virtual void ReportRemoveAction() = 0; + // The 'Not Interested In' menu item was selected. + virtual void ReportNotInterestedInAction() = 0; + // The 'Manage Interests' menu item was selected. + virtual void ReportManageInterestsAction() = 0; + // The user opened the context menu (three dot, or long press). + virtual void ReportContextMenuOpened() = 0; + // The user scrolled the feed by |distance_dp| and then stopped. + virtual void ReportStreamScrolled(int distance_dp) = 0; + + // The following methods are used for the internals page. + + virtual DebugStreamData GetDebugStreamData() = 0; + // Forces a Feed refresh from the server. + virtual void ForceRefreshForDebugging() = 0; + // Dumps some state information for debugging. + virtual std::string DumpStateForDebugging() = 0; }; } // namespace feed diff --git a/chromium/components/feed/core/v2/public/types.h b/chromium/components/feed/core/v2/public/types.h new file mode 100644 index 00000000000..0f2ec74f574 --- /dev/null +++ b/chromium/components/feed/core/v2/public/types.h @@ -0,0 +1,65 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_PUBLIC_TYPES_H_ +#define COMPONENTS_FEED_CORE_V2_PUBLIC_TYPES_H_ + +#include <string> + +#include "base/optional.h" +#include "base/strings/string_piece.h" +#include "base/time/time.h" +#include "base/util/type_safety/id_type.h" +#include "base/version.h" +#include "components/version_info/channel.h" +#include "url/gurl.h" + +namespace feed { + +// Information about the Chrome build. +struct ChromeInfo { + version_info::Channel channel{}; + base::Version version; +}; +// Device display metrics. +struct DisplayMetrics { + float density; + uint32_t width_pixels; + uint32_t height_pixels; +}; + +// A unique ID for an ephemeral change. +using EphemeralChangeId = util::IdTypeU32<class EphemeralChangeIdClass>; +using SurfaceId = util::IdTypeU32<class SurfaceIdClass>; + +struct NetworkResponseInfo { + // A union of net::Error (if the request failed) and the http + // status code(if the request succeeded in reaching the server). + int32_t status_code = 0; + base::TimeDelta fetch_duration; + base::Time fetch_time; + std::string bless_nonce; + GURL base_request_url; +}; + +// For the snippets-internals page. +struct DebugStreamData { + static const int kVersion = 1; // If a field changes, increment. + + DebugStreamData(); + ~DebugStreamData(); + DebugStreamData(const DebugStreamData&); + DebugStreamData& operator=(const DebugStreamData&); + + base::Optional<NetworkResponseInfo> fetch_info; + std::string load_stream_status; +}; + +std::string SerializeDebugStreamData(const DebugStreamData& data); +base::Optional<DebugStreamData> DeserializeDebugStreamData( + base::StringPiece base64_encoded); + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_PUBLIC_TYPES_H_ diff --git a/chromium/components/feed/core/v2/public/types_unittest.cc b/chromium/components/feed/core/v2/public/types_unittest.cc new file mode 100644 index 00000000000..e28339234b6 --- /dev/null +++ b/chromium/components/feed/core/v2/public/types_unittest.cc @@ -0,0 +1,63 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/public/types.h" + +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { +DebugStreamData MakeDebugStreamData() { + NetworkResponseInfo fetch_info; + fetch_info.status_code = 200; + fetch_info.fetch_duration = base::TimeDelta::FromSeconds(4); + fetch_info.fetch_time = + base::Time::UnixEpoch() + base::TimeDelta::FromMinutes(200); + fetch_info.bless_nonce = "nonce"; + fetch_info.base_request_url = GURL("https://www.google.com"); + + DebugStreamData data; + data.fetch_info = fetch_info; + data.load_stream_status = "loaded OK"; + return data; +} +} // namespace + +TEST(DebugStreamData, CanSerialize) { + const DebugStreamData test_data = MakeDebugStreamData(); + const auto serialized = SerializeDebugStreamData(test_data); + base::Optional<DebugStreamData> result = + DeserializeDebugStreamData(serialized); + ASSERT_TRUE(result); + + EXPECT_EQ(SerializeDebugStreamData(*result), serialized); + + ASSERT_TRUE(result->fetch_info); + EXPECT_EQ(test_data.fetch_info->status_code, result->fetch_info->status_code); + EXPECT_EQ(test_data.fetch_info->fetch_duration, + result->fetch_info->fetch_duration); + EXPECT_EQ(test_data.fetch_info->fetch_time, result->fetch_info->fetch_time); + EXPECT_EQ(test_data.fetch_info->bless_nonce, result->fetch_info->bless_nonce); + EXPECT_EQ(test_data.fetch_info->base_request_url, + result->fetch_info->base_request_url); + EXPECT_EQ(test_data.load_stream_status, result->load_stream_status); +} + +TEST(DebugStreamData, CanSerializeWithoutFetchInfo) { + DebugStreamData input = MakeDebugStreamData(); + input.fetch_info = base::nullopt; + + const auto serialized = SerializeDebugStreamData(input); + base::Optional<DebugStreamData> result = + DeserializeDebugStreamData(serialized); + ASSERT_TRUE(result); + + EXPECT_EQ(SerializeDebugStreamData(*result), serialized); +} + +TEST(DebugStreamData, FailsDeserializationGracefully) { + ASSERT_EQ(base::nullopt, DeserializeDebugStreamData({})); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/refresh_task_scheduler.h b/chromium/components/feed/core/v2/refresh_task_scheduler.h index 6f27dd57161..b12892d38c9 100644 --- a/chromium/components/feed/core/v2/refresh_task_scheduler.h +++ b/chromium/components/feed/core/v2/refresh_task_scheduler.h @@ -10,16 +10,15 @@ namespace feed { -// Schedules a repeating background task for refreshing the Feed. +// Schedules a background task for refreshing the Feed. // When the scheduled task executes, it calls FeedStream::ExecuteRefreshTask(). class RefreshTaskScheduler { public: RefreshTaskScheduler() = default; virtual ~RefreshTaskScheduler() = default; - // Schedules the task if it is not yet scheduled, or if the scheduling - // period changes. - virtual void EnsureScheduled(base::TimeDelta period) = 0; + // Schedules the task to run after |delay|. Overrides any previous schedule. + virtual void EnsureScheduled(base::TimeDelta delay) = 0; // Cancel the task if it was previously scheduled. virtual void Cancel() = 0; // After FeedStream::ExecuteRefreshTask is called, the callee must call this diff --git a/chromium/components/feed/core/v2/request_throttler.cc b/chromium/components/feed/core/v2/request_throttler.cc index 95d4d8b5edd..bfce9c98317 100644 --- a/chromium/components/feed/core/v2/request_throttler.cc +++ b/chromium/components/feed/core/v2/request_throttler.cc @@ -7,18 +7,18 @@ #include <vector> #include "base/time/clock.h" +#include "components/feed/core/v2/config.h" #include "components/feed/core/v2/prefs.h" #include "components/prefs/pref_service.h" namespace feed { namespace { int GetMaxRequestsPerDay(NetworkRequestType request_type) { - // TODO(harringtond): Decide what launchable values are. switch (request_type) { case NetworkRequestType::kFeedQuery: - return 20; + return GetFeedConfig().max_feed_query_requests_per_day; case NetworkRequestType::kUploadActions: - return 20; + return GetFeedConfig().max_action_upload_requests_per_day; } } diff --git a/chromium/components/feed/core/v2/request_throttler.h b/chromium/components/feed/core/v2/request_throttler.h index 344d600f24f..80d20cb32f6 100644 --- a/chromium/components/feed/core/v2/request_throttler.h +++ b/chromium/components/feed/core/v2/request_throttler.h @@ -10,7 +10,7 @@ class PrefService; namespace base { class Clock; -} +} // namespace base namespace feed { diff --git a/chromium/components/feed/core/v2/request_throttler_unittest.cc b/chromium/components/feed/core/v2/request_throttler_unittest.cc index 0cf0e98f3b0..4d1483ce0d5 100644 --- a/chromium/components/feed/core/v2/request_throttler_unittest.cc +++ b/chromium/components/feed/core/v2/request_throttler_unittest.cc @@ -8,6 +8,7 @@ #include "base/test/simple_test_clock.h" #include "components/feed/core/common/pref_names.h" +#include "components/feed/core/v2/config.h" #include "components/prefs/testing_pref_service.h" #include "testing/gtest/include/gtest/gtest.h" @@ -15,10 +16,17 @@ namespace feed { namespace { const int kMaximumQueryRequestsPerDay = 20; +const int kMaximumUploadActionsRequestsPerDay = 10; class FeedRequestThrottlerTest : public testing::Test { public: FeedRequestThrottlerTest() { + feed::Config config; + config.max_action_upload_requests_per_day = + kMaximumUploadActionsRequestsPerDay; + config.max_feed_query_requests_per_day = kMaximumQueryRequestsPerDay; + SetFeedConfigForTesting(config); + RegisterProfilePrefs(test_prefs_.registry()); base::Time now; @@ -40,7 +48,7 @@ TEST_F(FeedRequestThrottlerTest, RequestQuotaAllAtOnce) { } TEST_F(FeedRequestThrottlerTest, QuotaIsPerDay) { - for (int i = 0; i < kMaximumQueryRequestsPerDay; ++i) { + for (int i = 0; i < kMaximumUploadActionsRequestsPerDay; ++i) { EXPECT_TRUE(throttler_.RequestQuota(NetworkRequestType::kUploadActions)); } // Because we started at 12:01AM, we need to advance 24 hours before making diff --git a/chromium/components/feed/core/v2/scheduling.cc b/chromium/components/feed/core/v2/scheduling.cc index 21511470cf2..d65560e15d6 100644 --- a/chromium/components/feed/core/v2/scheduling.cc +++ b/chromium/components/feed/core/v2/scheduling.cc @@ -3,49 +3,90 @@ // found in the LICENSE file. #include "components/feed/core/v2/scheduling.h" + #include "base/time/time.h" +#include "base/value_conversions.h" +#include "base/values.h" +#include "components/feed/core/v2/config.h" namespace feed { +namespace { + +base::Value VectorToValue(const std::vector<base::TimeDelta>& values) { + base::Value result(base::Value::Type::LIST); + for (base::TimeDelta delta : values) { + result.Append(CreateTimeDeltaValue(delta)); + } + return result; +} + +bool ValueToVector(const base::Value& value, + std::vector<base::TimeDelta>* result) { + if (!value.is_list()) + return false; + for (const base::Value& entry : value.GetList()) { + base::TimeDelta delta; + if (!GetValueAsTimeDelta(entry, &delta)) + return false; + result->push_back(delta); + } + return true; +} +} // namespace -base::TimeDelta GetUserClassTriggerThreshold(UserClass user_class, - TriggerType trigger) { - switch (user_class) { - case UserClass::kRareSuggestionsViewer: - switch (trigger) { - case TriggerType::kNtpShown: - return base::TimeDelta::FromHours(4); - case TriggerType::kForegrounded: - return base::TimeDelta::FromHours(24); - case TriggerType::kFixedTimer: - return base::TimeDelta::FromHours(96); - } - case UserClass::kActiveSuggestionsViewer: - switch (trigger) { - case TriggerType::kNtpShown: - return base::TimeDelta::FromHours(4); - case TriggerType::kForegrounded: - return base::TimeDelta::FromHours(24); - case TriggerType::kFixedTimer: - return base::TimeDelta::FromHours(48); - } - case UserClass::kActiveSuggestionsConsumer: - switch (trigger) { - case TriggerType::kNtpShown: - return base::TimeDelta::FromHours(1); - case TriggerType::kForegrounded: - return base::TimeDelta::FromHours(12); - case TriggerType::kFixedTimer: - return base::TimeDelta::FromHours(24); - } +RequestSchedule::RequestSchedule() = default; +RequestSchedule::~RequestSchedule() = default; +RequestSchedule::RequestSchedule(const RequestSchedule&) = default; +RequestSchedule& RequestSchedule::operator=(const RequestSchedule&) = default; + +base::Value RequestScheduleToValue(const RequestSchedule& schedule) { + base::Value result(base::Value::Type::DICTIONARY); + result.SetKey("anchor", CreateTimeValue(schedule.anchor_time)); + result.SetKey("offsets", VectorToValue(schedule.refresh_offsets)); + return result; +} + +RequestSchedule RequestScheduleFromValue(const base::Value& value) { + if (!value.is_dict()) + return {}; + RequestSchedule result; + const base::Value* anchor = value.FindKey("anchor"); + const base::Value* offsets = + value.FindKeyOfType("offsets", base::Value::Type::LIST); + if (!anchor || !offsets) + return {}; + if (!GetValueAsTime(*anchor, &result.anchor_time) || + !ValueToVector(*offsets, &result.refresh_offsets)) + return {}; + return result; +} + +base::Time NextScheduledRequestTime(base::Time now, RequestSchedule* schedule) { + if (schedule->refresh_offsets.empty()) + return now + GetFeedConfig().default_background_refresh_interval; + // Attempt to detect system clock changes. If |anchor_time| is in the future, + // or too far in the past, we reset |anchor_time| to now. + if (now < schedule->anchor_time || + schedule->anchor_time + base::TimeDelta::FromDays(7) < now) { + schedule->anchor_time = now; + } + while (!schedule->refresh_offsets.empty()) { + base::Time request_time = + schedule->anchor_time + schedule->refresh_offsets[0]; + schedule->refresh_offsets.erase(schedule->refresh_offsets.begin()); + if (request_time < now) { + // The schedule time is in the past. This is most likely to happen if we + // fail to run one of our scheduled fetches. Just ignore this fetch so + // that we don't risk multiple fetches at a time. + continue; + } + return request_time; } + return now + GetFeedConfig().default_background_refresh_interval; } -bool ShouldWaitForNewContent(UserClass user_class, - bool has_content, - base::TimeDelta content_age) { - return !has_content || - content_age > GetUserClassTriggerThreshold(user_class, - TriggerType::kForegrounded); +bool ShouldWaitForNewContent(bool has_content, base::TimeDelta content_age) { + return !has_content || content_age > GetFeedConfig().stale_content_threshold; } } // namespace feed diff --git a/chromium/components/feed/core/v2/scheduling.h b/chromium/components/feed/core/v2/scheduling.h index 3361fc66c74..036180ab731 100644 --- a/chromium/components/feed/core/v2/scheduling.h +++ b/chromium/components/feed/core/v2/scheduling.h @@ -5,26 +5,39 @@ #ifndef COMPONENTS_FEED_CORE_V2_SCHEDULING_H_ #define COMPONENTS_FEED_CORE_V2_SCHEDULING_H_ +#include <vector> + #include "base/time/time.h" #include "components/feed/core/v2/enums.h" +namespace base { +class Value; +} namespace feed { constexpr base::TimeDelta kSuppressRefreshDuration = base::TimeDelta::FromMinutes(30); -// Returns a duration, T, depending on the UserClass and TriggerType. -// The following should be true: -// - At most one fetch is attempted per T. -// - Content is considered stale if time since last fetch is > T. We'll prefer -// to refresh stale content before showing it. -// - For TriggerType::kFixedTimer, T is the time between scheduled fetches. -base::TimeDelta GetUserClassTriggerThreshold(UserClass user_class, - TriggerType trigger); +// A schedule for making Feed refresh requests. +// |anchor_time| + |refresh_offsets[i]| is the time each fetch should be made. +struct RequestSchedule { + RequestSchedule(); + ~RequestSchedule(); + RequestSchedule(const RequestSchedule&); + RequestSchedule& operator=(const RequestSchedule&); + + base::Time anchor_time; + std::vector<base::TimeDelta> refresh_offsets; +}; + +base::Value RequestScheduleToValue(const RequestSchedule&); +RequestSchedule RequestScheduleFromValue(const base::Value&); +// Given a schedule, returns the next time a request should be made. +// Updates |schedule| accordingly. If |schedule| has no fetches remaining, +// returns a scheduled time using |Config::default_background_refresh_interval|. +base::Time NextScheduledRequestTime(base::Time now, RequestSchedule* schedule); // Returns whether we should wait for new content before showing stream content. -bool ShouldWaitForNewContent(UserClass user_class, - bool has_content, - base::TimeDelta content_age); +bool ShouldWaitForNewContent(bool has_content, base::TimeDelta content_age); } // namespace feed #endif // COMPONENTS_FEED_CORE_V2_SCHEDULING_H_ diff --git a/chromium/components/feed/core/v2/scheduling_unittest.cc b/chromium/components/feed/core/v2/scheduling_unittest.cc new file mode 100644 index 00000000000..a3669b4018b --- /dev/null +++ b/chromium/components/feed/core/v2/scheduling_unittest.cc @@ -0,0 +1,126 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +#include "components/feed/core/v2/scheduling.h" + +#include "base/check.h" +#include "base/json/json_writer.h" +#include "base/strings/string_util.h" +#include "base/values.h" +#include "components/feed/core/v2/config.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { +using base::TimeDelta; + +const base::Time kAnchorTime = + base::Time::UnixEpoch() + TimeDelta::FromHours(6); +const base::TimeDelta kDefaultScheduleInterval = base::TimeDelta::FromHours(24); + +std::string ToJSON(const base::Value& value) { + std::string json; + CHECK(base::JSONWriter::WriteWithOptions( + value, base::JSONWriter::OPTIONS_PRETTY_PRINT, &json)); + // Don't use \r\n on windows. + base::RemoveChars(json, "\r", &json); + return json; +} + +TEST(RequestSchedule, CanSerialize) { + RequestSchedule schedule; + schedule.anchor_time = kAnchorTime; + schedule.refresh_offsets = {TimeDelta::FromHours(1), TimeDelta::FromHours(6)}; + + const base::Value schedule_value = RequestScheduleToValue(schedule); + ASSERT_EQ(R"({ + "anchor": "11644495200000000", + "offsets": [ "3600000000", "21600000000" ] +} +)", + ToJSON(schedule_value)); + + RequestSchedule deserialized_schedule = + RequestScheduleFromValue(schedule_value); + EXPECT_EQ(schedule.anchor_time, deserialized_schedule.anchor_time); + EXPECT_EQ(schedule.refresh_offsets, deserialized_schedule.refresh_offsets); +} + +class NextScheduledRequestTimeTest : public testing::Test { + public: + void SetUp() override { + Config config = GetFeedConfig(); + config.default_background_refresh_interval = kDefaultScheduleInterval; + SetFeedConfigForTesting(config); + } +}; + +TEST_F(NextScheduledRequestTimeTest, NormalUsage) { + RequestSchedule schedule; + schedule.anchor_time = kAnchorTime; + schedule.refresh_offsets = {TimeDelta::FromHours(1), TimeDelta::FromHours(6)}; + + // |kNow| is in the normal range [kAnchorTime, kAnchorTime+1hr) + base::Time kNow = kAnchorTime + TimeDelta::FromMinutes(12); + EXPECT_EQ(kAnchorTime + TimeDelta::FromHours(1), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kAnchorTime + TimeDelta::FromHours(6), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kNow + kDefaultScheduleInterval, + NextScheduledRequestTime(kNow, &schedule)); +} + +TEST_F(NextScheduledRequestTimeTest, NowPastRequestTimeSkipsRequest) { + RequestSchedule schedule; + schedule.anchor_time = kAnchorTime; + schedule.refresh_offsets = {TimeDelta::FromHours(1), TimeDelta::FromHours(6)}; + + base::Time kNow = kAnchorTime + TimeDelta::FromMinutes(61); + EXPECT_EQ(kAnchorTime + TimeDelta::FromHours(6), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kNow + kDefaultScheduleInterval, + NextScheduledRequestTime(kNow, &schedule)); +} + +TEST_F(NextScheduledRequestTimeTest, NowPastAllRequestTimes) { + RequestSchedule schedule; + schedule.anchor_time = kAnchorTime; + schedule.refresh_offsets = {TimeDelta::FromHours(1), TimeDelta::FromHours(6)}; + + base::Time kNow = kAnchorTime + TimeDelta::FromHours(7); + EXPECT_EQ(kNow + kDefaultScheduleInterval, + NextScheduledRequestTime(kNow, &schedule)); +} + +TEST_F(NextScheduledRequestTimeTest, NowInPast) { + RequestSchedule schedule; + schedule.anchor_time = kAnchorTime; + schedule.refresh_offsets = {TimeDelta::FromHours(1), TimeDelta::FromHours(6)}; + + // Since |kNow| is in the past, deltas are recomputed using |kNow|. + base::Time kNow = kAnchorTime - TimeDelta::FromMinutes(12); + EXPECT_EQ(kNow + TimeDelta::FromHours(1), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kNow + TimeDelta::FromHours(6), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kNow + kDefaultScheduleInterval, + NextScheduledRequestTime(kNow, &schedule)); +} + +TEST_F(NextScheduledRequestTimeTest, NowInFarFuture) { + RequestSchedule schedule; + schedule.anchor_time = kAnchorTime; + schedule.refresh_offsets = {TimeDelta::FromHours(1), TimeDelta::FromHours(6)}; + + // Since |kNow| is in the far future, deltas are recomputed using |kNow|. + base::Time kNow = kAnchorTime + TimeDelta::FromDays(12); + EXPECT_EQ(kNow + TimeDelta::FromHours(1), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kNow + TimeDelta::FromHours(6), + NextScheduledRequestTime(kNow, &schedule)); + EXPECT_EQ(kNow + kDefaultScheduleInterval, + NextScheduledRequestTime(kNow, &schedule)); +} + +} // namespace +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_event_metrics.cc b/chromium/components/feed/core/v2/stream_event_metrics.cc deleted file mode 100644 index ffce8fa1198..00000000000 --- a/chromium/components/feed/core/v2/stream_event_metrics.cc +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -#include "components/feed/core/v2/stream_event_metrics.h" - -#include "base/metrics/histogram_macros.h" - -namespace feed { - -void StreamEventMetrics::OnLoadStream(LoadStreamStatus load_from_store_status, - LoadStreamStatus final_status) { - // TODO(harringtond): Add UMA for this, or record it with another histogram. -} - -void StreamEventMetrics::OnMaybeTriggerRefresh(TriggerType trigger, - bool clear_all_before_refresh) { - // TODO(harringtond): Either add UMA for this or remove it. -} - -void StreamEventMetrics::OnClearAll(base::TimeDelta time_since_last_clear) { - UMA_HISTOGRAM_CUSTOM_TIMES( - "ContentSuggestions.Feed.Scheduler.TimeSinceLastFetchOnClear", - time_since_last_clear, base::TimeDelta::FromSeconds(1), - base::TimeDelta::FromDays(7), - /*bucket_count=*/50); -} - -} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_event_metrics.h b/chromium/components/feed/core/v2/stream_event_metrics.h deleted file mode 100644 index 3940c2f428c..00000000000 --- a/chromium/components/feed/core/v2/stream_event_metrics.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_FEED_CORE_V2_STREAM_EVENT_METRICS_H_ -#define COMPONENTS_FEED_CORE_V2_STREAM_EVENT_METRICS_H_ - -#include "components/feed/core/v2/enums.h" -#include "components/feed/core/v2/feed_stream.h" - -namespace feed { - -// Reports UMA metrics for stream events. -class StreamEventMetrics : public FeedStream::EventObserver { - public: - void OnLoadStream(LoadStreamStatus load_from_store_status, - LoadStreamStatus final_status) override; - void OnMaybeTriggerRefresh(TriggerType trigger, - bool clear_all_before_refresh) override; - void OnClearAll(base::TimeDelta time_since_last_clear) override; -}; -} // namespace feed - -#endif // COMPONENTS_FEED_CORE_V2_STREAM_EVENT_METRICS_H_ diff --git a/chromium/components/feed/core/v2/stream_model.cc b/chromium/components/feed/core/v2/stream_model.cc index 74381e4516e..eba0ab62f6c 100644 --- a/chromium/components/feed/core/v2/stream_model.cc +++ b/chromium/components/feed/core/v2/stream_model.cc @@ -5,17 +5,22 @@ #include "components/feed/core/v2/stream_model.h" #include <algorithm> +#include <sstream> #include <utility> -#include "base/logging.h" +#include "base/check.h" +#include "base/json/string_escape.h" #include "base/strings/strcat.h" #include "base/strings/string_number_conversions.h" #include "components/feed/core/proto/v2/store.pb.h" #include "components/feed/core/proto/v2/wire/content_id.pb.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/protocol_translator.h" namespace feed { namespace { +using UiUpdate = StreamModel::UiUpdate; +using StoreUpdate = StreamModel::StoreUpdate; + bool HasClearAll(const std::vector<feedstore::StreamStructure>& structures) { for (const feedstore::StreamStructure& data : structures) { if (data.operation() == feedstore::StreamStructure::CLEAR_ALL) @@ -24,19 +29,15 @@ bool HasClearAll(const std::vector<feedstore::StreamStructure>& structures) { return false; } } // namespace -StreamModel::UiUpdate::UiUpdate() = default; -StreamModel::UiUpdate::~UiUpdate() = default; -StreamModel::UiUpdate::UiUpdate(const UiUpdate&) = default; -StreamModel::UiUpdate& StreamModel::UiUpdate::operator=(const UiUpdate&) = - default; -StreamModel::StoreUpdate::StoreUpdate() = default; -StreamModel::StoreUpdate::~StoreUpdate() = default; -StreamModel::StoreUpdate::StoreUpdate(const StoreUpdate&) = default; -StreamModel::StoreUpdate& StreamModel::StoreUpdate::operator=( - const StoreUpdate&) = default; -StreamModel::StoreUpdate::StoreUpdate(StoreUpdate&&) = default; -StreamModel::StoreUpdate& StreamModel::StoreUpdate::operator=(StoreUpdate&&) = - default; + +UiUpdate::UiUpdate() = default; +UiUpdate::~UiUpdate() = default; +UiUpdate::UiUpdate(const UiUpdate&) = default; +UiUpdate& UiUpdate::operator=(const UiUpdate&) = default; +StoreUpdate::StoreUpdate() = default; +StoreUpdate::~StoreUpdate() = default; +StoreUpdate::StoreUpdate(StoreUpdate&&) = default; +StoreUpdate& StoreUpdate::operator=(StoreUpdate&&) = default; StreamModel::StreamModel() = default; StreamModel::~StreamModel() = default; @@ -76,10 +77,51 @@ std::vector<std::string> StreamModel::GetSharedStateIds() const { void StreamModel::Update( std::unique_ptr<StreamModelUpdateRequest> update_request) { - feedstore::StreamData& stream_data = update_request->stream_data; std::vector<feedstore::StreamStructure>& stream_structures = update_request->stream_structures; - if (HasClearAll(stream_structures)) { + const bool has_clear_all = HasClearAll(stream_structures); + + switch (update_request->source) { + case StreamModelUpdateRequest::Source::kNetworkUpdate: + // In this case, the stream state has been saved to persistent + // storage by the caller. Next sequence number is always 1. + next_structure_sequence_number_ = 1; + break; + case StreamModelUpdateRequest::Source::kInitialLoadFromStore: + // In this case, use max_structure_sequence_number to derive the next + // sequence number. + next_structure_sequence_number_ = + update_request->max_structure_sequence_number + 1; + break; + case StreamModelUpdateRequest::Source::kNetworkLoadMore: { + // In this case, |StreamModel| is responsible for triggering the update + // to the store. There are two main cases: + // 1. The update request has a CLEAR_ALL (this is unexpected). + // In this case, we want to overwrite all stored stream data, since + // the old data is no longer useful. Start using sequence number 0. + // 2. The update request does not have a CLEAR_ALL. + // Save the new stream data with the next sequence number. + if (has_clear_all) { + next_structure_sequence_number_ = 0; + } + // Note: We might be overwriting some shared-states unnecessarily. + StoreUpdate store_update; + store_update.overwrite_stream_data = has_clear_all; + store_update.update_request = + std::make_unique<StreamModelUpdateRequest>(*update_request); + store_update.sequence_number = next_structure_sequence_number_++; + store_observer_->OnStoreChange(std::move(store_update)); + break; + } + } + + // Update non-tree data. + // TODO(harringtond): Once we start using StreamData.next_action_id, this line + // would be problematic. We should probably move next_action_id into a + // different record in FeedStore. + stream_data_ = update_request->stream_data; + + if (has_clear_all) { shared_states_.clear(); } @@ -91,13 +133,6 @@ void StreamModel::Update( base_feature_tree_.AddContent(std::move(content)); } - // Update non-tree data. - next_page_token_ = stream_data.next_page_token(); - last_added_time_ = - base::Time::UnixEpoch() + - base::TimeDelta::FromMilliseconds(stream_data.last_added_time_millis()); - consistency_token_ = stream_data.consistency_token(); - for (feedstore::StreamSharedState& shared_state : update_request->shared_states) { std::string id = ContentIdString(shared_state.content_id()); @@ -107,13 +142,6 @@ void StreamModel::Update( } } - // Set next_structure_sequence_number_ when doing the initial load. - if (update_request->source == - StreamModelUpdateRequest::Source::kInitialLoadFromStore) { - next_structure_sequence_number_ = - update_request->max_structure_sequence_number + 1; - } - // TODO(harringtond): Some StreamData fields not yet used. // next_action_id - do we need to load the model before uploading // actions? If not, we probably will want to move this out of @@ -213,13 +241,18 @@ const stream_model::FeatureTree* StreamModel::GetFinalFeatureTree() const { return const_cast<StreamModel*>(this)->GetFinalFeatureTree(); } +const std::string& StreamModel::GetNextPageToken() const { + return stream_data_.next_page_token(); +} + std::string StreamModel::DumpStateForTesting() { std::stringstream ss; ss << "StreamModel{\n"; - ss << "next_page_token='" << next_page_token_ << "'\n"; - ss << "consistency_token='" << consistency_token_ << "'\n"; + ss << "next_page_token='" << GetNextPageToken() << "'\n"; for (auto& entry : shared_states_) { - ss << "shared_state[" << entry.first << "]='" << entry.second.data << "'\n"; + ss << "shared_state[" << entry.first + << "]=" << base::GetQuotedJSONString(entry.second.data.substr(0, 100)) + << "\n"; } ss << GetFinalFeatureTree()->DumpStateForTesting(); ss << "}StreamModel\n"; diff --git a/chromium/components/feed/core/v2/stream_model.h b/chromium/components/feed/core/v2/stream_model.h index 2595ba228d3..f3e460b7966 100644 --- a/chromium/components/feed/core/v2/stream_model.h +++ b/chromium/components/feed/core/v2/stream_model.h @@ -49,13 +49,17 @@ class StreamModel { struct StoreUpdate { StoreUpdate(); ~StoreUpdate(); - StoreUpdate(const StoreUpdate&); StoreUpdate(StoreUpdate&&); - StoreUpdate& operator=(const StoreUpdate&); StoreUpdate& operator=(StoreUpdate&&); + // Sequence number to use when writing to the store. int32_t sequence_number = 0; + // Whether the |update_request| should overwrite all stream data. + bool overwrite_stream_data = false; + // Data to write. Either a list of operations or a + // |StreamModelUpdateRequest|. std::vector<feedstore::DataOperation> operations; + std::unique_ptr<StreamModelUpdateRequest> update_request; }; class Observer { @@ -64,11 +68,12 @@ class StreamModel { // Called when the UI model changes. virtual void OnUiUpdate(const UiUpdate& update) = 0; }; + class StoreObserver { public: // Called when the peristent store should be modified to reflect a model // change. - virtual void OnStoreChange(const StoreUpdate& update) = 0; + virtual void OnStoreChange(StoreUpdate update) = 0; }; StreamModel(); @@ -109,6 +114,8 @@ class StreamModel { // Rejects a change. Returns false if the change does not exist. bool RejectEphemeralChange(EphemeralChangeId id); + const std::string& GetNextPageToken() const; + // Outputs a string representing the model state for debugging or testing. std::string DumpStateForTesting(); @@ -137,9 +144,7 @@ class StreamModel { // The following data is associated with the stream, but lives outside of the // tree. - std::string next_page_token_; // TODO(harringtond): use this value. - std::string consistency_token_; // TODO(harringtond): use this value. - base::Time last_added_time_; // TODO(harringtond): use this value. + feedstore::StreamData stream_data_; base::flat_map<std::string, SharedState> shared_states_; int32_t next_structure_sequence_number_ = 0; diff --git a/chromium/components/feed/core/v2/stream_model/ephemeral_change.h b/chromium/components/feed/core/v2/stream_model/ephemeral_change.h index fe293b3cb7b..d34e43fe90c 100644 --- a/chromium/components/feed/core/v2/stream_model/ephemeral_change.h +++ b/chromium/components/feed/core/v2/stream_model/ephemeral_change.h @@ -8,8 +8,8 @@ #include <memory> #include <vector> #include "components/feed/core/proto/v2/store.pb.h" -#include "components/feed/core/v2/public/feed_stream_api.h" #include "components/feed/core/v2/stream_model/feature_tree.h" +#include "components/feed/core/v2/types.h" namespace feed { namespace stream_model { diff --git a/chromium/components/feed/core/v2/stream_model/feature_tree.cc b/chromium/components/feed/core/v2/stream_model/feature_tree.cc index 5c8373a77e3..4e6fef564b7 100644 --- a/chromium/components/feed/core/v2/stream_model/feature_tree.cc +++ b/chromium/components/feed/core/v2/stream_model/feature_tree.cc @@ -7,7 +7,7 @@ #include <algorithm> #include <sstream> -#include "base/logging.h" +#include "base/check.h" namespace feed { namespace stream_model { diff --git a/chromium/components/feed/core/v2/stream_model/feature_tree.h b/chromium/components/feed/core/v2/stream_model/feature_tree.h index 2aa79e8bd6c..5f5dee45c71 100644 --- a/chromium/components/feed/core/v2/stream_model/feature_tree.h +++ b/chromium/components/feed/core/v2/stream_model/feature_tree.h @@ -13,7 +13,7 @@ #include "base/util/type_safety/id_type.h" #include "components/feed/core/proto/v2/store.pb.h" #include "components/feed/core/v2/proto_util.h" -#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/types.h" namespace feed { namespace stream_model { diff --git a/chromium/components/feed/core/v2/stream_model_unittest.cc b/chromium/components/feed/core/v2/stream_model_unittest.cc index d3a0b4c2825..6c843db5dda 100644 --- a/chromium/components/feed/core/v2/stream_model_unittest.cc +++ b/chromium/components/feed/core/v2/stream_model_unittest.cc @@ -13,14 +13,14 @@ #include "base/strings/string_number_conversions.h" #include "components/feed/core/proto/v2/store.pb.h" #include "components/feed/core/proto/v2/wire/content_id.pb.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/test/stream_builder.h" #include "testing/gtest/include/gtest/gtest.h" namespace feed { namespace { -using StoreUpdate = StreamModel::StoreUpdate; using UiUpdate = StreamModel::UiUpdate; +using StoreUpdate = StreamModel::StoreUpdate; std::vector<std::string> GetContentFrames(const StreamModel& model) { std::vector<std::string> frames; @@ -59,7 +59,9 @@ class TestStoreObserver : public StreamModel::StoreObserver { } // StreamModel::StoreObserver. - void OnStoreChange(const StoreUpdate& records) override { update_ = records; } + void OnStoreChange(StoreUpdate records) override { + update_ = std::move(records); + } const base::Optional<StoreUpdate>& GetUpdate() const { return update_; } @@ -303,7 +305,7 @@ TEST(StreamModelTest, CommitEphemeralChange) { // Check that the observer's |OnStoreChange()| was called. ASSERT_TRUE(store_observer.GetUpdate()); - StoreUpdate store_update = *store_observer.GetUpdate(); + const StoreUpdate& store_update = *store_observer.GetUpdate(); ASSERT_EQ(3UL, store_update.operations.size()); EXPECT_EQ(feedstore::StreamStructure::CLUSTER, store_update.operations[0].structure().type()); diff --git a/chromium/components/feed/core/v2/stream_model_update_request_unittest.cc b/chromium/components/feed/core/v2/stream_model_update_request_unittest.cc deleted file mode 100644 index 1e70748d99a..00000000000 --- a/chromium/components/feed/core/v2/stream_model_update_request_unittest.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/feed/core/v2/stream_model_update_request.h" - -#include <string> - -#include "base/base_paths.h" -#include "base/files/file_path.h" -#include "base/files/file_util.h" -#include "base/path_service.h" -#include "base/strings/string_number_conversions.h" -#include "base/time/time.h" -#include "components/feed/core/proto/v2/wire/feed_response.pb.h" -#include "components/feed/core/proto/v2/wire/response.pb.h" -#include "components/feed/core/v2/proto_util.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace feed { -namespace { - -const char kResponsePbPath[] = "components/test/data/feed/response.binarypb"; -constexpr base::TimeDelta kResponseTime = base::TimeDelta::FromSeconds(42); -const base::Time kCurrentTime = - base::Time::UnixEpoch() + base::TimeDelta::FromDays(123); -// TODO(iwells): Replace response.binarypb with a response that uses the new -// wire protocol. -// -// Since we're currently using a Jardin response which includes a -// Piet shared state, and translation skips handling Piet shared states, we -// expect to have only 33 StreamStructures even though there are 34 wire -// operations. -const int kExpectedStreamStructureCount = 33; -const size_t kExpectedContentCount = 10; -const size_t kExpectedSharedStateCount = 0; - -std::string ContentIdToString(const feedwire::ContentId& content_id) { - return "{content_domain: \"" + content_id.content_domain() + - "\", id: " + base::NumberToString(content_id.id()) + ", type: \"" + - feedwire::ContentId::Type_Name(content_id.type()) + "\"}"; -} - -feedwire::Response TestWireResponse() { - // Read and parse response.binarypb. - base::FilePath response_file_path; - CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &response_file_path)); - response_file_path = response_file_path.AppendASCII(kResponsePbPath); - - CHECK(base::PathExists(response_file_path)); - - std::string response_data; - CHECK(base::ReadFileToString(response_file_path, &response_data)); - - feedwire::Response response; - CHECK(response.ParseFromString(response_data)); - return response; -} - -} // namespace - -// TODO(iwells): Test failure cases once the new protos are ready. - -TEST(StreamModelUpdateRequestTest, TranslateRealResponse) { - // Tests how proto translation works on a real response from the server. - // - // The response will periodically need to be updated as changes are made to - // the server. Update testdata/response.textproto and then run - // tools/generate_test_response_binarypb.sh. - - feedwire::Response response = TestWireResponse(); - feedwire::FeedResponse feed_response = response.feed_response(); - - // TODO(iwells): Make these exactly equal once we aren't using an old - // response. - ASSERT_EQ(feed_response.data_operation_size(), - kExpectedStreamStructureCount + 1); - - std::unique_ptr<StreamModelUpdateRequest> translated = - TranslateWireResponse(response, kResponseTime, kCurrentTime); - - ASSERT_TRUE(translated); - EXPECT_EQ(kCurrentTime, feedstore::GetLastAddedTime(translated->stream_data)); - ASSERT_EQ(translated->stream_structures.size(), - static_cast<size_t>(kExpectedStreamStructureCount)); - - const std::vector<feedstore::StreamStructure>& structures = - translated->stream_structures; - - // Check CLEAR_ALL: - EXPECT_EQ(structures[0].operation(), feedstore::StreamStructure::CLEAR_ALL); - - // TODO(iwells): Check the shared state once we have a new - - // Check UPDATE_OR_APPEND for the root: - EXPECT_EQ(structures[1].operation(), - feedstore::StreamStructure::UPDATE_OR_APPEND); - EXPECT_EQ(structures[1].type(), feedstore::StreamStructure::STREAM); - EXPECT_TRUE(structures[1].has_content_id()); - EXPECT_FALSE(structures[1].has_parent_id()); - - feedwire::ContentId root_content_id = structures[1].content_id(); - - // Content: - EXPECT_EQ(structures[2].operation(), - feedstore::StreamStructure::UPDATE_OR_APPEND); - EXPECT_EQ(structures[2].type(), feedstore::StreamStructure::CONTENT); - EXPECT_TRUE(structures[2].has_content_id()); - EXPECT_TRUE(structures[2].has_parent_id()); - - // TODO(iwells): Uncomment when these are available. - // EXPECT_TRUE(structures[3].has_content_info()); - // EXPECT_NE(structures[3].content_info().score(), 0.); - // EXPECT_NE(structures[3].content_info().availability_time_seconds(), 0); - // EXPECT_TRUE(structures[3].content_info().has_representation_data()); - // EXPECT_TRUE(structures[3].content_info().has_offline_metadata()); - - ASSERT_GT(translated->content.size(), 0UL); - EXPECT_EQ(ContentIdToString(translated->content[0].content_id()), - ContentIdToString(structures[2].content_id())); - // TODO(iwells): Check content.frame() once this is available. - - // Non-content structures: - EXPECT_EQ(structures[3].operation(), - feedstore::StreamStructure::UPDATE_OR_APPEND); - // TODO(iwells): This is a CARD. Remove once we have a new response. - EXPECT_EQ(structures[3].type(), feedstore::StreamStructure::UNKNOWN_TYPE); - EXPECT_TRUE(structures[3].has_content_id()); - EXPECT_TRUE(structures[3].has_parent_id()); - - EXPECT_EQ(structures[4].operation(), - feedstore::StreamStructure::UPDATE_OR_APPEND); - EXPECT_EQ(structures[4].type(), feedstore::StreamStructure::CLUSTER); - EXPECT_TRUE(structures[4].has_content_id()); - EXPECT_TRUE(structures[4].has_parent_id()); - EXPECT_EQ(ContentIdToString(structures[4].parent_id()), - ContentIdToString(root_content_id)); - - // The other members: - EXPECT_EQ(translated->content.size(), kExpectedContentCount); - EXPECT_EQ(translated->shared_states.size(), kExpectedSharedStateCount); - - EXPECT_EQ(translated->response_time, kResponseTime); -} - -} // namespace feed diff --git a/chromium/components/feed/core/v2/surface_updater.cc b/chromium/components/feed/core/v2/surface_updater.cc new file mode 100644 index 00000000000..2ac92f10aa2 --- /dev/null +++ b/chromium/components/feed/core/v2/surface_updater.cc @@ -0,0 +1,294 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/surface_updater.h" + +#include <tuple> + +#include "base/check.h" +#include "base/strings/string_number_conversions.h" +#include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/metrics_reporter.h" + +namespace feed { +namespace { + +using DrawState = SurfaceUpdater::DrawState; +using SurfaceInterface = FeedStreamApi::SurfaceInterface; + +// Give each kind of zero state a unique name, so that the UI knows if it +// changes. +const char* GetZeroStateSliceId(feedui::ZeroStateSlice::Type type) { + switch (type) { + case feedui::ZeroStateSlice::NO_CARDS_AVAILABLE: + return "no-cards"; + case feedui::ZeroStateSlice::CANT_REFRESH: // fall-through + default: + return "cant-refresh"; + } +} + +void AddSharedState(const StreamModel& model, + const std::string& shared_state_id, + feedui::StreamUpdate* stream_update) { + const std::string* shared_state_data = + model.FindSharedStateData(shared_state_id); + DCHECK(shared_state_data); + feedui::SharedState* added_shared_state = + stream_update->add_new_shared_states(); + added_shared_state->set_id(shared_state_id); + added_shared_state->set_xsurface_shared_state(*shared_state_data); +} + +void AddSliceUpdate(const StreamModel& model, + ContentRevision content_revision, + bool is_content_new, + feedui::StreamUpdate* stream_update) { + if (is_content_new) { + feedui::Slice* slice = stream_update->add_updated_slices()->mutable_slice(); + slice->set_slice_id(ToString(content_revision)); + const feedstore::Content* content = model.FindContent(content_revision); + DCHECK(content); + slice->mutable_xsurface_slice()->set_xsurface_frame(content->frame()); + } else { + stream_update->add_updated_slices()->set_slice_id( + ToString(content_revision)); + } +} + +void AddLoadingSpinner(bool is_at_top, feedui::StreamUpdate* update) { + feedui::Slice* slice = update->add_updated_slices()->mutable_slice(); + slice->mutable_loading_spinner_slice()->set_is_at_top(is_at_top); + slice->set_slice_id(is_at_top ? "loading-spinner" : "load-more-spinner"); +} + +feedui::StreamUpdate MakeStreamUpdate( + const std::vector<std::string>& updated_shared_state_ids, + const base::flat_set<ContentRevision>& already_sent_content, + const StreamModel* model, + const DrawState& state) { + DCHECK(!state.loading_initial || !state.loading_more) + << "logic bug: requested both top and bottom spinners."; + feedui::StreamUpdate stream_update; + // Add content from the model, if it's loaded. + bool has_content = false; + if (model) { + for (ContentRevision content_revision : model->GetContentList()) { + const bool is_updated = already_sent_content.count(content_revision) == 0; + AddSliceUpdate(*model, content_revision, is_updated, &stream_update); + has_content = true; + } + for (const std::string& name : updated_shared_state_ids) { + AddSharedState(*model, name, &stream_update); + } + } + + feedui::ZeroStateSlice::Type zero_state_type = state.zero_state_type; + // If there are no cards, and we aren't loading, force a zero-state. + // This happens when a model is loaded, but it has no content. + if (!state.loading_initial && !has_content && + state.zero_state_type == feedui::ZeroStateSlice::UNKNOWN) { + zero_state_type = feedui::ZeroStateSlice::NO_CARDS_AVAILABLE; + } + + if (zero_state_type != feedui::ZeroStateSlice::UNKNOWN) { + feedui::Slice* slice = stream_update.add_updated_slices()->mutable_slice(); + slice->mutable_zero_state_slice()->set_type(zero_state_type); + slice->set_slice_id(GetZeroStateSliceId(zero_state_type)); + } else { + // Add the initial-load spinner if applicable. + if (state.loading_initial) { + AddLoadingSpinner(/*is_at_top=*/true, &stream_update); + } + // Add a loading-more spinner if applicable. + if (state.loading_more) { + AddLoadingSpinner(/*is_at_top=*/false, &stream_update); + } + } + + return stream_update; +} + +feedui::StreamUpdate GetUpdateForNewSurface(const DrawState& state, + const StreamModel* model) { + std::vector<std::string> updated_shared_state_ids; + if (model) { + updated_shared_state_ids = model->GetSharedStateIds(); + } + return MakeStreamUpdate(std::move(updated_shared_state_ids), + /*already_sent_content=*/{}, model, state); +} + +base::flat_set<ContentRevision> GetContentSet(const StreamModel* model) { + if (!model) + return {}; + const std::vector<ContentRevision>& content_list = model->GetContentList(); + return base::flat_set<ContentRevision>(content_list.begin(), + content_list.end()); +} + +feedui::ZeroStateSlice::Type GetZeroStateType(LoadStreamStatus status) { + switch (status) { + case LoadStreamStatus::kNoResponseBody: + case LoadStreamStatus::kProtoTranslationFailed: + case LoadStreamStatus::kCannotLoadFromNetworkOffline: + case LoadStreamStatus::kCannotLoadFromNetworkThrottled: + return feedui::ZeroStateSlice::CANT_REFRESH; + case LoadStreamStatus::kNoStatus: + case LoadStreamStatus::kLoadedFromStore: + case LoadStreamStatus::kLoadedFromNetwork: + case LoadStreamStatus::kFailedWithStoreError: + case LoadStreamStatus::kNoStreamDataInStore: + case LoadStreamStatus::kModelAlreadyLoaded: + case LoadStreamStatus::kDataInStoreIsStale: + case LoadStreamStatus::kDataInStoreIsStaleTimestampInFuture: + case LoadStreamStatus::kCannotLoadFromNetworkSupressedForHistoryDelete: + case LoadStreamStatus::kLoadNotAllowedEulaNotAccepted: + case LoadStreamStatus::kLoadNotAllowedArticlesListHidden: + case LoadStreamStatus::kCannotParseNetworkResponseBody: + case LoadStreamStatus::kLoadMoreModelIsNotLoaded: + break; + } + return feedui::ZeroStateSlice::NO_CARDS_AVAILABLE; +} + +} // namespace + +bool SurfaceUpdater::DrawState::operator==(const DrawState& rhs) const { + return std::tie(loading_more, loading_initial, zero_state_type) == + std::tie(rhs.loading_more, rhs.loading_initial, rhs.zero_state_type); +} + +SurfaceUpdater::SurfaceUpdater(MetricsReporter* metrics_reporter) + : metrics_reporter_(metrics_reporter) {} +SurfaceUpdater::~SurfaceUpdater() = default; + +void SurfaceUpdater::SetModel(StreamModel* model) { + if (model_ == model) + return; + if (model_) + model_->SetObserver(nullptr); + model_ = model; + sent_content_.clear(); + if (model_) { + model_->SetObserver(this); + loading_initial_ = loading_initial_ && model_->GetContentList().empty(); + loading_more_ = false; + SendStreamUpdate(model_->GetSharedStateIds()); + last_draw_state_ = GetState(); + } +} + +void SurfaceUpdater::OnUiUpdate(const StreamModel::UiUpdate& update) { + DCHECK(model_); // The update comes from the model. + loading_initial_ = loading_initial_ && model_->GetContentList().empty(); + loading_more_ = loading_more_ && !update.content_list_changed; + + std::vector<std::string> updated_shared_state_ids; + for (const StreamModel::UiUpdate::SharedStateInfo& info : + update.shared_states) { + if (info.updated) + updated_shared_state_ids.push_back(info.shared_state_id); + } + + SendStreamUpdate(updated_shared_state_ids); +} + +void SurfaceUpdater::SurfaceAdded(SurfaceInterface* surface) { + SendUpdateToSurface(surface, GetUpdateForNewSurface(GetState(), model_)); + surfaces_.AddObserver(surface); +} + +void SurfaceUpdater::SurfaceRemoved(SurfaceInterface* surface) { + surfaces_.RemoveObserver(surface); +} + +void SurfaceUpdater::LoadStreamStarted() { + load_stream_failed_ = false; + loading_initial_ = true; + SendStreamUpdateIfNeeded(); +} + +void SurfaceUpdater::LoadStreamComplete(bool success, + LoadStreamStatus load_stream_status) { + loading_initial_ = false; + load_stream_status_ = load_stream_status; + load_stream_failed_ = !success; + SendStreamUpdateIfNeeded(); +} + +int SurfaceUpdater::GetSliceIndexFromSliceId(const std::string& slice_id) { + ContentRevision slice_rev = ToContentRevision(slice_id); + if (slice_rev.is_null()) + return -1; + int index = 0; + for (const ContentRevision& rev : model_->GetContentList()) { + if (rev == slice_rev) + return index; + ++index; + } + return -1; +} + +bool SurfaceUpdater::HasSurfaceAttached() const { + return surfaces_.might_have_observers(); +} + +void SurfaceUpdater::SetLoadingMore(bool is_loading) { + DCHECK(!loading_initial_) + << "SetLoadingMore while still loading the initial state"; + loading_more_ = is_loading; + SendStreamUpdateIfNeeded(); +} + +DrawState SurfaceUpdater::GetState() const { + DrawState new_state; + new_state.loading_more = loading_more_; + new_state.loading_initial = loading_initial_; + if (load_stream_failed_) + new_state.zero_state_type = GetZeroStateType(load_stream_status_); + return new_state; +} + +void SurfaceUpdater::SendStreamUpdateIfNeeded() { + if (last_draw_state_ == GetState()) { + return; + } + SendStreamUpdate({}); +} + +void SurfaceUpdater::SendStreamUpdate( + const std::vector<std::string>& updated_shared_state_ids) { + DrawState state = GetState(); + + feedui::StreamUpdate stream_update = + MakeStreamUpdate(updated_shared_state_ids, sent_content_, model_, state); + + for (SurfaceInterface& surface : surfaces_) { + SendUpdateToSurface(&surface, stream_update); + } + + sent_content_ = GetContentSet(model_); + last_draw_state_ = state; +} + +void SurfaceUpdater::SendUpdateToSurface(SurfaceInterface* surface, + const feedui::StreamUpdate& update) { + surface->StreamUpdate(update); + + // Call |MetricsReporter::SurfaceReceivedContent()| if appropriate. + + bool update_has_content = false; + for (const feedui::StreamUpdate_SliceUpdate& slice_update : + update.updated_slices()) { + if (slice_update.has_slice() && slice_update.slice().has_xsurface_slice()) { + update_has_content = true; + } + } + if (!update_has_content) + return; + metrics_reporter_->SurfaceReceivedContent(surface->GetSurfaceId()); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/surface_updater.h b/chromium/components/feed/core/v2/surface_updater.h new file mode 100644 index 00000000000..cfccb865067 --- /dev/null +++ b/chromium/components/feed/core/v2/surface_updater.h @@ -0,0 +1,105 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_SURFACE_UPDATER_H_ +#define COMPONENTS_FEED_CORE_V2_SURFACE_UPDATER_H_ + +#include <string> + +#include "base/containers/flat_set.h" +#include "base/observer_list.h" +#include "components/feed/core/proto/v2/ui.pb.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/stream_model.h" + +namespace feedui { +class StreamUpdate; +} // namespace feedui +namespace feed { +class MetricsReporter; + +// Keeps the UI up to date by calling |SurfaceInterface::StreamUpdate()|. +// Updates are triggered when |StreamModel| changes, or when loading state +// changes (for spinners and zero-state). +class SurfaceUpdater : public StreamModel::Observer { + public: + using SurfaceInterface = FeedStreamApi::SurfaceInterface; + + explicit SurfaceUpdater(MetricsReporter* metrics_reporter); + ~SurfaceUpdater() override; + SurfaceUpdater(const SurfaceUpdater&) = delete; + SurfaceUpdater& operator=(const SurfaceUpdater&) = delete; + + // Sets or unsets the model. When |model| is non-null, triggers the population + // of surfaces. When |model| is null, this does not send any updates to + // surfaces, so they will keep any content they may have been displaying + // before. We don't send a zero-state in this case, since we might want to + // immedately trigger a load. + void SetModel(StreamModel* model); + + // StreamModel::Observer. + void OnUiUpdate(const StreamModel::UiUpdate& update) override; + + // Signals from |FeedStream|. + void SurfaceAdded(SurfaceInterface* surface); + void SurfaceRemoved(SurfaceInterface* surface); + // Called to indicate the initial model load is in progress. + void LoadStreamStarted(); + void LoadStreamComplete(bool success, LoadStreamStatus load_stream_status); + // Called to indicate whether or not we are currently trying to load more + // content at the bottom of the stream. + void SetLoadingMore(bool is_loading); + + // Returns the 0-based index of the slice in the stream, or -1 if the slice is + // not found. Ignores all non-content slices. + int GetSliceIndexFromSliceId(const std::string& slice_id); + + // Returns whether or not at least one surface is attached. + bool HasSurfaceAttached() const; + + // State that together with |model_| determines what should be sent to a + // surface. |DrawState| is usually the same for all surfaces, except for the + // moment when a surface is first attached. + struct DrawState { + bool loading_more = false; + bool loading_initial = false; + feedui::ZeroStateSlice::Type zero_state_type = + feedui::ZeroStateSlice::UNKNOWN; + + bool operator==(const DrawState& rhs) const; + }; + + private: + DrawState GetState() const; + void SendStreamUpdateIfNeeded(); + void SendStreamUpdate( + const std::vector<std::string>& updated_shared_state_ids); + void SendUpdateToSurface(SurfaceInterface* surface, + const feedui::StreamUpdate& update); + + // Members that affect what is sent to surfaces. A value change of these may + // require sending an update to surfaces. + bool loading_more_ = false; + bool loading_initial_ = false; + bool load_stream_failed_ = false; + LoadStreamStatus load_stream_status_ = LoadStreamStatus::kNoStatus; + + // The |DrawState| when the last update was sent to all surfaces. + DrawState last_draw_state_; + + // The set of content that has been sent to all attached surfaces. + base::flat_set<ContentRevision> sent_content_; + + // Owned by |FeedStream|. Null when the model is not loaded. + StreamModel* model_ = nullptr; + // Owned by |FeedStream|. + MetricsReporter* metrics_reporter_; + + // Attached surfaces. + base::ObserverList<SurfaceInterface> surfaces_; +}; +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_SURFACE_UPDATER_H_ diff --git a/chromium/components/feed/core/v2/tasks/clear_all_task.cc b/chromium/components/feed/core/v2/tasks/clear_all_task.cc new file mode 100644 index 00000000000..f05bd0723e3 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/clear_all_task.cc @@ -0,0 +1,32 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/tasks/clear_all_task.h" + +#include "base/callback.h" +#include "base/logging.h" + +#include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/feed_stream.h" + +namespace feed { + +ClearAllTask::ClearAllTask(FeedStream* stream) : stream_(stream) {} +ClearAllTask::~ClearAllTask() = default; + +void ClearAllTask::Run() { + stream_->UnloadModel(); + stream_->GetStore()->ClearAll( + base::BindOnce(&ClearAllTask::StoreClearComplete, GetWeakPtr())); +} + +void ClearAllTask::StoreClearComplete(bool ok) { + DLOG_IF(ERROR, !ok) << "FeedStore::ClearAll failed"; + if (stream_->HasSurfaceAttached()) { + stream_->TriggerStreamLoad(); + } + TaskComplete(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/clear_all_task.h b/chromium/components/feed/core/v2/tasks/clear_all_task.h new file mode 100644 index 00000000000..fee02802773 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/clear_all_task.h @@ -0,0 +1,40 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TASKS_CLEAR_ALL_TASK_H_ +#define COMPONENTS_FEED_CORE_V2_TASKS_CLEAR_ALL_TASK_H_ + +#include "base/memory/weak_ptr.h" +#include "components/offline_pages/task/task.h" + +namespace feed { +class FeedStream; + +// Clears all local Feed data. +// 1. Unload model. +// 2. Clear store. +// 3. Trigger reload if surfaces are attached. +class ClearAllTask : public offline_pages::Task { + public: + explicit ClearAllTask(FeedStream* stream); + ~ClearAllTask() override; + ClearAllTask(const ClearAllTask&) = delete; + ClearAllTask& operator=(const ClearAllTask&) = delete; + + private: + base::WeakPtr<ClearAllTask> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + + // offline_pages::Task. + void Run() override; + + void StoreClearComplete(bool ok); + + FeedStream* stream_; + base::WeakPtrFactory<ClearAllTask> weak_ptr_factory_{this}; +}; +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TASKS_CLEAR_ALL_TASK_H_ diff --git a/chromium/components/feed/core/v2/tasks/load_more_task.cc b/chromium/components/feed/core/v2/tasks/load_more_task.cc new file mode 100644 index 00000000000..55db68a6fb1 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/load_more_task.cc @@ -0,0 +1,91 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/tasks/load_more_task.h" + +#include <memory> +#include <utility> + +#include "base/bind_helpers.h" +#include "base/check.h" +#include "base/time/clock.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "components/feed/core/proto/v2/wire/client_info.pb.h" +#include "components/feed/core/proto/v2/wire/feed_request.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/proto_util.h" +#include "components/feed/core/v2/protocol_translator.h" +#include "components/feed/core/v2/stream_model.h" +#include "components/feed/core/v2/tasks/upload_actions_task.h" + +namespace feed { + +LoadMoreTask::LoadMoreTask(FeedStream* stream, + base::OnceCallback<void(Result)> done_callback) + : stream_(stream), done_callback_(std::move(done_callback)) {} + +LoadMoreTask::~LoadMoreTask() = default; + +void LoadMoreTask::Run() { + // Check prerequisites. + StreamModel* model = stream_->GetModel(); + if (!model) + return Done(LoadStreamStatus::kLoadMoreModelIsNotLoaded); + + LoadStreamStatus final_status = + stream_->ShouldMakeFeedQueryRequest(/*is_load_more=*/true); + if (final_status != LoadStreamStatus::kNoStatus) + return Done(final_status); + + upload_actions_task_ = std::make_unique<UploadActionsTask>( + stream_, + base::BindOnce(&LoadMoreTask::UploadActionsComplete, GetWeakPtr())); + upload_actions_task_->Execute(base::DoNothing()); +} + +void LoadMoreTask::UploadActionsComplete(UploadActionsTask::Result result) { + // Send network request. + fetch_start_time_ = stream_->GetTickClock()->NowTicks(); + stream_->GetNetwork()->SendQueryRequest( + CreateFeedQueryLoadMoreRequest( + stream_->GetRequestMetadata(), + stream_->GetMetadata()->GetConsistencyToken(), + stream_->GetModel()->GetNextPageToken()), + base::BindOnce(&LoadMoreTask::QueryRequestComplete, GetWeakPtr())); +} + +void LoadMoreTask::QueryRequestComplete( + FeedNetwork::QueryRequestResult result) { + StreamModel* model = stream_->GetModel(); + DCHECK(model) << "Model was unloaded outside of a Task"; + + if (!result.response_body) + return Done(LoadStreamStatus::kNoResponseBody); + + RefreshResponseData translated_response = + stream_->GetWireResponseTranslator()->TranslateWireResponse( + *result.response_body, + StreamModelUpdateRequest::Source::kNetworkLoadMore, + stream_->GetClock()->Now()); + + if (!translated_response.model_update_request) + return Done(LoadStreamStatus::kProtoTranslationFailed); + + model->Update(std::move(translated_response.model_update_request)); + + if (translated_response.request_schedule) + stream_->SetRequestSchedule(*translated_response.request_schedule); + + Done(LoadStreamStatus::kLoadedFromNetwork); +} + +void LoadMoreTask::Done(LoadStreamStatus status) { + std::move(done_callback_).Run(Result(status)); + TaskComplete(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/load_more_task.h b/chromium/components/feed/core/v2/tasks/load_more_task.h new file mode 100644 index 00000000000..02069996912 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/load_more_task.h @@ -0,0 +1,61 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TASKS_LOAD_MORE_TASK_H_ +#define COMPONENTS_FEED_CORE_V2_TASKS_LOAD_MORE_TASK_H_ + +#include <memory> + +#include "base/callback.h" +#include "base/memory/weak_ptr.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/tasks/upload_actions_task.h" +#include "components/offline_pages/task/task.h" +#include "components/version_info/channel.h" + +namespace feed { +class FeedStream; + +// Fetches additional content from the network when the model is already loaded. +// Unlike |LoadStreamTask|, this task does not directly persist data to +// |FeedStore|. Instead, |StreamModel| handles persisting the additional +// content. +class LoadMoreTask : public offline_pages::Task { + public: + struct Result { + Result() = default; + explicit Result(LoadStreamStatus a_final_status) + : final_status(a_final_status) {} + // Final status of loading the stream. + LoadStreamStatus final_status = LoadStreamStatus::kNoStatus; + }; + + LoadMoreTask(FeedStream* stream, + base::OnceCallback<void(Result)> done_callback); + ~LoadMoreTask() override; + LoadMoreTask(const LoadMoreTask&) = delete; + LoadMoreTask& operator=(const LoadMoreTask&) = delete; + + private: + void Run() override; + base::WeakPtr<LoadMoreTask> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + + void UploadActionsComplete(UploadActionsTask::Result result); + void QueryRequestComplete(FeedNetwork::QueryRequestResult result); + void Done(LoadStreamStatus status); + + FeedStream* stream_; // Unowned. + base::TimeTicks fetch_start_time_; + std::unique_ptr<UploadActionsTask> upload_actions_task_; + + base::OnceCallback<void(Result)> done_callback_; + base::WeakPtrFactory<LoadMoreTask> weak_ptr_factory_{this}; +}; +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TASKS_LOAD_MORE_TASK_H_ diff --git a/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc index c3c2a261a31..7cd775c37b4 100644 --- a/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc +++ b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc @@ -11,9 +11,9 @@ #include "components/feed/core/proto/v2/store.pb.h" #include "components/feed/core/v2/feed_store.h" #include "components/feed/core/v2/proto_util.h" -#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/scheduling.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/types.h" namespace feed { @@ -24,13 +24,13 @@ LoadStreamFromStoreTask::Result& LoadStreamFromStoreTask::Result::operator=( Result&&) = default; LoadStreamFromStoreTask::LoadStreamFromStoreTask( + LoadType load_type, FeedStore* store, const base::Clock* clock, - UserClass user_class, base::OnceCallback<void(Result)> callback) - : store_(store), + : load_type_(load_type), + store_(store), clock_(clock), - user_class_(user_class), result_callback_(std::move(callback)), update_request_(std::make_unique<StreamModelUpdateRequest>()) {} @@ -47,6 +47,13 @@ void LoadStreamFromStoreTask::LoadStreamDone( Complete(LoadStreamStatus::kFailedWithStoreError); return; } + pending_actions_ = std::move(result.pending_actions); + + if (load_type_ == LoadType::kPendingActionsOnly) { + Complete(LoadStreamStatus::kLoadedFromStore); + return; + } + if (result.stream_structures.empty()) { Complete(LoadStreamStatus::kNoStreamDataInStore); return; @@ -57,7 +64,7 @@ void LoadStreamFromStoreTask::LoadStreamDone( if (content_age < base::TimeDelta()) { Complete(LoadStreamStatus::kDataInStoreIsStaleTimestampInFuture); return; - } else if (ShouldWaitForNewContent(user_class_, true, content_age)) { + } else if (ShouldWaitForNewContent(true, content_age)) { Complete(LoadStreamStatus::kDataInStoreIsStale); return; } @@ -117,8 +124,12 @@ void LoadStreamFromStoreTask::LoadContentDone( void LoadStreamFromStoreTask::Complete(LoadStreamStatus status) { Result task_result; task_result.status = status; - if (status == LoadStreamStatus::kLoadedFromStore) { + task_result.pending_actions = std::move(pending_actions_); + if (status == LoadStreamStatus::kLoadedFromStore && + load_type_ == LoadType::kFullLoad) { task_result.update_request = std::move(update_request_); + } else { + task_result.consistency_token = consistency_token_; } std::move(result_callback_).Run(std::move(task_result)); TaskComplete(); diff --git a/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h index 3718c2c9fd1..61dee09d676 100644 --- a/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h +++ b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h @@ -6,6 +6,7 @@ #define COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_FROM_STORE_TASK_H_ #include <memory> +#include <string> #include <vector> #include "base/callback.h" @@ -16,7 +17,7 @@ namespace base { class Clock; -} +} // namespace base namespace feed { struct StreamModelUpdateRequest; @@ -30,12 +31,24 @@ class LoadStreamFromStoreTask : public offline_pages::Task { Result(Result&&); Result& operator=(Result&&); LoadStreamStatus status = LoadStreamStatus::kNoStatus; + // Only provided if using |LoadType::kFullLoad| AND successful. std::unique_ptr<StreamModelUpdateRequest> update_request; + // This data is provided when |LoadType::kPendingActionsOnly|, or + // when loading fails. + std::string consistency_token; + // Pending actions to be uploaded if the stream is to be loaded from the + // network. + std::vector<feedstore::StoredAction> pending_actions; }; - LoadStreamFromStoreTask(FeedStore* store, + enum class LoadType { + kFullLoad = 0, + kPendingActionsOnly = 1, + }; + + LoadStreamFromStoreTask(LoadType load_type, + FeedStore* store, const base::Clock* clock, - UserClass user_class, base::OnceCallback<void(Result)> callback); ~LoadStreamFromStoreTask() override; LoadStreamFromStoreTask(const LoadStreamFromStoreTask&) = delete; @@ -55,13 +68,16 @@ class LoadStreamFromStoreTask : public offline_pages::Task { return weak_ptr_factory_.GetWeakPtr(); } + LoadType load_type_; FeedStore* store_; // Unowned. const base::Clock* clock_; - UserClass user_class_; bool ignore_staleness_ = false; base::OnceCallback<void(Result)> result_callback_; + // Data to be stuffed into the Result when the task is complete. std::unique_ptr<StreamModelUpdateRequest> update_request_; + std::string consistency_token_; + std::vector<feedstore::StoredAction> pending_actions_; base::WeakPtrFactory<LoadStreamFromStoreTask> weak_ptr_factory_{this}; }; diff --git a/chromium/components/feed/core/v2/tasks/load_stream_task.cc b/chromium/components/feed/core/v2/tasks/load_stream_task.cc index 82e10860897..72bcb2d9bfd 100644 --- a/chromium/components/feed/core/v2/tasks/load_stream_task.cc +++ b/chromium/components/feed/core/v2/tasks/load_stream_task.cc @@ -8,38 +8,74 @@ #include <utility> #include "base/bind_helpers.h" -#include "base/logging.h" +#include "base/check.h" #include "base/time/clock.h" +#include "base/time/tick_clock.h" #include "base/time/time.h" +#include "components/feed/core/proto/v2/wire/capability.pb.h" #include "components/feed/core/proto/v2/wire/client_info.pb.h" #include "components/feed/core/proto/v2/wire/feed_request.pb.h" #include "components/feed/core/proto/v2/wire/request.pb.h" #include "components/feed/core/v2/feed_network.h" #include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/proto_util.h" +#include "components/feed/core/v2/protocol_translator.h" #include "components/feed/core/v2/stream_model.h" -#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/tasks/upload_actions_task.h" namespace feed { +namespace { +using LoadType = LoadStreamTask::LoadType; +using Result = LoadStreamTask::Result; + +feedwire::FeedQuery::RequestReason GetRequestReason(LoadType load_type) { + switch (load_type) { + case LoadType::kInitialLoad: + return feedwire::FeedQuery::MANUAL_REFRESH; + case LoadType::kBackgroundRefresh: + return feedwire::FeedQuery::SCHEDULED_REFRESH; + } +} + +} // namespace + +Result::Result() = default; +Result::Result(LoadStreamStatus status) : final_status(status) {} +Result::~Result() = default; +Result::Result(const Result&) = default; +Result& Result::operator=(const Result&) = default; -LoadStreamTask::LoadStreamTask(FeedStream* stream, +LoadStreamTask::LoadStreamTask(LoadType load_type, + FeedStream* stream, base::OnceCallback<void(Result)> done_callback) - : stream_(stream), done_callback_(std::move(done_callback)) {} + : load_type_(load_type), + stream_(stream), + done_callback_(std::move(done_callback)) {} LoadStreamTask::~LoadStreamTask() = default; void LoadStreamTask::Run() { - // Phase 1. - // - Return early if the model is already loaded. - // - Try to load from persistent storage. + // Phase 1: Try to load from persistent storage. - // Don't load if the model is already loaded. - if (stream_->GetModel()) { - Done(LoadStreamStatus::kModelAlreadyLoaded); - return; + // TODO(harringtond): We're checking ShouldAttemptLoad() here and before the + // task is added to the task queue. Maybe we can simplify this. + + // First, ensure we still should load the model. + LoadStreamStatus should_not_attempt_reason = stream_->ShouldAttemptLoad( + /*model_loading=*/true); + if (should_not_attempt_reason != LoadStreamStatus::kNoStatus) { + return Done(should_not_attempt_reason); } + // Use |kConsistencyTokenOnly| to short-circuit loading from store if we don't + // need the full stream state. + auto load_from_store_type = + (load_type_ == LoadType::kInitialLoad) + ? LoadStreamFromStoreTask::LoadType::kFullLoad + : LoadStreamFromStoreTask::LoadType::kPendingActionsOnly; + load_from_store_task_ = std::make_unique<LoadStreamFromStoreTask>( - stream_->GetStore(), stream_->GetClock(), stream_->GetUserClass(), + load_from_store_type, stream_->GetStore(), stream_->GetClock(), base::BindOnce(&LoadStreamTask::LoadFromStoreComplete, GetWeakPtr())); load_from_store_task_->Execute(base::DoNothing()); } @@ -51,7 +87,8 @@ void LoadStreamTask::LoadFromStoreComplete( // - If loading from store works, update the model. // - Otherwise, try to load from the network. - if (result.status == LoadStreamStatus::kLoadedFromStore) { + if (load_type_ == LoadType::kInitialLoad && + result.status == LoadStreamStatus::kLoadedFromStore) { auto model = std::make_unique<StreamModel>(); model->Update(std::move(result.update_request)); stream_->LoadModel(std::move(model)); @@ -65,46 +102,55 @@ void LoadStreamTask::LoadFromStoreComplete( return; } - // TODO(harringtond): Add throttling. - // TODO(harringtond): Request parameters here are all placeholder values. - feedwire::Request request; - feedwire::ClientInfo& client_info = - *request.mutable_feed_request()->mutable_client_info(); - client_info.set_platform_type(feedwire::ClientInfo::ANDROID_ID); - client_info.set_app_type(feedwire::ClientInfo::CHROME); - request.mutable_feed_request()->mutable_feed_query()->set_reason( - feedwire::FeedQuery::MANUAL_REFRESH); - - fetch_start_time_ = base::TimeTicks::Now(); + // If making a request, first try to upload pending actions. + upload_actions_task_ = std::make_unique<UploadActionsTask>( + std::move(result.pending_actions), stream_, + base::BindOnce(&LoadStreamTask::UploadActionsComplete, GetWeakPtr())); + upload_actions_task_->Execute(base::DoNothing()); +} + +void LoadStreamTask::UploadActionsComplete(UploadActionsTask::Result result) { stream_->GetNetwork()->SendQueryRequest( - request, + CreateFeedQueryRefreshRequest( + GetRequestReason(load_type_), stream_->GetRequestMetadata(), + stream_->GetMetadata()->GetConsistencyToken()), base::BindOnce(&LoadStreamTask::QueryRequestComplete, GetWeakPtr())); } void LoadStreamTask::QueryRequestComplete( FeedNetwork::QueryRequestResult result) { DCHECK(!stream_->GetModel()); + + network_response_info_ = result.response_info; + if (!result.response_body) { Done(LoadStreamStatus::kNoResponseBody); return; } - std::unique_ptr<StreamModelUpdateRequest> update_request = + RefreshResponseData response_data = stream_->GetWireResponseTranslator()->TranslateWireResponse( - *result.response_body, base::TimeTicks::Now() - fetch_start_time_, + *result.response_body, + StreamModelUpdateRequest::Source::kNetworkUpdate, stream_->GetClock()->Now()); - if (!update_request) { + if (!response_data.model_update_request) { Done(LoadStreamStatus::kProtoTranslationFailed); return; } - stream_->GetStore()->SaveFullStream( - std::make_unique<StreamModelUpdateRequest>(*update_request), + stream_->GetStore()->OverwriteStream( + std::make_unique<StreamModelUpdateRequest>( + *response_data.model_update_request), base::DoNothing()); - auto model = std::make_unique<StreamModel>(); - model->Update(std::move(update_request)); - stream_->LoadModel(std::move(model)); + if (load_type_ != LoadType::kBackgroundRefresh) { + auto model = std::make_unique<StreamModel>(); + model->Update(std::move(response_data.model_update_request)); + stream_->LoadModel(std::move(model)); + } + + if (response_data.request_schedule) + stream_->SetRequestSchedule(*response_data.request_schedule); Done(LoadStreamStatus::kLoadedFromNetwork); } @@ -113,6 +159,8 @@ void LoadStreamTask::Done(LoadStreamStatus status) { Result result; result.load_from_store_status = load_from_store_status_; result.final_status = status; + result.load_type = load_type_; + result.network_response_info = network_response_info_; std::move(done_callback_).Run(result); TaskComplete(); } diff --git a/chromium/components/feed/core/v2/tasks/load_stream_task.h b/chromium/components/feed/core/v2/tasks/load_stream_task.h index ad19cd90d64..8bcc102c7b2 100644 --- a/chromium/components/feed/core/v2/tasks/load_stream_task.h +++ b/chromium/components/feed/core/v2/tasks/load_stream_task.h @@ -9,33 +9,53 @@ #include "base/callback.h" #include "base/memory/weak_ptr.h" +#include "base/optional.h" #include "components/feed/core/v2/enums.h" #include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/public/types.h" #include "components/feed/core/v2/tasks/load_stream_from_store_task.h" +#include "components/feed/core/v2/tasks/upload_actions_task.h" #include "components/offline_pages/task/task.h" +#include "components/version_info/channel.h" namespace feed { class FeedStream; -// Loads the stream model from storage or network. -// If successful, this directly forces a model load in |FeedStream()| -// before completing the task. -// TODO(harringtond): If we read data from the network, it needs to be -// persisted. +// Loads the stream model from storage or network. If data is refreshed from the +// network, it is persisted to |FeedStore| by overwriting any existing stream +// data. +// This task has two modes, see |LoadStreamTask::LoadType|. class LoadStreamTask : public offline_pages::Task { public: + enum class LoadType { + // Loads the stream model into memory. If successful, this directly forces a + // model load in |FeedStream()| before completing the task. + kInitialLoad, + // Refreshes the stored stream data from the network. This will fail if the + // model is already loaded. + kBackgroundRefresh, + }; + struct Result { - Result() = default; - explicit Result(LoadStreamStatus a_final_status) - : final_status(a_final_status) {} + Result(); + explicit Result(LoadStreamStatus status); + ~Result(); + Result(const Result&); + Result& operator=(const Result&); + // Final status of loading the stream. LoadStreamStatus final_status = LoadStreamStatus::kNoStatus; // Status of just loading the stream from the persistent store, if that // was attempted. LoadStreamStatus load_from_store_status = LoadStreamStatus::kNoStatus; + LoadType load_type; + // Information about the network request, if one was made. + base::Optional<NetworkResponseInfo> network_response_info; }; - explicit LoadStreamTask(FeedStream* stream, - base::OnceCallback<void(Result)> done_callback); + + LoadStreamTask(LoadType load_type, + FeedStream* stream, + base::OnceCallback<void(Result)> done_callback); ~LoadStreamTask() override; LoadStreamTask(const LoadStreamTask&) = delete; LoadStreamTask& operator=(const LoadStreamTask&) = delete; @@ -47,14 +67,21 @@ class LoadStreamTask : public offline_pages::Task { } void LoadFromStoreComplete(LoadStreamFromStoreTask::Result result); + void UploadActionsComplete(UploadActionsTask::Result result); void QueryRequestComplete(FeedNetwork::QueryRequestResult result); void Done(LoadStreamStatus status); + LoadType load_type_; FeedStream* stream_; // Unowned. std::unique_ptr<LoadStreamFromStoreTask> load_from_store_task_; + + // Information to be stuffed in |Result|. LoadStreamStatus load_from_store_status_ = LoadStreamStatus::kNoStatus; + base::Optional<NetworkResponseInfo> network_response_info_; + base::TimeTicks fetch_start_time_; base::OnceCallback<void(Result)> done_callback_; + std::unique_ptr<UploadActionsTask> upload_actions_task_; base::WeakPtrFactory<LoadStreamTask> weak_ptr_factory_{this}; }; } // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/upload_actions_task.cc b/chromium/components/feed/core/v2/tasks/upload_actions_task.cc new file mode 100644 index 00000000000..adcc5c87fbb --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/upload_actions_task.cc @@ -0,0 +1,305 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/tasks/upload_actions_task.h" + +#include <memory> +#include <vector> +#include "base/time/time.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/action_request.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_request.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_response.pb.h" +#include "components/feed/core/v2/config.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/request_throttler.h" + +namespace feed { +using feedstore::StoredAction; + +namespace { + +bool ShouldUpload(const StoredAction& action) { + base::Time action_time = + base::Time::UnixEpoch() + + base::TimeDelta::FromSeconds( + action.action().client_data().timestamp_seconds()); + base::TimeDelta age = base::Time::Now() - action_time; + if (age < base::TimeDelta()) + age = base::TimeDelta(); + + return action.upload_attempt_count() < + GetFeedConfig().max_action_upload_attempts && + age < GetFeedConfig().max_action_age; +} + +void ReportBatchStatus(UploadActionsBatchStatus status) { + // TODO(iwells): Get rid of the DVLOG and record status to a histogram. + DVLOG(1) << "UploadActionsBatchStatus: " << status; +} + +} // namespace + +class UploadActionsTask::Batch { + public: + Batch() + : feed_action_request_(std::make_unique<feedwire::FeedActionRequest>()) {} + Batch(const Batch&) = delete; + Batch& operator=(const Batch&) = delete; + ~Batch() = default; + + // Consumes one or more actions and erases them from |actions_in|. Actions + // that should be updated in the store and uploaded are added to |to_update| + // and IDs of actions that should be erased from the store are added to + // |to_erase|. + void BiteOffAFewActions(std::vector<StoredAction>* actions_in, + std::vector<StoredAction>* to_update, + std::vector<LocalActionId>* to_erase) { + size_t upload_size = 0ul; + for (StoredAction& action : *actions_in) { + if (ShouldUpload(action)) { + size_t message_size = action.ByteSizeLong(); + // In the weird event that a single action is larger than the limit, it + // will be uploaded by itself. + if (upload_size > 0ul && message_size + upload_size > + GetFeedConfig().max_action_upload_bytes) + break; + + *feed_action_request_->add_feed_action() = action.action(); + action.set_upload_attempt_count(action.upload_attempt_count() + 1); + uploaded_ids_.push_back(LocalActionId(action.id())); + to_update->push_back(std::move(action)); + + upload_size += message_size; + } else { + to_erase->push_back(LocalActionId(action.id())); + } + } + + size_t actions_consumed = to_update->size() + to_erase->size(); + actions_in->erase(actions_in->begin(), + actions_in->begin() + actions_consumed); + stale_count_ = to_erase->size(); + } + + size_t UploadCount() const { return uploaded_ids_.size(); } + size_t StaleCount() const { return stale_count_; } + + std::unique_ptr<feedwire::FeedActionRequest> disown_feed_action_request() { + return std::move(feed_action_request_); + } + std::vector<LocalActionId> disown_uploaded_ids() { + return std::move(uploaded_ids_); + } + + private: + std::unique_ptr<feedwire::FeedActionRequest> feed_action_request_; + std::vector<LocalActionId> uploaded_ids_; + size_t stale_count_ = 0; +}; + +UploadActionsTask::UploadActionsTask( + feedwire::FeedAction action, + bool upload_now, + FeedStream* stream, + base::OnceCallback<void(UploadActionsTask::Result)> callback) + : stream_(stream), + upload_now_(upload_now), + wire_action_(std::move(action)), + callback_(std::move(callback)) { + wire_action_->mutable_client_data()->set_timestamp_seconds( + (base::Time::Now() - base::Time::UnixEpoch()).InSeconds()); +} + +UploadActionsTask::UploadActionsTask( + std::vector<feedstore::StoredAction> pending_actions, + FeedStream* stream, + base::OnceCallback<void(UploadActionsTask::Result)> callback) + : stream_(stream), + pending_actions_(std::move(pending_actions)), + callback_(std::move(callback)) {} + +UploadActionsTask::UploadActionsTask( + FeedStream* stream, + base::OnceCallback<void(UploadActionsTask::Result)> callback) + : stream_(stream), + read_pending_actions_(true), + callback_(std::move(callback)) {} + +UploadActionsTask::~UploadActionsTask() = default; + +void UploadActionsTask::Run() { + consistency_token_ = stream_->GetMetadata()->GetConsistencyToken(); + + // From constructor 1: If there is an action to store, store it and maybe try + // to upload all pending actions. + if (wire_action_) { + StoredAction action; + action.set_id(stream_->GetMetadata()->GetNextActionId().GetUnsafeValue()); + *action.mutable_action() = std::move(*wire_action_); + // No need to set upload_attempt_count as it defaults to 0. + // WriteActions() sets the ID. + stream_->GetStore()->WriteActions( + {std::move(action)}, + base::BindOnce(&UploadActionsTask::OnStorePendingActionFinished, + weak_ptr_factory_.GetWeakPtr())); + return; + } + + // From constructor 3: Read actions and upload. + if (read_pending_actions_) { + ReadActions(); + return; + } + + // From constructor 2: Upload whatever was passed to us. + UploadPendingActions(); +} + +void UploadActionsTask::OnStorePendingActionFinished(bool write_ok) { + if (!write_ok) { + Done(UploadActionsStatus::kFailedToStorePendingAction); + return; + } + + if (!upload_now_) { + Done(UploadActionsStatus::kStoredPendingAction); + return; + } + + // If the new action was stored and upload_now was set, load all pending + // actions and try to upload. + ReadActions(); +} + +void UploadActionsTask::ReadActions() { + stream_->GetStore()->ReadActions( + base::BindOnce(&UploadActionsTask::OnReadPendingActionsFinished, + weak_ptr_factory_.GetWeakPtr())); +} + +void UploadActionsTask::OnReadPendingActionsFinished( + std::vector<feedstore::StoredAction> actions) { + pending_actions_ = std::move(actions); + UploadPendingActions(); +} + +void UploadActionsTask::UploadPendingActions() { + if (pending_actions_.empty()) { + Done(UploadActionsStatus::kNoPendingActions); + return; + } + UpdateAndUploadNextBatch(); +} + +void UploadActionsTask::UpdateAndUploadNextBatch() { + // Finish if all pending actions have been visited. + if (pending_actions_.empty()) { + ReportBatchStatus(UploadActionsBatchStatus::kSuccessfullyUploadedBatch); + UpdateTokenAndFinish(); + return; + } + + // Finish if there's no quota remaining for actions uploads. + if (!stream_->GetRequestThrottler()->RequestQuota( + NetworkRequestType::kUploadActions)) { + ReportBatchStatus(UploadActionsBatchStatus::kExhaustedUploadQuota); + UpdateTokenAndFinish(); + return; + } + + // Grab a few actions to be processed and erase them from pending_actions_. + auto batch = std::make_unique<Batch>(); + std::vector<feedstore::StoredAction> to_update; + std::vector<LocalActionId> to_erase; + batch->BiteOffAFewActions(&pending_actions_, &to_update, &to_erase); + + // Update upload_attempt_count, remove old actions, then try to upload. + stream_->GetStore()->UpdateActions( + std::move(to_update), std::move(to_erase), + base::BindOnce(&UploadActionsTask::OnUpdateActionsFinished, + weak_ptr_factory_.GetWeakPtr(), std::move(batch))); +} + +void UploadActionsTask::OnUpdateActionsFinished( + std::unique_ptr<UploadActionsTask::Batch> batch, + bool update_ok) { + // Stop if there are no actions to upload. + if (batch->UploadCount() == 0ul) { + ReportBatchStatus(UploadActionsBatchStatus::kAllActionsWereStale); + UpdateTokenAndFinish(); + return; + } + + // Skip uploading if these actions couldn't be updated in the store. + if (!update_ok) { + ReportBatchStatus(UploadActionsBatchStatus::kFailedToUpdateStore); + UpdateAndUploadNextBatch(); + return; + } + upload_attempt_count_ += batch->UploadCount(); + stale_count_ += batch->StaleCount(); + + std::unique_ptr<feedwire::FeedActionRequest> request = + batch->disown_feed_action_request(); + request->mutable_consistency_token()->set_token(consistency_token_); + + feedwire::ActionRequest action_request; + action_request.set_request_version( + feedwire::ActionRequest::FEED_UPLOAD_ACTION); + action_request.set_allocated_feed_action_request(request.release()); + + FeedNetwork* network = stream_->GetNetwork(); + DCHECK(network); + + network->SendActionRequest( + action_request, + base::BindOnce(&UploadActionsTask::OnUploadFinished, + weak_ptr_factory_.GetWeakPtr(), std::move(batch))); +} + +void UploadActionsTask::OnUploadFinished( + std::unique_ptr<UploadActionsTask::Batch> batch, + FeedNetwork::ActionRequestResult result) { + if (!result.response_body) { + ReportBatchStatus(UploadActionsBatchStatus::kFailedToUpload); + UpdateAndUploadNextBatch(); + return; + } + + consistency_token_ = std::move(result.response_body->feed_response() + .feed_response() + .consistency_token() + .token()); + + stream_->GetStore()->RemoveActions( + batch->disown_uploaded_ids(), + base::BindOnce(&UploadActionsTask::OnUploadedActionsRemoved, + weak_ptr_factory_.GetWeakPtr())); +} + +void UploadActionsTask::OnUploadedActionsRemoved(bool remove_ok) { + ReportBatchStatus(UploadActionsBatchStatus::kFailedToRemoveUploadedActions); + UpdateAndUploadNextBatch(); +} + +void UploadActionsTask::UpdateTokenAndFinish() { + if (consistency_token_.empty()) { + Done(UploadActionsStatus::kFinishedWithoutUpdatingConsistencyToken); + return; + } + + stream_->GetMetadata()->SetConsistencyToken(consistency_token_); + Done(UploadActionsStatus::kUpdatedConsistencyToken); +} + +void UploadActionsTask::Done(UploadActionsStatus status) { + DVLOG(1) << "UploadActionsTask finished with status " << status; + std::move(callback_).Run({status, upload_attempt_count_, stale_count_}); + TaskComplete(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/upload_actions_task.h b/chromium/components/feed/core/v2/tasks/upload_actions_task.h new file mode 100644 index 00000000000..d543a22ec95 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/upload_actions_task.h @@ -0,0 +1,117 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TASKS_UPLOAD_ACTIONS_TASK_H_ +#define COMPONENTS_FEED_CORE_V2_TASKS_UPLOAD_ACTIONS_TASK_H_ + +#include <memory> +#include <vector> +#include "base/callback.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_request.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_response.pb.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/types.h" +#include "components/offline_pages/task/task.h" + +namespace feed { +class FeedStream; + +// Uploads user actions and returns a consistency token to be used for loading +// the stream later. +// +// Repeat while pending actions remain to be processed: +// 1. Gather as many actions as can fit in the request size limit. If we +// encounter stale actions that should be erased, set those aside. +// 2. Increment upload_attempt_count for each action to be uploaded. Write +// these actions back to the store and erase actions that should be erased. +// 3. Try to upload the batch of actions. If the upload is successful, get +// the new consistency token and erase the uploaded actions from the store. +// +// If we have a new consistency token, it's the caller's responsibility to write +// it to storage. +class UploadActionsTask : public offline_pages::Task { + public: + struct Result { + UploadActionsStatus status; + + // For testing. Reports the number of actions for which upload was + // attempted. + size_t upload_attempt_count; + // For testing. Reports the number of actions which were erased because of + // staleness. + size_t stale_count; + }; + + // Store an action. Use |upload_now|=true to kick off an upload of all pending + // actions. |callback| is called with the new consistency token (or empty + // string if no token was received). + UploadActionsTask(feedwire::FeedAction action, + bool upload_now, + FeedStream* stream, + base::OnceCallback<void(Result)> callback); + // Upload |pending_actions| and update the store. Note: |pending_actions| + // should already be in the store before running the task. + UploadActionsTask(std::vector<feedstore::StoredAction> pending_actions, + FeedStream* stream, + base::OnceCallback<void(Result)> callback); + // Same as above, but reads pending actions and consistency token from the + // store and uploads those. + UploadActionsTask(FeedStream* stream, + base::OnceCallback<void(Result)> callback); + + ~UploadActionsTask() override; + UploadActionsTask(const UploadActionsTask&) = delete; + UploadActionsTask& operator=(const UploadActionsTask&) = delete; + + private: + class Batch; + + void Run() override; + + void OnStorePendingActionFinished(bool write_ok); + + void ReadActions(); + void OnReadPendingActionsFinished( + std::vector<feedstore::StoredAction> result); + void UploadPendingActions(); + void UpdateAndUploadNextBatch(); + void OnUpdateActionsFinished(std::unique_ptr<Batch> batch, bool update_ok); + void OnUploadFinished(std::unique_ptr<Batch> batch, + FeedNetwork::ActionRequestResult result); + void OnUploadedActionsRemoved(bool remove_ok); + void UpdateTokenAndFinish(); + void Done(UploadActionsStatus status); + + FeedStream* stream_; + bool upload_now_ = false; + bool read_pending_actions_ = false; + // Pending action to be stored. + base::Optional<feedwire::FeedAction> wire_action_; + + // Pending actions to be uploaded, set either by the constructor or by + // OnReadPendingActionsFinished(). Not set if we're just storing an action. + std::vector<feedstore::StoredAction> pending_actions_; + + // This copy of the consistency token is set in Run(), possibly updated + // through batch uploads, and then persisted before the task finishes. + std::string consistency_token_; + base::OnceCallback<void(Result)> callback_; + + // Number of actions for which upload was attempted. + size_t upload_attempt_count_ = 0; + // Number of stale actions. + size_t stale_count_ = 0; + + base::WeakPtrFactory<UploadActionsTask> weak_ptr_factory_{this}; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TASKS_UPLOAD_ACTIONS_TASK_H_ diff --git a/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc index 27f73cbb1be..f5b354833af 100644 --- a/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc +++ b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc @@ -3,19 +3,31 @@ // found in the LICENSE file. #include "components/feed/core/v2/tasks/wait_for_store_initialize_task.h" - #include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/feed_stream.h" namespace feed { -WaitForStoreInitializeTask::WaitForStoreInitializeTask(FeedStore* store) - : store_(store) {} +WaitForStoreInitializeTask::WaitForStoreInitializeTask(FeedStream* stream) + : stream_(stream), store_(stream->GetStore()) {} WaitForStoreInitializeTask::~WaitForStoreInitializeTask() = default; void WaitForStoreInitializeTask::Run() { // |this| stays alive as long as the |store_|, so Unretained is safe. - store_->Initialize(base::BindOnce(&WaitForStoreInitializeTask::TaskComplete, - base::Unretained(this))); + store_->Initialize(base::BindOnce( + &WaitForStoreInitializeTask::OnStoreInitialized, base::Unretained(this))); +} + +void WaitForStoreInitializeTask::OnStoreInitialized() { + store_->ReadMetadata(base::BindOnce( + &WaitForStoreInitializeTask::OnMetadataLoaded, base::Unretained(this))); +} + +void WaitForStoreInitializeTask::OnMetadataLoaded( + std::unique_ptr<feedstore::Metadata> metadata) { + if (metadata) + stream_->GetMetadata()->Populate(std::move(*metadata)); + TaskComplete(); } } // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h index f7640676c68..6edbcda1b4d 100644 --- a/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h +++ b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h @@ -5,16 +5,18 @@ #ifndef COMPONENTS_FEED_CORE_V2_TASKS_WAIT_FOR_STORE_INITIALIZE_TASK_H_ #define COMPONENTS_FEED_CORE_V2_TASKS_WAIT_FOR_STORE_INITIALIZE_TASK_H_ +#include "components/feed/core/proto/v2/store.pb.h" #include "components/offline_pages/task/task.h" namespace feed { +class FeedStream; class FeedStore; // Initializes |store|. This task is run first so that other tasks can assume // storage is initialized. class WaitForStoreInitializeTask : public offline_pages::Task { public: - explicit WaitForStoreInitializeTask(FeedStore* store); + explicit WaitForStoreInitializeTask(FeedStream* stream); ~WaitForStoreInitializeTask() override; WaitForStoreInitializeTask(const WaitForStoreInitializeTask&) = delete; WaitForStoreInitializeTask& operator=(const WaitForStoreInitializeTask&) = @@ -23,6 +25,10 @@ class WaitForStoreInitializeTask : public offline_pages::Task { private: void Run() override; + void OnStoreInitialized(); + void OnMetadataLoaded(std::unique_ptr<feedstore::Metadata> metadata); + + FeedStream* stream_; FeedStore* store_; }; diff --git a/chromium/components/feed/core/v2/tools/decode_feed_request.py b/chromium/components/feed/core/v2/tools/decode_feed_request.py new file mode 100755 index 00000000000..6dab6c86215 --- /dev/null +++ b/chromium/components/feed/core/v2/tools/decode_feed_request.py @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Usage: cat request_binary_file | ./decode_binary_feed_request.py + +CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..) +python3 \ + $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \ + --direction=reverse --chromium_path="$CHROMIUM_SRC" \ + --message=feedwire.Request diff --git a/chromium/components/feed/core/v2/tools/feed_response_to_textproto.sh b/chromium/components/feed/core/v2/tools/decode_feed_response.sh index 48c3da5d410..34dea8b200e 100755 --- a/chromium/components/feed/core/v2/tools/feed_response_to_textproto.sh +++ b/chromium/components/feed/core/v2/tools/decode_feed_response.sh @@ -6,20 +6,14 @@ # Converts a Feed HTTP response from binary to text using proto definitions from # Chromium. # -# Usage: feed_response_to_textproto.sh <in.binarypb> <out.textproto> - -IN_FILE=$1 -OUT_FILE=$2 -TMP_FILE=/tmp/trimmedfeedresponse.binarypb +# Usage: curl 'some url' | feed_response_to_textproto.sh CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..) -FEEDPROTO="$CHROMIUM_SRC/components/feed/core/proto" - -# Responses start with a 4-byte length value that must be removed. -tail -c +4 $IN_FILE > $TMP_FILE +# Responses start with a varint length value that must be removed. +python3 -c "import sys +while sys.stdin.buffer.read(1)[0]>127: + pass +sys.stdout.buffer.write(sys.stdin.buffer.read())" | \ python3 $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \ - --chromium_path=$CHROMIUM_SRC \ - --output_file=$OUT_FILE \ - --source_file=$TMP_FILE \ - --direction=reverse + --chromium_path=$CHROMIUM_SRC --direction=reverse diff --git a/chromium/components/feed/core/v2/tools/encode_feed_response.sh b/chromium/components/feed/core/v2/tools/encode_feed_response.sh new file mode 100755 index 00000000000..298a31b492a --- /dev/null +++ b/chromium/components/feed/core/v2/tools/encode_feed_response.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Converts a Feed response textproto to a binary proto. +# +# Usage: cat some.textproto | feed_response_to_textproto.sh > result.binarypb + +CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..) + +python3 $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \ + --chromium_path=$CHROMIUM_SRC --direction=forward diff --git a/chromium/components/feed/core/v2/tools/make_feed_query_request.sh b/chromium/components/feed/core/v2/tools/make_feed_query_request.sh new file mode 100755 index 00000000000..204e9730e77 --- /dev/null +++ b/chromium/components/feed/core/v2/tools/make_feed_query_request.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Usage: make_feed_query_request.sh request.textproto + +CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..) + +PLD=$(python3 \ + $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \ + --chromium_path=$CHROMIUM_SRC \ + --message=feedwire.Request \ + --output_format=base64 \ + --source_file=$1) + +BASE_URL="https://www.google.com/httpservice" +ENDPOINT="TrellisClankService/FeedQuery" +QUERY_URL="$BASE_URL/retry/$ENDPOINT" + +echo "$QUERY_URL?fmt=bin&hl=en-US&reqpld=$PLD" diff --git a/chromium/components/feed/core/v2/tools/protoc_util.py b/chromium/components/feed/core/v2/tools/protoc_util.py index 41104fe5db5..2d2ce70da06 100755 --- a/chromium/components/feed/core/v2/tools/protoc_util.py +++ b/chromium/components/feed/core/v2/tools/protoc_util.py @@ -12,52 +12,61 @@ import subprocess _protoc_path = None + def run_command(args, input): - """Uses subprocess to execute the command line args.""" - proc = subprocess.run( - args, - input=input, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True) - return proc.stdout + """Uses subprocess to execute the command line args.""" + proc = subprocess.run( + args, input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise RuntimeError(proc.stderr.decode('utf-8')) -def get_protoc_common_args(root_dir, proto_path): - """Returns a list of protoc common args as a list.""" - result = [ - '-I' + os.path.join(root_dir) - ] - for root, _, files in os.walk(os.path.join(root_dir, proto_path)): - result += [os.path.join(root, f) for f in files if f.endswith('.proto')] + return proc.stdout - return result + +def get_protoc_common_args(root_dir, proto_path): + """Returns a list of protoc common args as a list.""" + result = ['-I' + os.path.join(root_dir)] + full_path = os.path.join(root_dir, proto_path) + if os.path.isdir(full_path): + for root, _, files in os.walk(full_path): + result += [ + os.path.join(root, f) for f in files if f.endswith('.proto') + ] + else: + result += [full_path] + return result def encode_proto(text, message_name, root_dir, proto_path): - """Calls a command line to encode the text string and returns binary bytes.""" - return run_command([protoc_path(root_dir), '--encode=' + message_name] - + get_protoc_common_args(root_dir, proto_path), - text.encode()) + """Calls a command line to encode the text string and returns binary + bytes.""" + input_buffer = text + if isinstance(input_buffer, str): + input_buffer = text.encode() + return run_command([protoc_path(root_dir), '--encode=' + message_name + ] + get_protoc_common_args(root_dir, proto_path), + input_buffer) def decode_proto(data, message_name, root_dir, proto_path): - """Calls a command line to decode the binary bytes array into text string.""" - return run_command([protoc_path(root_dir), '--decode=' + message_name - ] + get_protoc_common_args(root_dir, proto_path), - data).decode('utf-8') + """Calls a command line to decode the binary bytes array into text + string.""" + return run_command([protoc_path(root_dir), '--decode=' + message_name + ] + get_protoc_common_args(root_dir, proto_path), + data).decode('utf-8') def protoc_path(root_dir): - """Returns the path to the proto compiler, protoc.""" - global _protoc_path - if not _protoc_path: - protoc_list = list( - glob.glob(os.path.join(root_dir, "out") + "/*/protoc")) + list( - glob.glob(os.path.join(root_dir, "out") + "/*/*/protoc")) - if not len(protoc_list): - print("Can't find a suitable build output directory", - "(it should have protoc)") - sys.exit(1) - _protoc_path = protoc_list[0] - return _protoc_path + """Returns the path to the proto compiler, protoc.""" + global _protoc_path + if not _protoc_path: + protoc_list = list( + glob.glob(os.path.join(root_dir, "out") + "/*/protoc")) + list( + glob.glob(os.path.join(root_dir, "out") + "/*/*/protoc")) + if not len(protoc_list): + print("Can't find a suitable build output directory", + "(it should have protoc)") + sys.exit(1) + _protoc_path = protoc_list[0] + return _protoc_path diff --git a/chromium/components/feed/core/v2/tools/stream_dump.py b/chromium/components/feed/core/v2/tools/stream_dump.py new file mode 100755 index 00000000000..c6a0040bbc4 --- /dev/null +++ b/chromium/components/feed/core/v2/tools/stream_dump.py @@ -0,0 +1,119 @@ +#!/usr/bin/python3 +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Usage: +# Dump the feedv2 stream database from a connected device to a directory on this +# computer. +# > stream_dump.py --device=FA77D0303076 --apk='com.chrome.canary' +# > ls /tmp/feed_dump +# +# Files are output as textproto. +# +# Make any desired modifications, and then upload the dump back to the connected +# device. +# > stream_dump.py --device=FA77D0303076 --apk='com.chrome.canary' --reverse +import argparse +import glob +import os +import plyvel +import protoc_util +import re +import subprocess +import sys + +from os.path import join, dirname, realpath + +parser = argparse.ArgumentParser() +parser.add_argument("--db", help="Path to db", default='/tmp/feed_dump/db') +parser.add_argument( + "--dump_to", help="Dump output directory", default='/tmp/feed_dump') +parser.add_argument( + "--reverse", help="Write dump back to database", action='store_true') +parser.add_argument("--device", help="adb device to use") +parser.add_argument( + "--apk", help="APK to dump from/to", default='com.chrome.canary') + +args = parser.parse_args() + +ROOT_DIR = realpath(join(dirname(__file__), "../../../../..")) +DUMP_DIR = args.dump_to +DB_PATH = args.db +STREAM_DB_PATH = join(DB_PATH, 'shared_proto_db') +DEVICE_DB_PATH = ( + "/data/data/{}/" + "app_chrome/Default/shared_proto_db").format(args.apk) +STORAGE_PROTO = 'components/feed/core/proto/v2/store.proto' + +# From the shared proto db ID, see +# components/leveldb_proto/public/shared_proto_database_client_list.h +KEY_PREFIX = '26_' + + +def adb_base_args(): + adb_path = join(ROOT_DIR, + "third_party/android_sdk/public/platform-tools/adb") + adb_device = args.device + if adb_device: + return [adb_path, "-s", adb_device] + return [adb_path] + + +def adb_pull_db(): + subprocess.check_call(adb_base_args() + ["pull", DEVICE_DB_PATH, DB_PATH]) + + +def adb_push_db(): + subprocess.check_call(adb_base_args() + + ["push", STREAM_DB_PATH, DEVICE_DB_PATH]) + + +# Extract a binary proto database entry into textproto. +def extract_db_entry(key, data): + return protoc_util.decode_proto(data, 'feedstore.Record', ROOT_DIR, + STORAGE_PROTO) + + +# Dump the database to a local directory as textproto files. +def dump(): + os.makedirs(DUMP_DIR, exist_ok=True) + os.makedirs(DB_PATH, exist_ok=True) + adb_pull_db() + db = plyvel.DB(STREAM_DB_PATH, create_if_missing=False) + with db.iterator() as it: + for i, (k, v) in enumerate(it): + k = k.decode('utf-8') + if not k.startswith(KEY_PREFIX): continue + key = k[3:] + with open(join(DUMP_DIR, 'entry{:03d}.key'.format(i)), 'w') as f: + f.write(key) + with open(join(DUMP_DIR, 'entry{:03d}.textproto'.format(i)), + 'w') as f: + f.write(extract_db_entry(k, v)) + print('Finished dumping to', DUMP_DIR) + db.close() + + +# Reverse of dump(). +def load(): + db = plyvel.DB(STREAM_DB_PATH, create_if_missing=False) + # For each textproto file, update its database entry. + # No attempt is made to delete keys for deleted files. + for f in os.listdir(DUMP_DIR): + if f.endswith('.textproto'): + f_base, _ = os.path.splitext(f) + with open(join(DUMP_DIR, f_base + '.key'), 'r') as file: + key = KEY_PREFIX + file.read().strip() + with open(join(DUMP_DIR, f), 'r') as file: + value_text_proto = file.read() + value_encoded = protoc_util.encode_proto( + value_text_proto, 'feedstore.Record', ROOT_DIR, STORAGE_PROTO) + db.put(key.encode(), value_encoded) + db.close() + adb_push_db() + + +if not args.reverse: + dump() +else: + load() diff --git a/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py b/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py index 14cb23e915e..ef07d256c37 100755 --- a/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py +++ b/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py @@ -15,10 +15,13 @@ Usage example: --source_file /tmp/original.textpb """ +import base64 import glob import os import protoc_util import subprocess +import sys +import urllib.parse from absl import app from absl import flags @@ -26,9 +29,16 @@ from absl import flags DEFAULT_MESSAGE = 'feedwire.Response' FLAGS = flags.FLAGS -FLAGS = flags.FLAGS flags.DEFINE_string('chromium_path', '', 'The path of your chromium depot.') -flags.DEFINE_string('output_file', '', 'The target output binary file path.') +flags.DEFINE_string( + 'output_file', + '', + 'The target output file path. If not set, writes to stdout.') +flags.DEFINE_string( + 'output_format', + 'bin', + 'When encoding text to binary, this may be set to base64 to encode output ' + + 'suitable for URLs.') flags.DEFINE_string('source_file', '', 'The source proto file, in textpb format, path.') flags.DEFINE_string('message', @@ -37,38 +47,45 @@ flags.DEFINE_string('message', flags.DEFINE_string('direction', 'forward', 'Set --direction=reverse to convert binary to text.') -COMPONENT_FEED_PROTO_PATH = 'components/feed/core/proto' +COMPONENT_FEED_PROTO_PATH = 'components/feed/core/proto/v2' -def text_to_binary(): - with open(FLAGS.source_file, mode='r') as file: - value_text_proto = file.read() +def read_input(): + if FLAGS.source_file: + with open(FLAGS.source_file, mode='r') as file: + return file.read() + return sys.stdin.buffer.read() - encoded = protoc_util.encode_proto(value_text_proto, FLAGS.message, +def text_to_binary(): + encoded = protoc_util.encode_proto(read_input(), FLAGS.message, FLAGS.chromium_path, COMPONENT_FEED_PROTO_PATH) - with open(FLAGS.output_file, mode='wb') as file: - file.write(encoded) -def binary_to_text(): - with open(FLAGS.source_file, mode='rb') as file: - value_text_proto = file.read() + if FLAGS.output_format == 'base64': + encoded = urllib.parse.quote( + base64.urlsafe_b64encode(encoded).decode('utf-8')).encode('utf-8') + + if FLAGS.output_file: + with open(FLAGS.output_file, mode='wb') as file: + file.write(encoded) + else: + sys.stdout.buffer.write(encoded) - encoded = protoc_util.decode_proto(value_text_proto, FLAGS.message, +def binary_to_text(): + encoded = protoc_util.decode_proto(read_input(), FLAGS.message, FLAGS.chromium_path, COMPONENT_FEED_PROTO_PATH) - with open(FLAGS.output_file, mode='w') as file: - file.write(encoded) + if FLAGS.output_file: + with open(FLAGS.output_file, mode='w') as file: + file.write(encoded) + else: + print(encoded) def main(argv): if len(argv) > 1: - raise app.UsageError('Too many command-line arguments.') + raise app.UsageError('Too many arguments. Unknown: ' + ' '.join(argv[1:])) if not FLAGS.chromium_path: raise app.UsageError('chromium_path flag must be set.') - if not FLAGS.source_file: - raise app.UsageError('source_file flag must be set.') - if not FLAGS.output_file: - raise app.UsageError('output_file flag must be set.') if FLAGS.direction != 'forward' and FLAGS.direction != 'reverse': raise app.UsageError('direction must be forward or reverse') diff --git a/chromium/components/feed/core/v2/types.cc b/chromium/components/feed/core/v2/types.cc new file mode 100644 index 00000000000..b3b0267e3ee --- /dev/null +++ b/chromium/components/feed/core/v2/types.cc @@ -0,0 +1,102 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/types.h" + +#include <utility> + +#include "base/base64.h" +#include "base/pickle.h" +#include "base/strings/string_number_conversions.h" +#include "components/feed/core/v2/public/types.h" + +// Note: This file contains implementation for both types.h and public/types.h. +// because our build system will not allow multiple types.cc files in the +// same target. + +namespace feed { +namespace { + +void PickleDebugStreamData(const DebugStreamData& data, base::Pickle* pickle) { + pickle->WriteInt(DebugStreamData::kVersion); + pickle->WriteBool(data.fetch_info.has_value()); + if (data.fetch_info) { + pickle->WriteInt(data.fetch_info->status_code); + pickle->WriteUInt64(data.fetch_info->fetch_duration.InMilliseconds()); + pickle->WriteUInt64((data.fetch_info->fetch_time - base::Time::UnixEpoch()) + .InMilliseconds()); + pickle->WriteString(data.fetch_info->bless_nonce); + pickle->WriteString(data.fetch_info->base_request_url.spec()); + } + pickle->WriteString(data.load_stream_status); +} + +base::Optional<DebugStreamData> UnpickleDebugStreamData( + base::PickleIterator iterator) { + DebugStreamData result; + int version; + if (!iterator.ReadInt(&version) || version != DebugStreamData::kVersion) + return base::nullopt; + bool has_fetch_info; + if (!iterator.ReadBool(&has_fetch_info)) + return base::nullopt; + if (has_fetch_info) { + NetworkResponseInfo fetch_info; + uint64_t fetch_duration_ms; + uint64_t fetch_time_ms; + std::string base_request_url; + if (!(iterator.ReadInt(&fetch_info.status_code) && + iterator.ReadUInt64(&fetch_duration_ms) && + iterator.ReadUInt64(&fetch_time_ms) && + iterator.ReadString(&fetch_info.bless_nonce) && + iterator.ReadString(&base_request_url))) + return base::nullopt; + fetch_info.fetch_duration = + base::TimeDelta::FromMilliseconds(fetch_duration_ms); + fetch_info.fetch_time = base::TimeDelta::FromMilliseconds(fetch_time_ms) + + base::Time::UnixEpoch(); + fetch_info.base_request_url = GURL(base_request_url); + result.fetch_info = std::move(fetch_info); + } + if (!iterator.ReadString(&result.load_stream_status)) + return base::nullopt; + return result; +} + +} // namespace + +std::string ToString(ContentRevision c) { + return base::NumberToString(c.value()); +} + +ContentRevision ToContentRevision(const std::string& str) { + uint32_t value; + if (!base::StringToUint(str, &value)) + return {}; + return ContentRevision(value); +} + +std::string SerializeDebugStreamData(const DebugStreamData& data) { + base::Pickle pickle; + PickleDebugStreamData(data, &pickle); + const uint8_t* pickle_data_ptr = static_cast<const uint8_t*>(pickle.data()); + return base::Base64Encode( + base::span<const uint8_t>(pickle_data_ptr, pickle.size())); +} + +base::Optional<DebugStreamData> DeserializeDebugStreamData( + base::StringPiece base64_encoded) { + std::string binary_data; + if (!base::Base64Decode(base64_encoded, &binary_data)) + return base::nullopt; + base::Pickle pickle(binary_data.data(), binary_data.size()); + return UnpickleDebugStreamData(base::PickleIterator(pickle)); +} + +DebugStreamData::DebugStreamData() = default; +DebugStreamData::~DebugStreamData() = default; +DebugStreamData::DebugStreamData(const DebugStreamData&) = default; +DebugStreamData& DebugStreamData::operator=(const DebugStreamData&) = default; + +} // namespace feed diff --git a/chromium/components/feed/core/v2/types.h b/chromium/components/feed/core/v2/types.h new file mode 100644 index 00000000000..4b6bf0b0238 --- /dev/null +++ b/chromium/components/feed/core/v2/types.h @@ -0,0 +1,39 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TYPES_H_ +#define COMPONENTS_FEED_CORE_V2_TYPES_H_ + +#include <string> + +#include "base/util/type_safety/id_type.h" +#include "components/feed/core/v2/public/types.h" + +namespace feed { + +// Make sure public types are included here too. +// See components/feed/core/v2/public/types.h. +using ::feed::ChromeInfo; +using ::feed::EphemeralChangeId; + +// Uniquely identifies a revision of a |feedstore::Content|. If Content changes, +// it is assigned a new revision number. +using ContentRevision = util::IdTypeU32<class ContentRevisionClass>; + +// ID for a stored pending action. +using LocalActionId = util::IdType32<class LocalActionIdClass>; + +std::string ToString(ContentRevision c); +ContentRevision ToContentRevision(const std::string& str); + +// Metadata sent with Feed requests. +struct RequestMetadata { + ChromeInfo chrome_info; + std::string language_tag; + DisplayMetrics display_metrics; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TYPES_H_ diff --git a/chromium/components/feed/core/v2/types_unittest.cc b/chromium/components/feed/core/v2/types_unittest.cc new file mode 100644 index 00000000000..c7dcfd33eba --- /dev/null +++ b/chromium/components/feed/core/v2/types_unittest.cc @@ -0,0 +1,63 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/types.h" + +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { +DebugStreamData MakeDebugStreamData() { + NetworkResponseInfo fetch_info; + fetch_info.status_code = 200; + fetch_info.fetch_duration = base::TimeDelta::FromSeconds(4); + fetch_info.fetch_time = + base::Time::UnixEpoch() + base::TimeDelta::FromMinutes(200); + fetch_info.bless_nonce = "nonce"; + fetch_info.base_request_url = GURL("https://www.google.com"); + + DebugStreamData data; + data.fetch_info = fetch_info; + data.load_stream_status = "loaded OK"; + return data; +} +} // namespace + +TEST(DebugStreamData, CanSerialize) { + const DebugStreamData test_data = MakeDebugStreamData(); + const auto serialized = SerializeDebugStreamData(test_data); + base::Optional<DebugStreamData> result = + DeserializeDebugStreamData(serialized); + ASSERT_TRUE(result); + + EXPECT_EQ(SerializeDebugStreamData(*result), serialized); + + ASSERT_TRUE(result->fetch_info); + EXPECT_EQ(test_data.fetch_info->status_code, result->fetch_info->status_code); + EXPECT_EQ(test_data.fetch_info->fetch_duration, + result->fetch_info->fetch_duration); + EXPECT_EQ(test_data.fetch_info->fetch_time, result->fetch_info->fetch_time); + EXPECT_EQ(test_data.fetch_info->bless_nonce, result->fetch_info->bless_nonce); + EXPECT_EQ(test_data.fetch_info->base_request_url, + result->fetch_info->base_request_url); + EXPECT_EQ(test_data.load_stream_status, result->load_stream_status); +} + +TEST(DebugStreamData, CanSerializeWithoutFetchInfo) { + DebugStreamData input = MakeDebugStreamData(); + input.fetch_info = base::nullopt; + + const auto serialized = SerializeDebugStreamData(input); + base::Optional<DebugStreamData> result = + DeserializeDebugStreamData(serialized); + ASSERT_TRUE(result); + + EXPECT_EQ(SerializeDebugStreamData(*result), serialized); +} + +TEST(DebugStreamData, FailsDeserializationGracefully) { + ASSERT_EQ(base::nullopt, DeserializeDebugStreamData({})); +} + +} // namespace feed diff --git a/chromium/components/feed/feed_feature_list.cc b/chromium/components/feed/feed_feature_list.cc index b8f44049912..f6b51cb172a 100644 --- a/chromium/components/feed/feed_feature_list.cc +++ b/chromium/components/feed/feed_feature_list.cc @@ -30,4 +30,8 @@ const base::Feature kInterestFeedFeedback{"InterestFeedFeedback", const base::Feature kReportFeedUserActions{"ReportFeedUserActions", base::FEATURE_DISABLED_BY_DEFAULT}; +const base::Feature kInterestFeedV2{"InterestFeedV2", + base::FEATURE_DISABLED_BY_DEFAULT}; + + } // namespace feed diff --git a/chromium/components/feed/feed_feature_list.h b/chromium/components/feed/feed_feature_list.h index fe5e1141bbf..f26edc66376 100644 --- a/chromium/components/feed/feed_feature_list.h +++ b/chromium/components/feed/feed_feature_list.h @@ -25,9 +25,11 @@ extern const base::Feature kInterestFeedNotifications; extern const base::Feature kInterestFeedFeedback; // Indicates if user card clicks and views in Chrome's feed should be reported -// for personalization. +// for personalization. Also enables the feed header menu to manage the feed. extern const base::Feature kReportFeedUserActions; +extern const base::Feature kInterestFeedV2; + } // namespace feed #endif // COMPONENTS_FEED_FEED_FEATURE_LIST_H_ |