diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-07-16 11:45:35 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-07-17 08:59:23 +0000 |
commit | 552906b0f222c5d5dd11b9fd73829d510980461a (patch) | |
tree | 3a11e6ed0538a81dd83b20cf3a4783e297f26d91 /chromium/components/feed | |
parent | 1b05827804eaf047779b597718c03e7d38344261 (diff) | |
download | qtwebengine-chromium-552906b0f222c5d5dd11b9fd73829d510980461a.tar.gz |
BASELINE: Update Chromium to 83.0.4103.122
Change-Id: Ie3a82f5bb0076eec2a7c6a6162326b4301ee291e
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/feed')
149 files changed, 8760 insertions, 579 deletions
diff --git a/chromium/components/feed/BUILD.gn b/chromium/components/feed/BUILD.gn index 4cc4b9650dc..b5c7af06453 100644 --- a/chromium/components/feed/BUILD.gn +++ b/chromium/components/feed/BUILD.gn @@ -19,15 +19,15 @@ static_library("feature_list") { "feed_feature_list.h", ] - deps = [ - "//base", - ] + deps = [ "//base" ] } source_set("unit_tests") { testonly = true deps = [ "core:core_unit_tests", + "core/v2:core_unit_tests", + "//components/feed/core/common:core_common_unit_tests", ] if (!is_ios) { diff --git a/chromium/components/feed/content/BUILD.gn b/chromium/components/feed/content/BUILD.gn index 86922f20148..2c66bca0be8 100644 --- a/chromium/components/feed/content/BUILD.gn +++ b/chromium/components/feed/content/BUILD.gn @@ -28,9 +28,7 @@ source_set("feed_content") { source_set("content_unit_tests") { testonly = true - sources = [ - "feed_offline_host_unittest.cc", - ] + sources = [ "feed_offline_host_unittest.cc" ] deps = [ ":feed_content", diff --git a/chromium/components/feed/core/BUILD.gn b/chromium/components/feed/core/BUILD.gn index 01ca5f49ed1..a2e1aa17519 100644 --- a/chromium/components/feed/core/BUILD.gn +++ b/chromium/components/feed/core/BUILD.gn @@ -28,26 +28,22 @@ source_set("feed_core") { "feed_networking_host.h", "feed_scheduler_host.cc", "feed_scheduler_host.h", - "pref_names.cc", - "pref_names.h", - "refresh_throttler.cc", - "refresh_throttler.h", "time_serialization.cc", "time_serialization.h", - "user_classifier.cc", - "user_classifier.h", ] public_deps = [ "//base", "//components/feed:feature_list", "//components/feed/core/proto", + "//components/feed/core/shared_prefs:feed_shared_prefs", "//components/leveldb_proto", "//net", "//ui/base/mojom:mojom", ] deps = [ + "//components/feed/core/common:feed_core_common", "//components/prefs", "//components/signin/public/identity_manager", "//components/variations", @@ -63,9 +59,7 @@ source_set("feed_core") { if (is_android) { java_cpp_enum("feed_core_java_enums_srcjar") { - sources = [ - "feed_scheduler_host.h", - ] + sources = [ "feed_scheduler_host.h" ] } } @@ -79,14 +73,13 @@ source_set("core_unit_tests") { "feed_logging_metrics_unittest.cc", "feed_networking_host_unittest.cc", "feed_scheduler_host_unittest.cc", - "refresh_throttler_unittest.cc", - "user_classifier_unittest.cc", ] deps = [ ":feed_core", "//base", "//base/test:test_support", + "//components/feed/core/common:feed_core_common", "//components/leveldb_proto:test_support", "//components/prefs:test_support", "//components/signin/public/identity_manager:test_support", diff --git a/chromium/components/feed/core/common/BUILD.gn b/chromium/components/feed/core/common/BUILD.gn new file mode 100644 index 00000000000..61b32e02cb4 --- /dev/null +++ b/chromium/components/feed/core/common/BUILD.gn @@ -0,0 +1,51 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +if (is_android) { + import("//build/config/android/rules.gni") +} + +source_set("feed_core_common") { + sources = [ + "enums.h", + "pref_names.cc", + "pref_names.h", + "refresh_throttler.cc", + "refresh_throttler.h", + "user_classifier.cc", + "user_classifier.h", + ] + deps = [ "//components/prefs" ] + + public_deps = [ + "//base", + "//components/feed:feature_list", + "//components/feed/core/proto", + ] +} + +source_set("core_common_unit_tests") { + testonly = true + sources = [ + "refresh_throttler_unittest.cc", + "user_classifier_unittest.cc", + ] + + deps = [ + ":feed_core_common", + "//base", + "//base/test:test_support", + "//components/leveldb_proto:test_support", + "//components/prefs:test_support", + "//components/signin/public/identity_manager:test_support", + "//components/variations:test_support", + "//components/web_resource", + "//net:test_support", + "//services/network:test_support", + "//services/network/public/cpp", + "//services/network/public/mojom", + "//third_party/zlib/google:compression_utils", + "//ui/gfx:test_support", + ] +} diff --git a/chromium/components/feed/core/common/README.md b/chromium/components/feed/core/common/README.md new file mode 100644 index 00000000000..09538074edb --- /dev/null +++ b/chromium/components/feed/core/common/README.md @@ -0,0 +1 @@ +This directory contains code common to feed v2 and v1. diff --git a/chromium/components/feed/core/common/enums.h b/chromium/components/feed/core/common/enums.h new file mode 100644 index 00000000000..23fc0ea27c1 --- /dev/null +++ b/chromium/components/feed/core/common/enums.h @@ -0,0 +1,52 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_COMMON_ENUMS_H_ +#define COMPONENTS_FEED_CORE_COMMON_ENUMS_H_ + +// This file contains enumerations common to Feed v1 and v2. + +namespace feed { + +// The TriggerType enum specifies values for the events that can trigger +// refreshing articles. When adding values, be certain to also update the +// corresponding definition in enums.xml. +enum class TriggerType { + kNtpShown = 0, + kForegrounded = 1, + kFixedTimer = 2, + kMaxValue = kFixedTimer +}; + +// Different groupings of usage. A user will belong to exactly one of these at +// any given point in time. Can change at runtime. +enum class UserClass { + kRareSuggestionsViewer, // Almost never opens surfaces that show + // suggestions, like the NTP. + kActiveSuggestionsViewer, // Frequently shown suggestions, but does not + // usually open them. + kActiveSuggestionsConsumer, // Frequently opens news articles. +}; + +// Enum for the status of the refresh, reported through UMA. +// If any new values are added, update FeedSchedulerRefreshStatus in +// enums.xml. +// These values are persisted to logs. Entries should not be renumbered and +// numeric values should never be reused. +enum class ShouldRefreshResult { + kShouldRefresh = 0, + kDontRefreshOutstandingRequest = 1, + kDontRefreshTriggerDisabled = 2, + kDontRefreshNetworkOffline = 3, + kDontRefreshEulaNotAccepted = 4, + kDontRefreshArticlesHidden = 5, + kDontRefreshRefreshSuppressed = 6, + kDontRefreshNotStale = 7, + kDontRefreshRefreshThrottled = 8, + kMaxValue = kDontRefreshRefreshThrottled, +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_COMMON_ENUMS_H_ diff --git a/chromium/components/feed/core/pref_names.cc b/chromium/components/feed/core/common/pref_names.cc index d6d779d20dc..1abd2fc4d1d 100644 --- a/chromium/components/feed/core/pref_names.cc +++ b/chromium/components/feed/core/common/pref_names.cc @@ -2,19 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/pref_names.h" +#include "components/feed/core/common/pref_names.h" -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/user_classifier.h" #include "components/prefs/pref_registry_simple.h" namespace feed { namespace prefs { -const char kEnableSnippets[] = "ntp_snippets.enable"; - -const char kArticlesListVisible[] = "ntp_snippets.list_visible"; - const char kBackgroundRefreshPeriod[] = "feed.background_refresh_period"; const char kLastFetchAttemptTime[] = "feed.last_fetch_attempt"; @@ -35,6 +31,11 @@ const char kUserClassifierLastTimeToUseSuggestions[] = const char kHostOverrideHost[] = "feed.host_override.host"; const char kHostOverrideBlessNonce[] = "feed.host_override.bless_nonce"; +const char kThrottlerRequestCountListPrefName[] = + "feedv2.request_throttler.request_counts"; +const char kThrottlerLastRequestTime[] = + "feedv2.request_throttler.last_request_time"; + } // namespace prefs void RegisterProfilePrefs(PrefRegistrySimple* registry) { @@ -45,6 +46,9 @@ void RegisterProfilePrefs(PrefRegistrySimple* registry) { registry->RegisterTimePref(prefs::kLastFetchAttemptTime, base::Time()); registry->RegisterTimeDeltaPref(prefs::kBackgroundRefreshPeriod, base::TimeDelta()); + registry->RegisterListPref(feed::prefs::kThrottlerRequestCountListPrefName); + registry->RegisterTimePref(feed::prefs::kThrottlerLastRequestTime, + base::Time()); UserClassifier::RegisterProfilePrefs(registry); } diff --git a/chromium/components/feed/core/pref_names.h b/chromium/components/feed/core/common/pref_names.h index 85898c1a03d..f3bc8f04ccb 100644 --- a/chromium/components/feed/core/pref_names.h +++ b/chromium/components/feed/core/common/pref_names.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_FEED_CORE_PREF_NAMES_H_ -#define COMPONENTS_FEED_CORE_PREF_NAMES_H_ +#ifndef COMPONENTS_FEED_CORE_COMMON_PREF_NAMES_H_ +#define COMPONENTS_FEED_CORE_COMMON_PREF_NAMES_H_ class PrefRegistrySimple; @@ -11,14 +11,6 @@ namespace feed { namespace prefs { -// The pref name for whether suggested articles is allowed at all. When false, -// all Feed Java objects will be destroyed/nulled. Typically set by policy. -extern const char kEnableSnippets[]; - -// The pref name for whether the suggested articles section is expanded or -// collapsed. Only when it is expanded are the articles themselves visible. -extern const char kArticlesListVisible[]; - // The pref name for the period of time between background refreshes. extern const char kBackgroundRefreshPeriod[]; @@ -50,10 +42,17 @@ extern const char kHostOverrideHost[]; // The pref name for the feed host override auth token. extern const char kHostOverrideBlessNonce[]; +// The following prefs are used only by v2. + +// The pref name for the request throttler counts. +extern const char kThrottlerRequestCountListPrefName[]; +// The pref name for the request throttler's last request time. +extern const char kThrottlerLastRequestTime[]; + } // namespace prefs void RegisterProfilePrefs(PrefRegistrySimple* registry); } // namespace feed -#endif // COMPONENTS_FEED_CORE_PREF_NAMES_H_ +#endif // COMPONENTS_FEED_CORE_COMMON_PREF_NAMES_H_ diff --git a/chromium/components/feed/core/refresh_throttler.cc b/chromium/components/feed/core/common/refresh_throttler.cc index 223ffd34293..b121d1ac1e6 100644 --- a/chromium/components/feed/core/refresh_throttler.cc +++ b/chromium/components/feed/core/common/refresh_throttler.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/refresh_throttler.h" +#include "components/feed/core/common/refresh_throttler.h" #include <limits> #include <set> @@ -14,7 +14,7 @@ #include "base/metrics/histogram_base.h" #include "base/strings/stringprintf.h" #include "base/time/clock.h" -#include "components/feed/core/pref_names.h" +#include "components/feed/core/common/pref_names.h" #include "components/feed/feed_feature_list.h" #include "components/prefs/pref_service.h" @@ -34,23 +34,22 @@ enum class RequestStatus { // When adding a new type here, extend also the "RequestThrottlerTypes" // <histogram_suffixes> in histograms.xml with the |name| string. First value in // the pair is the name, second is the default requests per day. -std::pair<std::string, int> GetThrottlerParams( - UserClassifier::UserClass user_class) { +std::pair<std::string, int> GetThrottlerParams(UserClass user_class) { switch (user_class) { - case UserClassifier::UserClass::kRareSuggestionsViewer: + case UserClass::kRareSuggestionsViewer: return {"SuggestionFetcherRareNTPUser", 5}; - case UserClassifier::UserClass::kActiveSuggestionsViewer: + case UserClass::kActiveSuggestionsViewer: return {"SuggestionFetcherActiveNTPUser", 20}; - case UserClassifier::UserClass::kActiveSuggestionsConsumer: + case UserClass::kActiveSuggestionsConsumer: return {"SuggestionFetcherActiveSuggestionsConsumer", 20}; } } } // namespace -RefreshThrottler::RefreshThrottler(UserClassifier::UserClass user_class, +RefreshThrottler::RefreshThrottler(UserClass user_class, PrefService* pref_service, - base::Clock* clock) + const base::Clock* clock) : pref_service_(pref_service), clock_(clock) { DCHECK(pref_service); DCHECK(clock); diff --git a/chromium/components/feed/core/refresh_throttler.h b/chromium/components/feed/core/common/refresh_throttler.h index 69c37ee249f..d8547991fe9 100644 --- a/chromium/components/feed/core/refresh_throttler.h +++ b/chromium/components/feed/core/common/refresh_throttler.h @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_FEED_CORE_REFRESH_THROTTLER_H_ -#define COMPONENTS_FEED_CORE_REFRESH_THROTTLER_H_ +#ifndef COMPONENTS_FEED_CORE_COMMON_REFRESH_THROTTLER_H_ +#define COMPONENTS_FEED_CORE_COMMON_REFRESH_THROTTLER_H_ #include <string> #include "base/macros.h" -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/user_classifier.h" class PrefService; @@ -28,9 +28,9 @@ namespace feed { // - "NewTabPage.RequestThrottler.PerDay_|name|" - the daily count of requests. class RefreshThrottler { public: - RefreshThrottler(UserClassifier::UserClass user_class, + RefreshThrottler(UserClass user_class, PrefService* pref_service, - base::Clock* clock); + const base::Clock* clock); // Returns whether quota is available for another request, persists the usage // of said quota, and reports this information to UMA. @@ -51,7 +51,7 @@ class RefreshThrottler { PrefService* pref_service_; // Used to access current time, injected for testing. - base::Clock* clock_; + const base::Clock* clock_; // The name used by this throttler, based off UserClass, which will be used as // a suffix when constructing histogram or finch param names. @@ -71,4 +71,4 @@ class RefreshThrottler { } // namespace feed -#endif // COMPONENTS_FEED_CORE_REFRESH_THROTTLER_H_ +#endif // COMPONENTS_FEED_CORE_COMMON_REFRESH_THROTTLER_H_ diff --git a/chromium/components/feed/core/refresh_throttler_unittest.cc b/chromium/components/feed/core/common/refresh_throttler_unittest.cc index 7b26fb64656..74ab0d46e9b 100644 --- a/chromium/components/feed/core/refresh_throttler_unittest.cc +++ b/chromium/components/feed/core/common/refresh_throttler_unittest.cc @@ -2,15 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/refresh_throttler.h" +#include "components/feed/core/common/refresh_throttler.h" #include <limits> #include <memory> #include "base/test/scoped_feature_list.h" #include "base/test/simple_test_clock.h" -#include "components/feed/core/pref_names.h" -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/common/user_classifier.h" #include "components/feed/feed_feature_list.h" #include "components/prefs/testing_pref_service.h" #include "testing/gtest/include/gtest/gtest.h" @@ -38,8 +38,7 @@ class RefreshThrottlerTest : public testing::Test { {{"quota_SuggestionFetcherActiveNTPUser", "2"}}); throttler_ = std::make_unique<RefreshThrottler>( - UserClassifier::UserClass::kActiveSuggestionsViewer, &test_prefs_, - &test_clock_); + UserClass::kActiveSuggestionsViewer, &test_prefs_, &test_clock_); } protected: diff --git a/chromium/components/feed/core/user_classifier.cc b/chromium/components/feed/core/common/user_classifier.cc index c3061321216..e286833e528 100644 --- a/chromium/components/feed/core/user_classifier.cc +++ b/chromium/components/feed/core/common/user_classifier.cc @@ -2,10 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/user_classifier.h" #include <algorithm> #include <cfloat> +#include <cmath> #include <string> #include "base/metrics/histogram_macros.h" @@ -13,11 +14,9 @@ #include "base/stl_util.h" #include "base/strings/string_number_conversions.h" #include "base/time/clock.h" -#include "components/feed/core/pref_names.h" -#include "components/feed/feed_feature_list.h" +#include "components/feed/core/common/pref_names.h" #include "components/prefs/pref_registry_simple.h" #include "components/prefs/pref_service.h" -#include "components/variations/variations_associated_data.h" namespace feed { @@ -25,30 +24,27 @@ namespace { // The discount rate for computing the discounted-average rates. Must be // strictly larger than 0 and strictly smaller than 1! -const double kDiscountRatePerDay = 0.25; -const char kDiscountRatePerDayParam[] = "user_classifier_discount_rate_per_day"; +constexpr double kDiscountRatePerDay = 0.25; +static_assert(kDiscountRatePerDay > 0 && kDiscountRatePerDay < 1, + "invalid value"); +// Compute discount_rate_per_hour such that +// kDiscountRatePerDay = 1 - e^{-kDiscountRatePerHour * 24}. +const double kDiscountRatePerHour = + std::log(1.0 / (1.0 - kDiscountRatePerDay)) / 24.0; // Never consider any larger interval than this (so that extreme situations such // as losing your phone or going for a long offline vacation do not skew the // average too much). -// When overriding via variation parameters, it is better to use smaller values -// than |kMaxHours| as this it the maximum value reported in the histograms. const double kMaxHours = 7 * 24; -const char kMaxHoursParam[] = "user_classifier_max_hours"; // Ignore events within |kMinHours| hours since the last event (|kMinHours| is // the length of the browsing session where subsequent events of the same type // do not count again). const double kMinHours = 0.5; -const char kMinHoursParam[] = "user_classifier_min_hours"; // Classification constants. const double kActiveConsumerClicksAtLeastOncePerHours = 96; -const char kActiveConsumerClicksAtLeastOncePerHoursParam[] = - "user_classifier_active_consumer_clicks_at_least_once_per_hours"; const double kRareUserViewsAtMostOncePerHours = 96; -const char kRareUserViewsAtMostOncePerHoursParam[] = - "user_classifier_rare_user_views_at_most_once_per_hours"; // Histograms for logging the estimated average hours to next event. During // launch these must match legacy histogram names. @@ -61,67 +57,35 @@ const char kHistogramAverageHoursToUseSuggestions[] = const UserClassifier::Event kEvents[] = { UserClassifier::Event::kSuggestionsViewed, UserClassifier::Event::kSuggestionsUsed}; - -// Arrays of pref names, indexed by Event's int value. -const char* kRateKeys[] = {prefs::kUserClassifierAverageSuggestionsViwedPerHour, - prefs::kUserClassifierAverageSuggestionsUsedPerHour}; -const char* kLastTimeKeys[] = {prefs::kUserClassifierLastTimeToViewSuggestions, - prefs::kUserClassifierLastTimeToUseSuggestions}; - -// Default lengths of the intervals for new users for the events. -const double kInitialHoursBetweenEvents[] = {24, 120}; -const char* kInitialHoursBetweenEventsParams[] = { - "user_classifier_default_interval_suggestions_viewed", - "user_classifier_default_interval_suggestions_used"}; - -// This verifies that each of the arrays has exactly the same number of values -// as the number of enum values in UserClassifier::Event. These arrays are all -// indexed by the integer value of UserClassifier::Event values. static_assert(base::size(kEvents) == - static_cast<int>(UserClassifier::Event::kMaxValue) + 1 && - base::size(kRateKeys) == - static_cast<int>(UserClassifier::Event::kMaxValue) + 1 && - base::size(kLastTimeKeys) == - static_cast<int>(UserClassifier::Event::kMaxValue) + 1 && - base::size(kInitialHoursBetweenEvents) == - static_cast<int>(UserClassifier::Event::kMaxValue) + 1 && - base::size(kInitialHoursBetweenEventsParams) == - static_cast<int>(UserClassifier::Event::kMaxValue) + 1, - "Fill in info for all event types."); - -// Computes the discount rate. -double GetDiscountRatePerHour() { - double discount_rate_per_day = variations::GetVariationParamByFeatureAsDouble( - kInterestFeedContentSuggestions, kDiscountRatePerDayParam, - kDiscountRatePerDay); - // Check for illegal values. - if (discount_rate_per_day <= 0 || discount_rate_per_day >= 1) { - DLOG(WARNING) << "Illegal value " << discount_rate_per_day - << " for the parameter " << kDiscountRatePerDayParam - << " (must be strictly between 0 and 1; the default " - << kDiscountRatePerDay << " is used, instead)."; - discount_rate_per_day = kDiscountRatePerDay; - } - // Compute discount_rate_per_hour such that - // discount_rate_per_day = 1 - e^{-discount_rate_per_hour * 24}. - return std::log(1.0 / (1.0 - discount_rate_per_day)) / 24.0; -} + static_cast<int>(UserClassifier::Event::kMaxValue) + 1, + "kEvents should have all enum values."); -double GetInitialHoursBetweenEvents(UserClassifier::Event event) { - return variations::GetVariationParamByFeatureAsDouble( - kInterestFeedContentSuggestions, - kInitialHoursBetweenEventsParams[static_cast<int>(event)], - kInitialHoursBetweenEvents[static_cast<int>(event)]); +const char* GetRateKey(UserClassifier::Event event) { + switch (event) { + case UserClassifier::Event::kSuggestionsViewed: + return prefs::kUserClassifierAverageSuggestionsViwedPerHour; + case UserClassifier::Event::kSuggestionsUsed: + return prefs::kUserClassifierAverageSuggestionsUsedPerHour; + } } -double GetMinHours() { - return variations::GetVariationParamByFeatureAsDouble( - kInterestFeedContentSuggestions, kMinHoursParam, kMinHours); +const char* GetLastTimeKey(UserClassifier::Event event) { + switch (event) { + case UserClassifier::Event::kSuggestionsViewed: + return prefs::kUserClassifierLastTimeToViewSuggestions; + case UserClassifier::Event::kSuggestionsUsed: + return prefs::kUserClassifierLastTimeToUseSuggestions; + } } -double GetMaxHours() { - return variations::GetVariationParamByFeatureAsDouble( - kInterestFeedContentSuggestions, kMaxHoursParam, kMaxHours); +double GetInitialHoursBetweenEvents(UserClassifier::Event event) { + switch (event) { + case UserClassifier::Event::kSuggestionsViewed: + return 24; + case UserClassifier::Event::kSuggestionsUsed: + return 120; + } } // Returns the new value of the rate using its |old_value|, assuming @@ -177,22 +141,9 @@ double GetRateForEstimateHoursBetweenEvents(double estimate_hours, } // namespace -UserClassifier::UserClassifier(PrefService* pref_service, base::Clock* clock) - : pref_service_(pref_service), - clock_(clock), - discount_rate_per_hour_(GetDiscountRatePerHour()), - min_hours_(GetMinHours()), - max_hours_(GetMaxHours()), - active_consumer_clicks_at_least_once_per_hours_( - variations::GetVariationParamByFeatureAsDouble( - kInterestFeedContentSuggestions, - kActiveConsumerClicksAtLeastOncePerHoursParam, - kActiveConsumerClicksAtLeastOncePerHours)), - rare_viewer_opens_surface_at_most_once_per_hours_( - variations::GetVariationParamByFeatureAsDouble( - kInterestFeedContentSuggestions, - kRareUserViewsAtMostOncePerHoursParam, - kRareUserViewsAtMostOncePerHours)) { +UserClassifier::UserClassifier(PrefService* pref_service, + const base::Clock* clock) + : pref_service_(pref_service), clock_(clock) { // The pref_service_ can be null in tests. if (!pref_service_) { return; @@ -214,25 +165,19 @@ UserClassifier::~UserClassifier() = default; // static void UserClassifier::RegisterProfilePrefs(PrefRegistrySimple* registry) { - double discount_rate = GetDiscountRatePerHour(); - double min_hours = GetMinHours(); - double max_hours = GetMaxHours(); - for (Event event : kEvents) { double default_rate = GetRateForEstimateHoursBetweenEvents( - GetInitialHoursBetweenEvents(event), discount_rate, min_hours, - max_hours); - registry->RegisterDoublePref(kRateKeys[static_cast<int>(event)], - default_rate); - registry->RegisterTimePref(kLastTimeKeys[static_cast<int>(event)], - base::Time()); + GetInitialHoursBetweenEvents(event), kDiscountRatePerHour, kMinHours, + kMaxHours); + registry->RegisterDoublePref(GetRateKey(event), default_rate); + registry->RegisterTimePref(GetLastTimeKey(event), base::Time()); } } void UserClassifier::OnEvent(Event event) { double metric_value = UpdateRateOnEvent(event); - double avg = GetEstimateHoursBetweenEvents( - metric_value, discount_rate_per_hour_, min_hours_, max_hours_); + double avg = GetEstimateHoursBetweenEvents(metric_value, kDiscountRatePerHour, + kMinHours, kMaxHours); // We use kMaxHours as the max value below as the maximum value for the // histograms must be constant. switch (event) { @@ -249,23 +194,23 @@ void UserClassifier::OnEvent(Event event) { double UserClassifier::GetEstimatedAvgTime(Event event) const { double rate = GetUpToDateRate(event); - return GetEstimateHoursBetweenEvents(rate, discount_rate_per_hour_, - min_hours_, max_hours_); + return GetEstimateHoursBetweenEvents(rate, kDiscountRatePerHour, kMinHours, + kMaxHours); } -UserClassifier::UserClass UserClassifier::GetUserClass() const { +UserClass UserClassifier::GetUserClass() const { // The pref_service_ can be null in tests. if (!pref_service_) { return UserClass::kActiveSuggestionsViewer; } if (GetEstimatedAvgTime(Event::kSuggestionsViewed) >= - rare_viewer_opens_surface_at_most_once_per_hours_) { + kRareUserViewsAtMostOncePerHours) { return UserClass::kRareSuggestionsViewer; } if (GetEstimatedAvgTime(Event::kSuggestionsUsed) <= - active_consumer_clicks_at_least_once_per_hours_) { + kActiveConsumerClicksAtLeastOncePerHours) { return UserClass::kActiveSuggestionsConsumer; } @@ -304,9 +249,9 @@ double UserClassifier::UpdateRateOnEvent(Event event) { } double hours_since_last_time = - std::min(max_hours_, GetHoursSinceLastTime(event)); + std::min(kMaxHours, GetHoursSinceLastTime(event)); // Ignore events within the same "browsing session". - if (hours_since_last_time < min_hours_) { + if (hours_since_last_time < kMinHours) { return GetUpToDateRate(event); } @@ -315,7 +260,7 @@ double UserClassifier::UpdateRateOnEvent(Event event) { double rate = GetRate(event); // Add 1 to the discounted rate as the event has happened right now. double new_rate = - 1 + DiscountRate(rate, hours_since_last_time, discount_rate_per_hour_); + 1 + DiscountRate(rate, hours_since_last_time, kDiscountRatePerHour); SetRate(event, new_rate); return new_rate; } @@ -326,11 +271,11 @@ double UserClassifier::GetUpToDateRate(Event event) const { return 0; } - double hours_since_last_time = - std::min(max_hours_, GetHoursSinceLastTime(event)); + const double hours_since_last_time = + std::min(kMaxHours, GetHoursSinceLastTime(event)); - double rate = GetRate(event); - return DiscountRate(rate, hours_since_last_time, discount_rate_per_hour_); + const double rate = GetRate(event); + return DiscountRate(rate, hours_since_last_time, kDiscountRatePerHour); } double UserClassifier::GetHoursSinceLastTime(Event event) const { @@ -339,29 +284,28 @@ double UserClassifier::GetHoursSinceLastTime(Event event) const { } base::TimeDelta since_last_time = - clock_->Now() - - pref_service_->GetTime(kLastTimeKeys[static_cast<int>(event)]); + clock_->Now() - pref_service_->GetTime(GetLastTimeKey(event)); return since_last_time.InSecondsF() / 3600; } bool UserClassifier::HasLastTime(Event event) const { - return pref_service_->HasPrefPath(kLastTimeKeys[static_cast<int>(event)]); + return pref_service_->HasPrefPath(GetLastTimeKey(event)); } void UserClassifier::SetLastTimeToNow(Event event) { - pref_service_->SetTime(kLastTimeKeys[static_cast<int>(event)], clock_->Now()); + pref_service_->SetTime(GetLastTimeKey(event), clock_->Now()); } double UserClassifier::GetRate(Event event) const { - return pref_service_->GetDouble(kRateKeys[static_cast<int>(event)]); + return pref_service_->GetDouble(GetRateKey(event)); } void UserClassifier::SetRate(Event event, double rate) { - pref_service_->SetDouble(kRateKeys[static_cast<int>(event)], rate); + pref_service_->SetDouble(GetRateKey(event), rate); } void UserClassifier::ClearRate(Event event) { - pref_service_->ClearPref(kRateKeys[static_cast<int>(event)]); + pref_service_->ClearPref(GetRateKey(event)); } } // namespace feed diff --git a/chromium/components/feed/core/user_classifier.h b/chromium/components/feed/core/common/user_classifier.h index e1c1ed3e1e7..154df8b88d9 100644 --- a/chromium/components/feed/core/user_classifier.h +++ b/chromium/components/feed/core/common/user_classifier.h @@ -2,13 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef COMPONENTS_FEED_CORE_USER_CLASSIFIER_H_ -#define COMPONENTS_FEED_CORE_USER_CLASSIFIER_H_ +#ifndef COMPONENTS_FEED_CORE_COMMON_USER_CLASSIFIER_H_ +#define COMPONENTS_FEED_CORE_COMMON_USER_CLASSIFIER_H_ #include <memory> #include <string> #include "base/macros.h" +#include "components/feed/core/common/enums.h" class PrefRegistrySimple; class PrefService; @@ -24,16 +25,6 @@ namespace feed { // Based on these long-term user rates, it classifies the user in a UserClass. class UserClassifier { public: - // Different groupings of usage. A user will belong to exactly one of these at - // any given point in time. Can change at runtime. - enum class UserClass { - kRareSuggestionsViewer, // Almost never opens surfaces that show - // suggestions, like the NTP. - kActiveSuggestionsViewer, // Frequently shown suggestions, but does not - // usually open them. - kActiveSuggestionsConsumer, // Frequently opens news articles. - }; - // For estimating the average length of the intervals between two successive // events, we keep a simple frequency model, a single value that we call // "rate" below. @@ -55,8 +46,8 @@ class UserClassifier { }; // The provided |pref_service| may be nullptr in unit-tests. - UserClassifier(PrefService* pref_service, base::Clock* clock); - ~UserClassifier(); + UserClassifier(PrefService* pref_service, const base::Clock* clock); + virtual ~UserClassifier(); // Registers profile prefs for all rates. Called from pref_names.cc. static void RegisterProfilePrefs(PrefRegistrySimple* registry); @@ -70,7 +61,8 @@ class UserClassifier { double GetEstimatedAvgTime(Event event) const; // Return the classification of the current user. - UserClass GetUserClass() const; + // Virtual for testing. + virtual UserClass GetUserClass() const; std::string GetUserClassDescriptionForDebugging() const; // Resets the classification (emulates a fresh upgrade / install). @@ -95,20 +87,11 @@ class UserClassifier { void ClearRate(Event event); PrefService* pref_service_; - base::Clock* clock_; - - // Params of the rate. - const double discount_rate_per_hour_; - const double min_hours_; - const double max_hours_; - - // Params of the classification. - const double active_consumer_clicks_at_least_once_per_hours_; - const double rare_viewer_opens_surface_at_most_once_per_hours_; + const base::Clock* clock_; DISALLOW_COPY_AND_ASSIGN(UserClassifier); }; } // namespace feed -#endif // COMPONENTS_FEED_CORE_USER_CLASSIFIER_H_ +#endif // COMPONENTS_FEED_CORE_COMMON_USER_CLASSIFIER_H_ diff --git a/chromium/components/feed/core/user_classifier_unittest.cc b/chromium/components/feed/core/common/user_classifier_unittest.cc index 84aa9a6cb59..6d0eebb0eee 100644 --- a/chromium/components/feed/core/user_classifier_unittest.cc +++ b/chromium/components/feed/core/common/user_classifier_unittest.cc @@ -2,17 +2,15 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/user_classifier.h" #include <memory> #include <string> #include <utility> #include "base/test/metrics/histogram_tester.h" -#include "base/test/scoped_feature_list.h" #include "base/test/simple_test_clock.h" #include "base/time/time.h" -#include "components/feed/feed_feature_list.h" #include "components/prefs/pref_registry_simple.h" #include "components/prefs/testing_pref_service.h" #include "testing/gmock/include/gmock/gmock.h" @@ -59,7 +57,7 @@ class FeedUserClassifierTest : public testing::Test { TEST_F(FeedUserClassifierTest, ShouldBeActiveSuggestionsViewerInitially) { UserClassifier* user_classifier = CreateUserClassifier(); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); + Eq(UserClass::kActiveSuggestionsViewer)); } TEST_F(FeedUserClassifierTest, @@ -69,7 +67,7 @@ TEST_F(FeedUserClassifierTest, // After one click still only an active user. user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); + Eq(UserClass::kActiveSuggestionsViewer)); // After a few more clicks, become an active consumer. for (int i = 0; i < 5; i++) { @@ -77,31 +75,7 @@ TEST_F(FeedUserClassifierTest, user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); } EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsConsumer)); -} - -TEST_F(FeedUserClassifierTest, - ShouldBecomeActiveSuggestionsConsumerByClickingOftenWithDecreasedParam) { - // Increase the param to one half. - base::test::ScopedFeatureList scoped_feature_list; - scoped_feature_list.InitAndEnableFeatureWithParameters( - kInterestFeedContentSuggestions, - {{"user_classifier_active_consumer_clicks_at_least_once_per_hours", - "36"}}); - UserClassifier* user_classifier = CreateUserClassifier(); - - // After two clicks still only an active user. - user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); - test_clock()->Advance(base::TimeDelta::FromHours(1)); - user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); - EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); - - // One more click to become an active consumer. - test_clock()->Advance(base::TimeDelta::FromHours(1)); - user_classifier->OnEvent(UserClassifier::Event::kSuggestionsUsed); - EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsConsumer)); + Eq(UserClass::kActiveSuggestionsConsumer)); } TEST_F(FeedUserClassifierTest, @@ -111,32 +85,12 @@ TEST_F(FeedUserClassifierTest, // After two days of waiting still an active user. test_clock()->Advance(base::TimeDelta::FromDays(2)); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); + Eq(UserClass::kActiveSuggestionsViewer)); // Two more days to become a rare user. test_clock()->Advance(base::TimeDelta::FromDays(2)); EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kRareSuggestionsViewer)); -} - -TEST_F(FeedUserClassifierTest, - ShouldBecomeRareSuggestionsViewerByNoActivityWithDecreasedParam) { - // Decrease the param to one half. - base::test::ScopedFeatureList scoped_feature_list; - scoped_feature_list.InitAndEnableFeatureWithParameters( - kInterestFeedContentSuggestions, - {{"user_classifier_rare_user_views_at_most_once_per_hours", "48"}}); - UserClassifier* user_classifier = CreateUserClassifier(); - - // After one days of waiting still an active user. - test_clock()->Advance(base::TimeDelta::FromDays(1)); - EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kActiveSuggestionsViewer)); - - // One more day to become a rare user. - test_clock()->Advance(base::TimeDelta::FromDays(1)); - EXPECT_THAT(user_classifier->GetUserClass(), - Eq(UserClassifier::UserClass::kRareSuggestionsViewer)); + Eq(UserClass::kRareSuggestionsViewer)); } class FeedUserClassifierEventTest @@ -220,31 +174,6 @@ TEST_P(FeedUserClassifierEventTest, ShouldIgnoreSubsequentEventsForHalfAnHour) { EXPECT_THAT(user_classifier->GetEstimatedAvgTime(event), Lt(old_rate)); } -TEST_P(FeedUserClassifierEventTest, - ShouldIgnoreSubsequentEventsWithIncreasedLimit) { - UserClassifier::Event event = GetParam().first; - // Increase the min_hours to 1.0, i.e. 60 minutes. - base::test::ScopedFeatureList scoped_feature_list; - scoped_feature_list.InitAndEnableFeatureWithParameters( - kInterestFeedContentSuggestions, {{"user_classifier_min_hours", "1.0"}}); - UserClassifier* user_classifier = CreateUserClassifier(); - - // The initial event. - user_classifier->OnEvent(event); - // Subsequent events get ignored for the next 60 minutes. - for (int i = 0; i < 11; i++) { - test_clock()->Advance(base::TimeDelta::FromMinutes(5)); - double old_rate = user_classifier->GetEstimatedAvgTime(event); - user_classifier->OnEvent(event); - EXPECT_THAT(user_classifier->GetEstimatedAvgTime(event), Eq(old_rate)); - } - // An event 60 minutes after the initial event is finally not ignored. - test_clock()->Advance(base::TimeDelta::FromMinutes(5)); - double old_rate = user_classifier->GetEstimatedAvgTime(event); - user_classifier->OnEvent(event); - EXPECT_THAT(user_classifier->GetEstimatedAvgTime(event), Lt(old_rate)); -} - TEST_P(FeedUserClassifierEventTest, ShouldCapDelayBetweenEvents) { UserClassifier::Event event = GetParam().first; UserClassifier* user_classifier = CreateUserClassifier(); @@ -267,33 +196,6 @@ TEST_P(FeedUserClassifierEventTest, ShouldCapDelayBetweenEvents) { Eq(rate_after_a_year)); } -TEST_P(FeedUserClassifierEventTest, - ShouldCapDelayBetweenEventsWithDecreasedLimit) { - UserClassifier::Event event = GetParam().first; - // Decrease the max_hours to 72, i.e. 3 days. - base::test::ScopedFeatureList scoped_feature_list; - scoped_feature_list.InitAndEnableFeatureWithParameters( - kInterestFeedContentSuggestions, {{"user_classifier_max_hours", "72"}}); - UserClassifier* user_classifier = CreateUserClassifier(); - - // The initial event. - user_classifier->OnEvent(event); - // Wait for an insane amount of time - test_clock()->Advance(base::TimeDelta::FromDays(365)); - user_classifier->OnEvent(event); - double rate_after_a_year = user_classifier->GetEstimatedAvgTime(event); - - // Now repeat the same with s/one year/two days. - user_classifier->ClearClassificationForDebugging(); - user_classifier->OnEvent(event); - test_clock()->Advance(base::TimeDelta::FromDays(3)); - user_classifier->OnEvent(event); - - // The results should be the same. - EXPECT_THAT(user_classifier->GetEstimatedAvgTime(event), - Eq(rate_after_a_year)); -} - INSTANTIATE_TEST_SUITE_P( All, // An empty prefix for the parametrized tests names (no need to // distinguish the only instance we make here). diff --git a/chromium/components/feed/core/feed_content_database.cc b/chromium/components/feed/core/feed_content_database.cc index 4ea33bd29b1..df1d0588a9f 100644 --- a/chromium/components/feed/core/feed_content_database.cc +++ b/chromium/components/feed/core/feed_content_database.cc @@ -12,6 +12,7 @@ #include "base/strings/string_util.h" #include "base/system/sys_info.h" #include "base/task/post_task.h" +#include "base/task/thread_pool.h" #include "base/threading/thread_task_runner_handle.h" #include "components/feed/core/feed_content_mutation.h" #include "components/feed/core/feed_content_operation.h" @@ -53,9 +54,8 @@ FeedContentDatabase::FeedContentDatabase( leveldb_proto::ProtoDatabaseProvider* proto_database_provider, const base::FilePath& database_folder) : database_status_(InitStatus::kNotInitialized), - task_runner_( - base::CreateSequencedTaskRunner({base::ThreadPool(), base::MayBlock(), - base::TaskPriority::USER_VISIBLE})), + task_runner_(base::ThreadPool::CreateSequencedTaskRunner( + {base::MayBlock(), base::TaskPriority::USER_VISIBLE})), storage_database_(proto_database_provider->GetDB<ContentStorageProto>( leveldb_proto::ProtoDbType::FEED_CONTENT_DATABASE, database_folder.AppendASCII(kContentDatabaseFolder), diff --git a/chromium/components/feed/core/feed_content_database_unittest.cc b/chromium/components/feed/core/feed_content_database_unittest.cc index d455fe6e270..ab9e3431c77 100644 --- a/chromium/components/feed/core/feed_content_database_unittest.cc +++ b/chromium/components/feed/core/feed_content_database_unittest.cc @@ -7,6 +7,7 @@ #include <map> #include "base/bind.h" +#include "base/task/thread_pool.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/task_environment.h" #include "components/feed/core/feed_content_mutation.h" @@ -57,9 +58,8 @@ class FeedContentDatabaseTest : public testing::Test { auto storage_db = std::make_unique<FakeDB<ContentStorageProto>>(&content_db_storage_); - task_runner_ = - base::CreateSequencedTaskRunner({base::ThreadPool(), base::MayBlock(), - base::TaskPriority::USER_VISIBLE}); + task_runner_ = base::ThreadPool::CreateSequencedTaskRunner( + {base::MayBlock(), base::TaskPriority::USER_VISIBLE}); content_db_ = storage_db.get(); feed_db_ = std::make_unique<FeedContentDatabase>(std::move(storage_db), diff --git a/chromium/components/feed/core/feed_journal_database.cc b/chromium/components/feed/core/feed_journal_database.cc index 64539b6d404..63a4540af73 100644 --- a/chromium/components/feed/core/feed_journal_database.cc +++ b/chromium/components/feed/core/feed_journal_database.cc @@ -10,6 +10,7 @@ #include "base/metrics/histogram_macros.h" #include "base/system/sys_info.h" #include "base/task/post_task.h" +#include "base/task/thread_pool.h" #include "base/threading/thread_task_runner_handle.h" #include "components/feed/core/feed_journal_mutation.h" #include "components/feed/core/feed_journal_operation.h" @@ -38,9 +39,8 @@ FeedJournalDatabase::FeedJournalDatabase( leveldb_proto::ProtoDatabaseProvider* proto_database_provider, const base::FilePath& database_folder) : database_status_(InitStatus::kNotInitialized), - task_runner_( - base::CreateSequencedTaskRunner({base::ThreadPool(), base::MayBlock(), - base::TaskPriority::USER_VISIBLE})), + task_runner_(base::ThreadPool::CreateSequencedTaskRunner( + {base::MayBlock(), base::TaskPriority::USER_VISIBLE})), storage_database_(proto_database_provider->GetDB<JournalStorageProto>( leveldb_proto::ProtoDbType::FEED_JOURNAL_DATABASE, database_folder.AppendASCII(kJournalDatabaseFolder), diff --git a/chromium/components/feed/core/feed_journal_database_unittest.cc b/chromium/components/feed/core/feed_journal_database_unittest.cc index 6f2a238835c..cd0b2debc01 100644 --- a/chromium/components/feed/core/feed_journal_database_unittest.cc +++ b/chromium/components/feed/core/feed_journal_database_unittest.cc @@ -8,6 +8,7 @@ #include <utility> #include "base/bind.h" +#include "base/task/thread_pool.h" #include "base/test/metrics/histogram_tester.h" #include "base/test/task_environment.h" #include "components/feed/core/feed_journal_mutation.h" @@ -63,9 +64,8 @@ class FeedJournalDatabaseTest : public testing::Test { auto storage_db = std::make_unique<FakeDB<JournalStorageProto>>(&journal_db_storage_); - task_runner_ = - base::CreateSequencedTaskRunner({base::ThreadPool(), base::MayBlock(), - base::TaskPriority::USER_VISIBLE}); + task_runner_ = base::ThreadPool::CreateSequencedTaskRunner( + {base::MayBlock(), base::TaskPriority::USER_VISIBLE}); journal_db_ = storage_db.get(); feed_db_ = std::make_unique<FeedJournalDatabase>(std::move(storage_db), diff --git a/chromium/components/feed/core/feed_logging_metrics_unittest.cc b/chromium/components/feed/core/feed_logging_metrics_unittest.cc index 100a458def0..3463ccc3d13 100644 --- a/chromium/components/feed/core/feed_logging_metrics_unittest.cc +++ b/chromium/components/feed/core/feed_logging_metrics_unittest.cc @@ -8,8 +8,8 @@ #include "base/test/metrics/histogram_tester.h" #include "base/test/simple_test_clock.h" #include "base/time/time.h" -#include "components/feed/core/pref_names.h" -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/common/user_classifier.h" #include "components/prefs/testing_pref_service.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -22,8 +22,6 @@ using testing::SizeIs; namespace feed { namespace { -GURL kVisitedUrl("http://visited_url.com/"); - // Fixed "now" to make tests more deterministic. char kNowString[] = "2018-06-11 15:41"; @@ -39,9 +37,15 @@ enum FeedActionType { DOWNLOAD = 5, }; +// TODO(https://crbug.com/1042727): Fix test GURL scoping and remove this getter +// function. +GURL VisitedUrl() { + return GURL("http://visited_url.com/"); +} + void CheckURLVisit(const GURL& url, FeedLoggingMetrics::CheckURLVisitCallback callback) { - if (url == kVisitedUrl) { + if (url == VisitedUrl()) { std::move(callback).Run(true); } else { std::move(callback).Run(false); @@ -178,7 +182,7 @@ TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionWindowOpened) { TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionDismissedCommitIfVisited) { base::HistogramTester histogram_tester; - feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, kVisitedUrl, + feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, VisitedUrl(), true); EXPECT_THAT(histogram_tester.GetAllSamples( "NewTabPage.ContentSuggestions.DismissedVisited.Commit"), @@ -198,7 +202,7 @@ TEST_F(FeedLoggingMetricsTest, TEST_F(FeedLoggingMetricsTest, ShouldLogOnSuggestionDismissedUndoIfUndoDismissAndVisited) { base::HistogramTester histogram_tester; - feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, kVisitedUrl, + feed_logging_metrics()->OnSuggestionDismissed(/*position=*/10, VisitedUrl(), false); EXPECT_THAT(histogram_tester.GetAllSamples( "NewTabPage.ContentSuggestions.DismissedVisited.Undo"), diff --git a/chromium/components/feed/core/feed_networking_host.cc b/chromium/components/feed/core/feed_networking_host.cc index 04c6848fac8..e6ab4ac9321 100644 --- a/chromium/components/feed/core/feed_networking_host.cc +++ b/chromium/components/feed/core/feed_networking_host.cc @@ -14,12 +14,13 @@ #include "base/time/tick_clock.h" #include "base/time/time.h" #include "base/values.h" -#include "components/feed/core/pref_names.h" +#include "components/feed/core/common/pref_names.h" #include "components/feed/feed_feature_list.h" #include "components/prefs/pref_service.h" #include "components/signin/public/identity_manager/access_token_info.h" #include "components/signin/public/identity_manager/identity_manager.h" #include "components/signin/public/identity_manager/primary_account_access_token_fetcher.h" +#include "components/signin/public/identity_manager/scope_set.h" #include "components/variations/net/variations_http_headers.h" #include "google_apis/gaia/google_service_auth_error.h" #include "net/base/load_flags.h" @@ -77,7 +78,8 @@ class NetworkFetch { signin::AccessTokenInfo access_token_info); void StartLoader(); std::unique_ptr<network::SimpleURLLoader> MakeLoader(); - void SetRequestHeaders(network::ResourceRequest* request) const; + void SetRequestHeaders(bool has_request_body, + network::ResourceRequest* request) const; void PopulateRequestBody(network::SimpleURLLoader* loader); void OnSimpleLoaderComplete(std::unique_ptr<std::string> response); @@ -150,7 +152,7 @@ void NetworkFetch::Start(FeedNetworkingHost::ResponseCallback done_callback) { } void NetworkFetch::StartAccessTokenFetch() { - identity::ScopeSet scopes{kAuthenticationScope}; + signin::ScopeSet scopes{kAuthenticationScope}; // It's safe to pass base::Unretained(this) since deleting the token fetcher // will prevent the callback from being completed. token_fetcher_ = std::make_unique<signin::PrimaryAccountAccessTokenFetcher>( @@ -229,10 +231,10 @@ std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader() { if (host_overridden_) { resource_request->credentials_mode = network::mojom::CredentialsMode::kInclude; - resource_request->site_for_cookies = url; + resource_request->site_for_cookies = net::SiteForCookies::FromUrl(url); } - SetRequestHeaders(resource_request.get()); + SetRequestHeaders(!request_body_.empty(), resource_request.get()); auto simple_loader = network::SimpleURLLoader::Create( std::move(resource_request), traffic_annotation); @@ -243,10 +245,13 @@ std::unique_ptr<network::SimpleURLLoader> NetworkFetch::MakeLoader() { return simple_loader; } -void NetworkFetch::SetRequestHeaders(network::ResourceRequest* request) const { - request->headers.SetHeader(net::HttpRequestHeaders::kContentType, - kContentType); - request->headers.SetHeader(kContentEncoding, kGzip); +void NetworkFetch::SetRequestHeaders(bool has_request_body, + network::ResourceRequest* request) const { + if (has_request_body) { + request->headers.SetHeader(net::HttpRequestHeaders::kContentType, + kContentType); + request->headers.SetHeader(kContentEncoding, kGzip); + } variations::SignedIn signed_in_status = variations::SignedIn::kNo; if (!access_token_.empty()) { @@ -306,7 +311,7 @@ void NetworkFetch::OnSimpleLoaderComplete( status_code = simple_loader_->ResponseInfo()->headers->response_code(); if (status_code == net::HTTP_UNAUTHORIZED) { - identity::ScopeSet scopes{kAuthenticationScope}; + signin::ScopeSet scopes{kAuthenticationScope}; CoreAccountId account_id = identity_manager_->GetPrimaryAccountId(); if (!account_id.empty()) { identity_manager_->RemoveAccessTokenFromCache(account_id, scopes, diff --git a/chromium/components/feed/core/feed_networking_host_unittest.cc b/chromium/components/feed/core/feed_networking_host_unittest.cc index 51901925d3d..9361ec2dd7d 100644 --- a/chromium/components/feed/core/feed_networking_host_unittest.cc +++ b/chromium/components/feed/core/feed_networking_host_unittest.cc @@ -14,7 +14,7 @@ #include "base/test/scoped_feature_list.h" #include "base/test/simple_test_tick_clock.h" #include "base/test/task_environment.h" -#include "components/feed/core/pref_names.h" +#include "components/feed/core/common/pref_names.h" #include "components/feed/feed_feature_list.h" #include "components/prefs/testing_pref_service.h" #include "components/signin/public/identity_manager/identity_test_environment.h" @@ -246,7 +246,6 @@ TEST_F(FeedNetworkingHostTest, ShouldSetHeadersCorrectly) { MockResponseDoneCallback done_callback; net::HttpRequestHeaders headers; base::RunLoop interceptor_run_loop; - base::HistogramTester histogram_tester; test_factory()->SetInterceptor( base::BindLambdaForTesting([&](const network::ResourceRequest& request) { @@ -254,9 +253,9 @@ TEST_F(FeedNetworkingHostTest, ShouldSetHeadersCorrectly) { interceptor_run_loop.Quit(); })); - SendRequestAndRespond("http://foobar.com/feed", "POST", "", "", - net::HTTP_OK, network::URLLoaderCompletionStatus(), - &done_callback); + SendRequestAndRespond("http://foobar.com/feed", "POST", "body", "", + net::HTTP_OK, network::URLLoaderCompletionStatus(), + &done_callback); std::string content_encoding; std::string authorization; @@ -267,6 +266,23 @@ TEST_F(FeedNetworkingHostTest, ShouldSetHeadersCorrectly) { EXPECT_EQ(authorization, "Bearer access_token"); } +TEST_F(FeedNetworkingHostTest, ShouldNotSendContentEncodingForEmptyBody) { + MockResponseDoneCallback done_callback; + net::HttpRequestHeaders headers; + base::RunLoop interceptor_run_loop; + + test_factory()->SetInterceptor( + base::BindLambdaForTesting([&](const network::ResourceRequest& request) { + headers = request.headers; + interceptor_run_loop.Quit(); + })); + + SendRequestAndRespond("http://foobar.com/feed", "GET", "", "", net::HTTP_OK, + network::URLLoaderCompletionStatus(), &done_callback); + + EXPECT_FALSE(headers.HasHeader("content-encoding")); +} + TEST_F(FeedNetworkingHostTest, ShouldReportSizeHistograms) { std::string uncompressed_request_string(2048, 'a'); std::string response_string(1024, 'b'); diff --git a/chromium/components/feed/core/feed_scheduler_host.cc b/chromium/components/feed/core/feed_scheduler_host.cc index 627cf56386b..5b492c90e98 100644 --- a/chromium/components/feed/core/feed_scheduler_host.cc +++ b/chromium/components/feed/core/feed_scheduler_host.cc @@ -16,7 +16,8 @@ #include "base/strings/stringprintf.h" #include "base/time/clock.h" #include "base/time/time.h" -#include "components/feed/core/pref_names.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/shared_prefs/pref_names.h" #include "components/feed/core/time_serialization.h" #include "components/feed/feed_feature_list.h" #include "components/prefs/pref_service.h" @@ -27,9 +28,6 @@ namespace feed { namespace { -using TriggerType = FeedSchedulerHost::TriggerType; -using UserClass = UserClassifier::UserClass; - // Enum for the relation between boolean fields the Feed and host both track. // Reported through UMA and must match the corresponding definition in // enums.xml @@ -41,21 +39,16 @@ enum class FeedHostMismatch { kMaxValue = kBothAreSet, }; -// Copies boolean args into temps to avoid evaluating them multiple times. -#define UMA_HISTOGRAM_MISMATCH(name, feed_is_set, host_is_set) \ - do { \ - bool copied_feed_is_set = feed_is_set; \ - bool copied_host_is_set = host_is_set; \ - FeedHostMismatch status = FeedHostMismatch::kNeitherAreSet; \ - if (copied_feed_is_set && copied_host_is_set) { \ - status = FeedHostMismatch::kBothAreSet; \ - } else if (copied_feed_is_set) { \ - status = FeedHostMismatch::kFeedIsSetOnly; \ - } else if (copied_host_is_set) { \ - status = FeedHostMismatch::kHostIsSetOnly; \ - } \ - UMA_HISTOGRAM_ENUMERATION(name, status); \ - } while (false); +FeedHostMismatch GetMismatch(bool feed_is_set, bool host_is_set) { + if (feed_is_set && host_is_set) { + return FeedHostMismatch::kBothAreSet; + } else if (feed_is_set) { + return FeedHostMismatch::kFeedIsSetOnly; + } else if (host_is_set) { + return FeedHostMismatch::kHostIsSetOnly; + } + return FeedHostMismatch::kNeitherAreSet; +} struct ParamPair { std::string name; @@ -136,15 +129,15 @@ void TryRun(base::OnceClosure closure) { } } -// Converts UserClassifier::UserClass to a string that corresponds to the +// Converts UserClass to a string that corresponds to the // entries in histogram suffix "UserClasses". -std::string UserClassToHistogramSuffix(UserClassifier::UserClass user_class) { +std::string UserClassToHistogramSuffix(UserClass user_class) { switch (user_class) { - case UserClassifier::UserClass::kRareSuggestionsViewer: + case UserClass::kRareSuggestionsViewer: return "RareNTPUser"; - case UserClassifier::UserClass::kActiveSuggestionsViewer: + case UserClass::kActiveSuggestionsViewer: return "ActiveNTPUser"; - case UserClassifier::UserClass::kActiveSuggestionsConsumer: + case UserClass::kActiveSuggestionsConsumer: return "ActiveSuggestionsConsumer"; } } @@ -155,7 +148,7 @@ std::string UserClassToHistogramSuffix(UserClassifier::UserClass user_class) { // because this method is only called as a result of a direct user interaction, // like opening the NTP or foregrounding the browser. void ReportAgeWithSuffix(const std::string& qualified_trigger, - UserClassifier::UserClass user_class, + UserClass user_class, base::TimeDelta sample) { std::string name = base::StringPrintf( "NewTabPage.ContentSuggestions.%s.%s", qualified_trigger.c_str(), @@ -165,10 +158,9 @@ void ReportAgeWithSuffix(const std::string& qualified_trigger, /*bucket_count=*/50); } -void ReportReasonForNotRefreshingByBehavior( - NativeRequestBehavior behavior, - FeedSchedulerHost::ShouldRefreshResult status) { - DCHECK_NE(status, FeedSchedulerHost::kShouldRefresh); +void ReportReasonForNotRefreshingByBehavior(NativeRequestBehavior behavior, + ShouldRefreshResult status) { + DCHECK_NE(status, ShouldRefreshResult::kShouldRefresh); switch (behavior) { case kNoRequestWithWait: UMA_HISTOGRAM_ENUMERATION( @@ -197,24 +189,23 @@ void ReportReasonForNotRefreshingByBehavior( } } -void ReportReasonForNotRefreshingByTrigger( - FeedSchedulerHost::TriggerType trigger_type, - FeedSchedulerHost::ShouldRefreshResult status) { - DCHECK_NE(status, FeedSchedulerHost::kShouldRefresh); +void ReportReasonForNotRefreshingByTrigger(TriggerType trigger_type, + ShouldRefreshResult status) { + DCHECK_NE(status, ShouldRefreshResult::kShouldRefresh); switch (trigger_type) { - case FeedSchedulerHost::TriggerType::kNtpShown: + case TriggerType::kNtpShown: UMA_HISTOGRAM_ENUMERATION( "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." "RequestByNtpShown", status); break; - case FeedSchedulerHost::TriggerType::kForegrounded: + case TriggerType::kForegrounded: UMA_HISTOGRAM_ENUMERATION( "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." "RequestByForegrounded", status); break; - case FeedSchedulerHost::TriggerType::kFixedTimer: + case TriggerType::kFixedTimer: UMA_HISTOGRAM_ENUMERATION( "ContentSuggestions.Feed.Scheduler.ShouldRefreshResult." "RequestByFixedTimer", @@ -240,18 +231,18 @@ FeedSchedulerHost::FeedSchedulerHost(PrefService* profile_prefs, eula_accepted_notifier_->Init(this); } - throttlers_.emplace(UserClassifier::UserClass::kRareSuggestionsViewer, - std::make_unique<RefreshThrottler>( - UserClassifier::UserClass::kRareSuggestionsViewer, - profile_prefs_, clock_)); - throttlers_.emplace(UserClassifier::UserClass::kActiveSuggestionsViewer, - std::make_unique<RefreshThrottler>( - UserClassifier::UserClass::kActiveSuggestionsViewer, - profile_prefs_, clock_)); - throttlers_.emplace(UserClassifier::UserClass::kActiveSuggestionsConsumer, - std::make_unique<RefreshThrottler>( - UserClassifier::UserClass::kActiveSuggestionsConsumer, - profile_prefs_, clock_)); + throttlers_.emplace( + UserClass::kRareSuggestionsViewer, + std::make_unique<RefreshThrottler>(UserClass::kRareSuggestionsViewer, + profile_prefs_, clock_)); + throttlers_.emplace( + UserClass::kActiveSuggestionsViewer, + std::make_unique<RefreshThrottler>(UserClass::kActiveSuggestionsViewer, + profile_prefs_, clock_)); + throttlers_.emplace( + UserClass::kActiveSuggestionsConsumer, + std::make_unique<RefreshThrottler>(UserClass::kActiveSuggestionsConsumer, + profile_prefs_, clock_)); } FeedSchedulerHost::~FeedSchedulerHost() = default; @@ -292,9 +283,10 @@ NativeRequestBehavior FeedSchedulerHost::ShouldSessionRequestData( // Both the Feed and the scheduler track if there are outstanding requests. // It's possible that this data gets out of sync. We treat the Feed as // authoritative and we change our values to match. - UMA_HISTOGRAM_MISMATCH("ContentSuggestions.Feed.Scheduler.OutstandingRequest", - has_outstanding_request, - !outstanding_request_until_.is_null()); + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.OutstandingRequest", + GetMismatch(has_outstanding_request, + !outstanding_request_until_.is_null())); if (has_outstanding_request == outstanding_request_until_.is_null()) { if (has_outstanding_request) { outstanding_request_until_ = @@ -312,8 +304,9 @@ NativeRequestBehavior FeedSchedulerHost::ShouldSessionRequestData( bool scheduler_thinks_has_content = !profile_prefs_->FindPreference(prefs::kLastFetchAttemptTime) ->IsDefaultValue(); - UMA_HISTOGRAM_MISMATCH("ContentSuggestions.Feed.Scheduler.HasContent", - has_content, scheduler_thinks_has_content); + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Scheduler.HasContent", + GetMismatch(has_content, scheduler_thinks_has_content)); if (has_content != scheduler_thinks_has_content) { if (has_content) { profile_prefs_->SetTime(prefs::kLastFetchAttemptTime, @@ -342,7 +335,7 @@ NativeRequestBehavior FeedSchedulerHost::ShouldSessionRequestData( NativeRequestBehavior behavior; ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kNtpShown); - if (kShouldRefresh == refresh_status) { + if (ShouldRefreshResult::kShouldRefresh == refresh_status) { if (!has_content) { behavior = kRequestWithWait; } else if (IsContentStale(content_creation_date_time)) { @@ -406,7 +399,7 @@ void FeedSchedulerHost::OnForegrounded() { DCHECK(refresh_callback_); ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kForegrounded); - if (kShouldRefresh == refresh_status) { + if (ShouldRefreshResult::kShouldRefresh == refresh_status) { refresh_callback_.Run(); } else { ReportReasonForNotRefreshingByTrigger(TriggerType::kForegrounded, @@ -426,7 +419,7 @@ void FeedSchedulerHost::OnFixedTimer(base::OnceClosure on_completion) { } ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kFixedTimer); - if (kShouldRefresh == refresh_status) { + if (ShouldRefreshResult::kShouldRefresh == refresh_status) { // There shouldn't typically be anything in |fixed_timer_completion_| right // now, but if there was, run it before we replace it. TryRun(std::move(fixed_timer_completion_)); @@ -476,7 +469,7 @@ bool FeedSchedulerHost::OnArticlesCleared(bool suppress_refreshes) { } ShouldRefreshResult refresh_status = ShouldRefresh(TriggerType::kNtpShown); - if (kShouldRefresh == refresh_status) { + if (ShouldRefreshResult::kShouldRefresh == refresh_status) { // Instead of using |refresh_callback_|, instead return our desire to // refresh back up to our caller. This allows more information to be given // all at once to the Feed which allows it to act more intelligently. @@ -509,41 +502,40 @@ void FeedSchedulerHost::OnEulaAccepted() { OnForegrounded(); } -FeedSchedulerHost::ShouldRefreshResult FeedSchedulerHost::ShouldRefresh( - TriggerType trigger) { +ShouldRefreshResult FeedSchedulerHost::ShouldRefresh(TriggerType trigger) { if (clock_->Now() < outstanding_request_until_) { DVLOG(2) << "Outstanding request stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshOutstandingRequest; + return ShouldRefreshResult::kDontRefreshOutstandingRequest; } if (base::Contains(disabled_triggers_, trigger)) { DVLOG(2) << "Disabled trigger stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshTriggerDisabled; + return ShouldRefreshResult::kDontRefreshTriggerDisabled; } if (net::NetworkChangeNotifier::IsOffline()) { DVLOG(2) << "Network is offline stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshNetworkOffline; + return ShouldRefreshResult::kDontRefreshNetworkOffline; } if (eula_accepted_notifier_ && !eula_accepted_notifier_->IsEulaAccepted()) { DVLOG(2) << "EULA not being accepted stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshEulaNotAccepted; + return ShouldRefreshResult::kDontRefreshEulaNotAccepted; } if (!profile_prefs_->GetBoolean(prefs::kArticlesListVisible)) { DVLOG(2) << "Articles being hidden stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshArticlesHidden; + return ShouldRefreshResult::kDontRefreshArticlesHidden; } base::TimeDelta attempt_age = clock_->Now() - profile_prefs_->GetTime(prefs::kLastFetchAttemptTime); - UserClassifier::UserClass user_class = user_classifier_.GetUserClass(); + UserClass user_class = user_classifier_.GetUserClass(); if (trigger == TriggerType::kNtpShown && !time_until_first_shown_trigger_reported_) { time_until_first_shown_trigger_reported_ = true; @@ -560,7 +552,7 @@ FeedSchedulerHost::ShouldRefreshResult FeedSchedulerHost::ShouldRefresh( if (clock_->Now() < suppress_refreshes_until_) { DVLOG(2) << "Refresh suppression until " << suppress_refreshes_until_ << " stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshRefreshSuppressed; + return ShouldRefreshResult::kDontRefreshRefreshSuppressed; } // https://crbug.com/988165: When kThrottleBackgroundFetches == false, skip @@ -569,7 +561,7 @@ FeedSchedulerHost::ShouldRefreshResult FeedSchedulerHost::ShouldRefresh( if (attempt_age < GetTriggerThreshold(trigger)) { DVLOG(2) << "Last attempt age of " << attempt_age << " stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshNotStale; + return ShouldRefreshResult::kDontRefreshNotStale; } auto throttlerIter = throttlers_.find(user_class); @@ -577,7 +569,7 @@ FeedSchedulerHost::ShouldRefreshResult FeedSchedulerHost::ShouldRefresh( !throttlerIter->second->RequestQuota()) { DVLOG(2) << "Throttler stopped refresh from trigger " << static_cast<int>(trigger); - return kDontRefreshRefreshThrottled; + return ShouldRefreshResult::kDontRefreshRefreshThrottled; } } @@ -602,7 +594,7 @@ FeedSchedulerHost::ShouldRefreshResult FeedSchedulerHost::ShouldRefresh( last_fetch_trigger_type_ = std::make_unique<TriggerType>(trigger); - return kShouldRefresh; + return ShouldRefreshResult::kShouldRefresh; } bool FeedSchedulerHost::IsContentStale(base::Time content_creation_date_time) { diff --git a/chromium/components/feed/core/feed_scheduler_host.h b/chromium/components/feed/core/feed_scheduler_host.h index 9df2d204d9e..e1e137586ec 100644 --- a/chromium/components/feed/core/feed_scheduler_host.h +++ b/chromium/components/feed/core/feed_scheduler_host.h @@ -13,8 +13,9 @@ #include "base/gtest_prod_util.h" #include "base/macros.h" #include "base/memory/weak_ptr.h" -#include "components/feed/core/refresh_throttler.h" -#include "components/feed/core/user_classifier.h" +#include "components/feed/core/common/enums.h" +#include "components/feed/core/common/refresh_throttler.h" +#include "components/feed/core/common/user_classifier.h" #include "components/web_resource/eula_accepted_notifier.h" class PrefService; @@ -49,33 +50,6 @@ enum NativeRequestBehavior { // content. class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { public: - // The TriggerType enum specifies values for the events that can trigger - // refreshing articles. When adding values, be certain to also update the - // corresponding definition in enums.xml. - enum class TriggerType { - kNtpShown = 0, - kForegrounded = 1, - kFixedTimer = 2, - kMaxValue = kFixedTimer - }; - - // Enum for the status of the refresh, reported through UMA. - // If any new values are added, update the corresponding definition in - // enums.xml. - // These values are persisted to logs. Entries should not be renumbered and - // numeric values should never be reused. - enum ShouldRefreshResult { - kShouldRefresh = 0, - kDontRefreshOutstandingRequest = 1, - kDontRefreshTriggerDisabled = 2, - kDontRefreshNetworkOffline = 3, - kDontRefreshEulaNotAccepted = 4, - kDontRefreshArticlesHidden = 5, - kDontRefreshRefreshSuppressed = 6, - kDontRefreshNotStale = 7, - kDontRefreshRefreshThrottled = 8, - kMaxValue = kDontRefreshRefreshThrottled, - }; FeedSchedulerHost(PrefService* profile_prefs, PrefService* local_state, @@ -224,8 +198,7 @@ class FeedSchedulerHost : web_resource::EulaAcceptedNotifier::Observer { // In the case the user transitions between user classes, hold onto a // throttler for any situation. - base::flat_map<UserClassifier::UserClass, std::unique_ptr<RefreshThrottler>> - throttlers_; + base::flat_map<UserClass, std::unique_ptr<RefreshThrottler>> throttlers_; // Status of the last fetch for debugging. int last_fetch_status_ = 0; diff --git a/chromium/components/feed/core/feed_scheduler_host_unittest.cc b/chromium/components/feed/core/feed_scheduler_host_unittest.cc index e0f1e40dbc6..33dfcf9aec6 100644 --- a/chromium/components/feed/core/feed_scheduler_host_unittest.cc +++ b/chromium/components/feed/core/feed_scheduler_host_unittest.cc @@ -13,10 +13,11 @@ #include "base/test/metrics/histogram_tester.h" #include "base/test/scoped_feature_list.h" #include "base/test/simple_test_clock.h" -#include "components/feed/core/pref_names.h" -#include "components/feed/core/refresh_throttler.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/common/refresh_throttler.h" +#include "components/feed/core/common/user_classifier.h" +#include "components/feed/core/shared_prefs/pref_names.h" #include "components/feed/core/time_serialization.h" -#include "components/feed/core/user_classifier.h" #include "components/feed/feed_feature_list.h" #include "components/prefs/pref_registry_simple.h" #include "components/prefs/testing_pref_service.h" @@ -58,7 +59,7 @@ class FeedSchedulerHostTest : public ::testing::Test { local_state()->registry()->RegisterBooleanPref(::prefs::kEulaAccepted, true); profile_prefs()->registry()->RegisterBooleanPref( - prefs::kArticlesListVisible, true); + feed::prefs::kArticlesListVisible, true); Time now; EXPECT_TRUE(Time::FromUTCString(kNowString, &now)); @@ -92,7 +93,7 @@ class FeedSchedulerHostTest : public ::testing::Test { // into kRareNtpUser classification. test_clock()->Advance(TimeDelta::FromDays(7)); - ASSERT_EQ(UserClassifier::UserClass::kRareSuggestionsViewer, + ASSERT_EQ(UserClass::kRareSuggestionsViewer, scheduler()->GetUserClassifierForDebugging()->GetUserClass()); } @@ -107,7 +108,7 @@ class FeedSchedulerHostTest : public ::testing::Test { // Depending on which events occurred over which period of time in the test // before this function was called, it may not necessarily be sufficient to // push the user into the active consumer class. - ASSERT_EQ(UserClassifier::UserClass::kActiveSuggestionsConsumer, + ASSERT_EQ(UserClass::kActiveSuggestionsConsumer, scheduler()->GetUserClassifierForDebugging()->GetUserClass()); } @@ -177,23 +178,22 @@ class FeedSchedulerHostTest : public ::testing::Test { TEST_F(FeedSchedulerHostTest, GetTriggerThreshold) { // Make sure that there is no missing configuration in the Cartesian product // of states between TriggerType and UserClass. - std::vector<FeedSchedulerHost::TriggerType> triggers = { - FeedSchedulerHost::TriggerType::kNtpShown, - FeedSchedulerHost::TriggerType::kForegrounded, - FeedSchedulerHost::TriggerType::kFixedTimer}; + std::vector<TriggerType> triggers = {TriggerType::kNtpShown, + TriggerType::kForegrounded, + TriggerType::kFixedTimer}; // Classification starts out as an active NTP user. - for (FeedSchedulerHost::TriggerType trigger : triggers) { + for (TriggerType trigger : triggers) { EXPECT_FALSE(scheduler()->GetTriggerThreshold(trigger).is_zero()); } ClassifyAsActiveSuggestionsConsumer(); - for (FeedSchedulerHost::TriggerType trigger : triggers) { + for (TriggerType trigger : triggers) { EXPECT_FALSE(scheduler()->GetTriggerThreshold(trigger).is_zero()); } ClassifyAsRareNtpUser(); - for (FeedSchedulerHost::TriggerType trigger : triggers) { + for (TriggerType trigger : triggers) { EXPECT_FALSE(scheduler()->GetTriggerThreshold(trigger).is_zero()); } } @@ -1034,14 +1034,14 @@ TEST_F(FeedSchedulerHostTest, RefreshThrottler) { TEST_F(FeedSchedulerHostTest, GetUserClassifierForDebuggingRareUser) { ClassifyAsRareNtpUser(); - EXPECT_EQ(UserClassifier::UserClass::kRareSuggestionsViewer, + EXPECT_EQ(UserClass::kRareSuggestionsViewer, scheduler()->GetUserClassifierForDebugging()->GetUserClass()); } TEST_F(FeedSchedulerHostTest, GetUserClassifierForDebuggingActiveConsumer) { ClassifyAsActiveSuggestionsConsumer(); - EXPECT_EQ(UserClassifier::UserClass::kActiveSuggestionsConsumer, + EXPECT_EQ(UserClass::kActiveSuggestionsConsumer, scheduler()->GetUserClassifierForDebugging()->GetUserClass()); } @@ -1069,19 +1069,19 @@ TEST_F(FeedSchedulerHostTest, GetLastFetchStatusForDebugging) { TEST_F(FeedSchedulerHostTest, GetLastFetchTriggerTypeForDebugging) { scheduler()->OnForegrounded(); - EXPECT_EQ(FeedSchedulerHost::TriggerType::kForegrounded, + EXPECT_EQ(TriggerType::kForegrounded, *scheduler()->GetLastFetchTriggerTypeForDebugging()); scheduler()->OnArticlesCleared(/*suppress_refreshes*/ false); - EXPECT_EQ(FeedSchedulerHost::TriggerType::kNtpShown, + EXPECT_EQ(TriggerType::kNtpShown, *scheduler()->GetLastFetchTriggerTypeForDebugging()); ClassifyAsActiveSuggestionsConsumer(); // Fixed timer at 48 hours. test_clock()->Advance(TimeDelta::FromHours(49)); scheduler()->OnFixedTimer(base::OnceClosure()); - EXPECT_EQ(FeedSchedulerHost::TriggerType::kFixedTimer, + EXPECT_EQ(TriggerType::kFixedTimer, *scheduler()->GetLastFetchTriggerTypeForDebugging()); } diff --git a/chromium/components/feed/core/proto/BUILD.gn b/chromium/components/feed/core/proto/BUILD.gn index 7ceb58bafa7..d9da12241c5 100644 --- a/chromium/components/feed/core/proto/BUILD.gn +++ b/chromium/components/feed/core/proto/BUILD.gn @@ -16,6 +16,44 @@ proto_library("proto") { ] } +proto_library("proto_v2") { + proto_in_dir = "../../../../" + sources = [ + "v2/store.proto", + "v2/ui.proto", + "v2/wire/action_payload.proto", + "v2/wire/action_request.proto", + "v2/wire/action_response.proto", + "v2/wire/capability.proto", + "v2/wire/client_info.proto", + "v2/wire/consistency_token.proto", + "v2/wire/content_id.proto", + "v2/wire/data_operation.proto", + "v2/wire/display_info.proto", + "v2/wire/duration.proto", + "v2/wire/expiration_info.proto", + "v2/wire/feature.proto", + "v2/wire/feed_action.proto", + "v2/wire/feed_action_request.proto", + "v2/wire/feed_action_response.proto", + "v2/wire/feed_id.proto", + "v2/wire/feed_query.proto", + "v2/wire/feed_request.proto", + "v2/wire/feed_response.proto", + "v2/wire/in_place_update_handle.proto", + "v2/wire/next_page_token.proto", + "v2/wire/payload_metadata.proto", + "v2/wire/render_data.proto", + "v2/wire/request.proto", + "v2/wire/response.proto", + "v2/wire/response_status_code.proto", + "v2/wire/stream_structure.proto", + "v2/wire/templates.proto", + "v2/wire/token.proto", + "v2/wire/version.proto", + ] +} + if (is_android) { proto_java_library("proto_java") { proto_path = "../../../../" @@ -75,4 +113,9 @@ if (is_android) { "wire/version.proto", ] } + + proto_java_library("proto_java_v2") { + proto_path = "../../../../" + sources = [ "v2/ui.proto" ] + } } diff --git a/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto b/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto index f506539af59..e66a0b818d7 100644 --- a/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto +++ b/chromium/components/feed/core/proto/libraries/api/internal/stream_data.proto @@ -72,8 +72,7 @@ message StreamSharedState { optional string content_id = 1; oneof share_state { // A Piet shared state item. - components.feed.core.proto.wire.PietSharedStateItem piet_shared_state_item = - 2; + feedwire1.PietSharedStateItem piet_shared_state_item = 2; } } @@ -157,7 +156,7 @@ message StreamPayload { // The consistency token used to ensure that we are recording actions to // the same server store. - components.feed.core.proto.wire.ConsistencyToken consistency_token = 9; + feedwire1.ConsistencyToken consistency_token = 9; } reserved 8; } @@ -228,7 +227,7 @@ message StreamUploadableAction { // When the action was recorded optional int64 timestamp_seconds = 4; - optional wire.ActionPayload payload = 6; + optional feedwire1.ActionPayload payload = 6; reserved 1, 5; // deprecated fields } diff --git a/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto b/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto index ed9766bdfa2..419ea251146 100644 --- a/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto +++ b/chromium/components/feed/core/proto/ui/action/ui_feed_action.proto @@ -30,7 +30,7 @@ message FeedAction { } // Metadata needed by the host to handle the action. -// Next Id: 9 +// Next Id: 19 message FeedActionMetadata { // The type of action, used by the host to perform any custom logic needed for // a specific type of action. @@ -57,6 +57,9 @@ message FeedActionMetadata { HIDE_ELEMENT = 13; SHOW_TOOLTIP = 14; NOT_INTERESTED_IN = 15; + SEE_SUGGESTED_SITES = 16; + SEND_FEEDBACK = 17; + MANAGE_INTERESTS = 18; reserved 9, 10; // Deprecated } optional Type type = 1; @@ -102,6 +105,10 @@ message OpenUrlData { // opening the url. Once this finishes, the client will attach to the url its // latest frequency token as the value of this query param. optional string consistency_token_query_param_name = 2; + // The content ID that was interacted with to cause a URL open. + optional feedwire1.ContentId content_id = 3; + // Roundtripped server data on a per-action level. + optional feedwire1.ActionPayload payload = 4; } // Data needed by Stream to open a context menu. @@ -114,19 +121,19 @@ message DismissData { // The ContentId needed by the server to suppress reshowing the dismissed // content. This will usually be the ContentId of the card which holds the // content, not the ContentId of the content itself. - optional components.feed.core.proto.wire.ContentId content_id = 1; + optional feedwire1.ContentId content_id = 1; // The DataOperations which are needed to actually perform the dismiss on the // client. This is typically a singleton list of a remove operation on the // Cluster that the content belongs to. - repeated components.feed.core.proto.wire.DataOperation data_operations = 2; + repeated feedwire1.DataOperation data_operations = 2; // Data used by the client to show a confirmation message with option to undo. // This confirmation and undo option will only appear if the UndoAction is // present and the client can handle this capability. optional UndoAction undo_action = 3; // Roundtripped server data on a per-action level. - optional components.feed.core.proto.wire.ActionPayload payload = 4; + optional feedwire1.ActionPayload payload = 4; } // Data needed by the client to handle the not interested action. @@ -136,9 +143,9 @@ message NotInterestedInData { // present and the client can handle this capability. optional UndoAction undo_action = 1; // The data needed by Stream to preform the dismiss. - repeated components.feed.core.proto.wire.DataOperation data_operations = 2; + repeated feedwire1.DataOperation data_operations = 2; // Roundtripped server data on a per-action level. - optional components.feed.core.proto.wire.ActionPayload payload = 3; + optional feedwire1.ActionPayload payload = 3; enum RecordedInterestType { UNKNOWN_INTEREST_TYPE = 0; TOPIC = 1; diff --git a/chromium/components/feed/core/proto/ui/piet/accessibility.proto b/chromium/components/feed/core/proto/ui/piet/accessibility.proto index b92fb90073a..b73760bc401 100644 --- a/chromium/components/feed/core/proto/ui/piet/accessibility.proto +++ b/chromium/components/feed/core/proto/ui/piet/accessibility.proto @@ -43,6 +43,19 @@ message Accessibility { // ID coming from a template. ParameterizedTextBindingRef accessibility_id_binding = 5; } + + oneof context_data { + // A string that may be spoken by the system that describes the result of an + // action. For example, "Opens the article." This provides additional + // context over the description. + + // NOTE: Only supported by iOS and maps to accessibilityHint on an + // accessible element. + ParameterizedText context = 6; + + // In case this is coming from a template. + ParameterizedTextBindingRef context_binding = 7; + } } // Semantic roles played by a UI element related to accessibility. diff --git a/chromium/components/feed/core/proto/ui/piet/errors.proto b/chromium/components/feed/core/proto/ui/piet/errors.proto index e5734305482..54e2818e7ba 100644 --- a/chromium/components/feed/core/proto/ui/piet/errors.proto +++ b/chromium/components/feed/core/proto/ui/piet/errors.proto @@ -122,6 +122,17 @@ enum ErrorCode { // Fields start at ID 1. // --------------------------------------------------------------------------- + // When the client tries to reference a SharedState that is not found, the + // Frame cannot be rendered, and likely no frames can be rendered. + // This error code must be reported by the Piet host app, as the Piet + // implementation assumes that all shared states have been provided already. + ERR_MISSING_SHARED_STATE = 12 /* [ + // Something is seriously wrong if a SharedState is missing + (server_error) = FATAL, + // Clients cannot render any Frame when SharedState is missing (no Template) + (client_error) = FATAL + ] */; + // When a Template cannot be located, it only affects Frames that reference // it, so we can proceed to render other unaffected Frames, making this an // ERROR, not FATAL. @@ -214,6 +225,14 @@ enum ErrorCode { (client_error) = ERROR ] */; + // Bindings are not supported in Frames or within Bound Elements. + ERR_UNSUPPORTED_CONTEXT_FOR_BINDING = 11 /* [ + // Bindings will not work in this context + (server_error) = ERROR, + // Clients might support bindings in this context. + (client_error) = WARNING + ] */; + // --------------------------------------------------------------------------- // Missing required proto fields, or invalid values. // Fields start at ID 101. diff --git a/chromium/components/feed/core/proto/ui/stream/stream_structure.proto b/chromium/components/feed/core/proto/ui/stream/stream_structure.proto index b529dbe99cb..f7d4389071d 100644 --- a/chromium/components/feed/core/proto/ui/stream/stream_structure.proto +++ b/chromium/components/feed/core/proto/ui/stream/stream_structure.proto @@ -19,9 +19,7 @@ option cc_enable_arenas = true; // Top level feature which shows a stream of cards. Provides any UI information // which may be needed in order to render the stream of cards. message Stream { - extend components.feed.core.proto.wire.Feature { - optional Stream stream_extension = 185431437; - } + extend feedwire1.Feature { optional Stream stream_extension = 185431437; } // Empty for now as don't support any custom information. } @@ -29,9 +27,7 @@ message Stream { // Feature which represents a cluster in a Stream. May have a Card or Content // as children. message Cluster { - extend components.feed.core.proto.wire.Feature { - optional Cluster cluster_extension = 190812910; - } + extend feedwire1.Feature { optional Cluster cluster_extension = 190812910; } // Empty for now as we don't support any custom information. } @@ -39,9 +35,7 @@ message Cluster { // Experimental feature which represents a carousel in a Stream. May have a list // of Cards or Content as children. message Carousel { - extend components.feed.core.proto.wire.Feature { - optional Carousel carousel_extension = 244251946; - } + extend feedwire1.Feature { optional Carousel carousel_extension = 244251946; } // Please use CL numbers you own for extension numbers. extensions 10000 to max; @@ -50,9 +44,7 @@ message Carousel { // Feature which represents a full card in a Stream. Allows metadata to be sent // to describe how to render the card. message Card { - extend components.feed.core.proto.wire.Feature { - optional Card card_extension = 185431438; - } + extend feedwire1.Feature { optional Card card_extension = 185431438; } // Please use CL numbers you own for extension numbers. extensions 10000 to max; @@ -98,9 +90,7 @@ message OfflineMetadata { // inside or outside a card. Actual data on what to display will be sent on an // extension. message Content { - extend components.feed.core.proto.wire.Feature { - optional Content content_extension = 185431439; - } + extend feedwire1.Feature { optional Content content_extension = 185431439; } enum Type { UNKNOWN_CONTENT = 0; @@ -122,7 +112,7 @@ message PietContent { // Content Ids of Piet Shared States which should be provided to Piet in order // to show its content. - repeated components.feed.core.proto.wire.ContentId piet_shared_states = 1; + repeated feedwire1.ContentId piet_shared_states = 1; // The Piet frame to render. optional components.feed.core.proto.ui.piet.Frame frame = 2; diff --git a/chromium/components/feed/core/proto/v2/store.proto b/chromium/components/feed/core/proto/v2/store.proto new file mode 100644 index 00000000000..99df5c61363 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/store.proto @@ -0,0 +1,173 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto3"; + +package feedstore; + +import "components/feed/core/proto/v2/wire/content_id.proto"; +import "components/feed/core/proto/v2/wire/feed_action.proto"; + +option optimize_for = LITE_RUNTIME; + +// Actual data stored by the client. +// This data is sourced from the wire protocol, which is converted upon receipt. +// This would replace both Journal and Content stores. +// +// This is the 'value' in the key/value store. +// Keys are defined as: +// 'S/<stream-id>' -> stream_data +// 'T/<stream-id>/<sequence-number>' -> stream_structures +// 'c/<content-id>' -> content +// 'a/<id>' -> action +// 's/<content-id>' -> shared_state +// 'N' -> next_stream_state +message Record { + oneof data { + StreamData stream_data = 1; + StreamStructureSet stream_structures = 2; + Content content = 3; + StoredAction local_action = 4; + StreamSharedState shared_state = 5; + // The result of a background refresh, to be processed later. + StreamAndContentState next_stream_state = 6; + } +} + +// Data about the Feed stream. There is at most one instance of StreamData. +message StreamData { + // Root ContentId, as provided by the server. + feedwire.ContentId content_id = 1; + // Token used to request a 'more' request to the server. + bytes next_page_token = 2; + // Token used to read or write to the same storage. + bytes consistency_token = 3; + // The unix timestamp in milliseconds that the most recent content was added. + int64 last_added_time_millis = 4; + // Next sequential ID to be used for StoredAction.id. + int32 next_action_id = 5; + // The content ID of the shared state for this stream. + feedwire.ContentId shared_state_id = 6; +} + +// A set of StreamStructures that should be applied to a stream. +message StreamStructureSet { + string stream_id = 1; + int32 sequence_number = 2; + repeated StreamStructure structures = 3; +} + +// This is the structure of the stream. It is defined through the parent/child +// relationship and an operation. This message will be journaled. Reading +// the journal from start to end will fully define the structure of the stream. +message StreamStructure { + // The defined set of DataOperations + // These operations align with the Operation enum defined in + // data_operation.proto. + enum Operation { + UNKNOWN = 0; + // Clear all the content in the session, creating a new session + CLEAR_ALL = 1; + // Append if not present or update + UPDATE_OR_APPEND = 2; + // Remove the node from its parent + REMOVE = 3; + } + Operation operation = 1; + // The ContentId of the content. + feedwire.ContentId content_id = 2; + // The parent ContentId, or unset if this is the root. + feedwire.ContentId parent_id = 3; + + // Type of node as denoted by the server. This type has no meaning for the + // client. + enum Type { + // Default type for operations that don't affect the stream (e.g. operations + // on shared states). + UNKNOWN_TYPE = 0; + // The root of the stream. + STREAM = 1; + // An internal tree node, which may have children. + CARD = 2; + // A leaf node, which provides content. + CONTENT = 3; + // An internal tree node, which may have children. + CLUSTER = 4; + } + Type type = 4; + // Set iff type=CONTENT + ContentInfo content_info = 5; +} + +message DataOperation { + StreamStructure structure = 1; + // Provided if structure adds content. + Content content = 2; +} + +message RepresentationData { + // URI (usually a URL) of what the content links to + string uri = 1; + // Unix timestamp (seconds) when content was published + int64 published_time_seconds = 2; +} + +message ContentInfo { + // Score given by server. + float score = 1; + // Unix timestamp (seconds) that content was received by Chrome. + int64 availability_time_seconds = 2; + RepresentationData representation_data = 3; + OfflineMetadata offline_metadata = 4; +} + +message OfflineMetadata { + // Title of the content. + string title = 1; + + // Url for image for the content. + string image_url = 2; + + // Publisher of the content. + string publisher = 3; + + // Url for the favicon for the content. + string favicon_url = 4; + + // Short string from the content, typically the start of an article. + string snippet = 5; +} + +message Content { + feedwire.ContentId content_id = 1; + // Opaque content. The UI layer knows how to parse and render this as a slice. + bytes frame = 2; +} + +// This represents a shared state item. +message StreamSharedState { + feedwire.ContentId content_id = 1; + // Opaque data required to render content. + bytes shared_state_data = 2; +} + +// A stored action awaiting upload. +message StoredAction { + // Unique ID for this stored action, provided by the client. + // This is a sequential number, so that the action with the lowest id value + // was recorded chronologically first. + int32 id = 1; + // How many times we have tried to upload the action. + int32 upload_attempt_count = 2; + // The action to upload. + feedwire.FeedAction action = 3; +} + +// The internal version of the server response. Includes feature tree and +// content. +message StreamAndContentState { + StreamData stream_data = 1; + repeated Content content = 2; + repeated StreamSharedState shared_state = 3; +} diff --git a/chromium/components/feed/core/proto/v2/ui.proto b/chromium/components/feed/core/proto/v2/ui.proto new file mode 100644 index 00000000000..c1258bc010b --- /dev/null +++ b/chromium/components/feed/core/proto/v2/ui.proto @@ -0,0 +1,110 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto3"; + +package feedui; + +option optimize_for = LITE_RUNTIME; + +option java_package = "org.chromium.components.feed.proto"; +option java_outer_classname = "FeedUiProto"; + +// This is a simplified and complete set of protos that define UI. +// It includes everything from search.now.ui needed in the UI, and excludes +// other data to reduce complexity. These proto messages should be constructible +// from the store protos. + +// A stream is a list of chunks in order. +// Each StreamUpdate contains the full list of chunks, +// but subsequent StreamUpdates after the first may refer to +// chunks previously received by chunk_id. +message StreamUpdate { + // Either a reference to an existing slice, or a new slice. + message SliceUpdate { + oneof update { + Slice slice = 1; + string slice_id = 2; + } + } + // One entry for each slice in the stream, in the order they should be + // presented. Existing slices not present in updated_slices should be dropped. + repeated SliceUpdate updated_slices = 1; + // Additional shared states to be used. Usually just one, and sent only on the + // first update. + repeated SharedState new_shared_states = 2; +} + +// A horizontal slice of UI to be presented in the vertical-scrolling feed. +message Slice { + oneof SliceData { + XSurfaceSlice xsurface_slice = 1; + ZeroStateSlice zero_state_slice = 3; + } + string slice_id = 2; +} + +// This slice is sent when no feed data can be loaded. +message ZeroStateSlice { + enum Type { + UNKNOWN = 0; + // A generic error that explains there are no cards available. + NO_CARDS_AVAILABLE = 1; + // An error indicating there were problems refreshing the feed. + CANT_REFRESH = 2; + }; + Type type = 1; +} + +message XSurfaceSlice { + bytes xsurface_frame = 1; +} + +// Wraps an XSurface shared state with a unique ID. +message SharedState { + string id = 1; + bytes xsurface_shared_state = 2; +} + +// An event on the UI. +message UiEvent { + enum Type { + UNKNOWN = 0; + CARD_TAPPED = 1; + CARD_VIEWED = 2; + CARD_SWIPED = 3; + MORE_BUTTON_VIEWED = 4; + MORE_BUTTON_CLICKED = 5; + SPINNER_STARTED = 6; + SPINNER_FINISHED = 7; + SPINNER_DESTROYED_WITHOUT_COMPLETING = 8; + PIET_FRAME_RENDERING_EVENT = 9; + SCROLL_STREAM = 10; + } + enum SpinnerType { + UNKNOWN_SPINNER_TYPE = 0; + // Spinner shown on initial load of the Feed. + INITIAL_LOAD = 1; + // Spinner shown when Feed is refreshed. + ZERO_STATE_REFRESH = 2; + // Spinner shown when more button is clicked. + MORE_BUTTON = 3; + // Spinner shown when a synthetic token is consumed. + SYNTHETIC_TOKEN = 4; + // Spinner shown when a spinner is shown for loading the infinite feed. + INFINITE_FEED = 5; + } + // For CARD_* type events. + string chunk_id = 1; + // For MORE_BUTTON_* type events. + int32 more_button_position = 2; + // For SPINNER_* type events. + SpinnerType spinner_type = 3; + // For SPINNER_FINISHED and SPINNER_DESTROYED_WITHOUT_COMPLETING. + int32 spinner_time_shown = 4; + // For PIET_FRAME_RENDERING_EVENT. + repeated int32 piet_error_codes = 5; + // For SCROLL_STREAM. + int32 scroll_distance = 6; +} diff --git a/chromium/components/feed/core/proto/v2/wire/action_payload.proto b/chromium/components/feed/core/proto/v2/wire/action_payload.proto new file mode 100644 index 00000000000..cc072bfb427 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/action_payload.proto @@ -0,0 +1,21 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// The data needed by the server to handle action recording. This information is +// opaque to the client and will be the information that is round-tripped so the +// server can properly handle the action. For the Not Interested In action, this +// data will contain the ids needed to record that the user is not interested +// in that particular topic or source. +// NOTE: it is important to keep this to a bare minimum amount of data. +message ActionPayload { + // Reserved fields for renderable unit extensions + // Please use CL numbers you own for extension numbers. + extensions 257605906; // ActionPayloadData +} diff --git a/chromium/components/feed/core/proto/v2/wire/action_request.proto b/chromium/components/feed/core/proto/v2/wire/action_request.proto new file mode 100644 index 00000000000..784a50dea03 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/action_request.proto @@ -0,0 +1,25 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/feed_action_request.proto"; + +// Top level request message. +message ActionRequest { + // Supported versions of request messages are specified as extensions to + // the top-level request message. An enum is used to denote the expected + // extensions for this request. + enum RequestVersion { + UNKNOWN_ACTION_REQUEST_VERSION = 0; + FEED_UPLOAD_ACTION = 1; + } + optional RequestVersion request_version = 1; + + optional FeedActionRequest feed_action_request = 1000; +} diff --git a/chromium/components/feed/core/proto/v2/wire/action_response.proto b/chromium/components/feed/core/proto/v2/wire/action_response.proto new file mode 100644 index 00000000000..213bf3a7473 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/action_response.proto @@ -0,0 +1,23 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Top level response message. +message ActionResponse { + // Supported versions of response messages are specified as extensions to + // the top-level response message. An enum is used to denote the expected + // extensions for this response. + enum ResponseVersion { + UNKNOWN_ACTION_RESPONSE_VERSION = 0; + FEED_UPLOAD_ACTION_RESPONSE = 1; + } + optional ResponseVersion response_version = 1; + + extensions 1000; // FeedActionResponse +} diff --git a/chromium/components/feed/core/proto/v2/wire/capability.proto b/chromium/components/feed/core/proto/v2/wire/capability.proto new file mode 100644 index 00000000000..73b95309fda --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/capability.proto @@ -0,0 +1,39 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Feature capability of either the client or the server. +// Next ID: 20 +enum Capability { + UNKNOWN_CAPABILITY = 0; + // The client is capable of a basic UI. + BASE_UI = 1; + + INFINITE_FEED = 5; + // Enable Dismiss command + DISMISS_COMMAND = 9; + // Enable Undo in Dismiss + UNDO_FOR_DISMISS_COMMAND = 10; + REDACTED_11 = 11; + // The client is only considered capable of supporting a minimal heirloomed + // feed. + HEIRLOOMED_FEED = 13; + // The client is capable of supporting sports features. + SPORTS_FEATURE = 14; + // The client is capable of supporting ads content. + PAID_CONTENT = 15; + // Enable open video command. + OPEN_VIDEO_COMMAND = 16; + REDACTED_17 = 17; + // Enable inline video autoplay. + INLINE_VIDEO_AUTOPLAY = 18; + // Enable the card menu. + CARD_MENU = 19; + reserved 2 to 4, 6 to 8, 12; +} diff --git a/chromium/components/feed/core/proto/v2/wire/client_info.proto b/chromium/components/feed/core/proto/v2/wire/client_info.proto new file mode 100644 index 00000000000..a4e7753cd34 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/client_info.proto @@ -0,0 +1,61 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/display_info.proto"; +import "components/feed/core/proto/v2/wire/version.proto"; + +// Information about the client performing the request similar to a user-agent +// string in HTTP. +// Next ID: 10. +message ClientInfo { + enum PlatformType { + UNKNOWN_PLATFORM = 0; + ANDROID_ID = 1; // ANDROID collides with a C++ preprocessor macro. + IOS = 2; + } + + enum AppType { CHROME = 3; } + + // The type of OS that the client is running. + optional PlatformType platform_type = 1; + + // The version of the OS that the client is running. + optional Version platform_version = 2; + + // The type of client app. + optional AppType app_type = 3; + + // The version of the client app. + optional Version app_version = 4; + + // A string identifying the language and region preferences of the client. + // Follows the BCP 47 format such as "en-US" or "fr-CA" + optional string locale = 5; + + // The information about the screen of the client. This is repeated because + // there are some devices that might have multiple display screens. + // (Ex fold-able phones) + repeated DisplayInfo display_info = 6; + + // Identifier of the user's device. For Android devices, contains a hash of + // the gaia email and android_id, which uniquely identifies the device for + // the user. Currently set by Android clients version 4.1 and later. + optional string client_instance_id = 7; + + // An Android device level identifier used for advertising, required for + // conversion tracking, see more at: + // https://support.google.com/googleplay/android-developer/answer/6048248 + optional string advertising_id = 8; + + // Two-letter country code as detected by the device. On Android devices, + // this comes from GServices check-in which uses the SIM card MCC (mobile + // country code), with fallback to IP geo lookup. + optional string device_country = 9; +} diff --git a/chromium/components/feed/core/proto/v2/wire/consistency_token.proto b/chromium/components/feed/core/proto/v2/wire/consistency_token.proto new file mode 100644 index 00000000000..48a6b2028bd --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/consistency_token.proto @@ -0,0 +1,15 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// A consistency token. +message ConsistencyToken { + // Indicates the min version of storage to read from. + optional bytes token = 1; +} diff --git a/chromium/components/feed/core/proto/v2/wire/content_id.proto b/chromium/components/feed/core/proto/v2/wire/content_id.proto new file mode 100644 index 00000000000..b098b3092f1 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/content_id.proto @@ -0,0 +1,49 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// An identifier for a piece of content served by Now or delivered to the Now +// client(s). +// See [INTERNAL LINK] for the design of this feature. +// ContentId comprises a unique key for all content. The client will never have +// more than one piece of content with the same ContentID. +message ContentId { + optional string content_domain = 1; + + // The type of content this represents. Generally, this is somewhat redundant, + // as this ContentId proto will be embedded within a particular parent proto + // that implies its type. It is repeated here for the purpose of making + // ContentId fully self-contained, able to completely specify a piece of + // content's ID without additional context. + // Since Type is one of the components of content's uniqueness, it is safe + // and reasonable for two related pieces of content with different types + // (e.g. a card and its attached notification) to share the same id and + // content_domain, and to differ only in their type. However, Type is **not** + // included when determing if two ContentIds are equivalent. + enum Type { + // Undefined type - DO NOT USE + TYPE_UNDEFINED = 0; + CARD = 1; + CLUSTER = 3; + // A feature, which is the indivisible unit of Feed content. + FEATURE = 4; + // A ContentId used only for identifying nodes in a tree structure. + TREE_STRUCTURE = 7; + // A ContentId for a collection. + COLLECTION = 8; + // A ContentId for a token, e.g. a NextPage token. + TOKEN = 9; + + reserved 2; + } + // The type of content this represents + optional Type type = 2; + + optional fixed64 id = 3; +} diff --git a/chromium/components/feed/core/proto/v2/wire/data_operation.proto b/chromium/components/feed/core/proto/v2/wire/data_operation.proto new file mode 100644 index 00000000000..0651eb21797 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/data_operation.proto @@ -0,0 +1,55 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/feature.proto"; +import "components/feed/core/proto/v2/wire/in_place_update_handle.proto"; +import "components/feed/core/proto/v2/wire/payload_metadata.proto"; +import "components/feed/core/proto/v2/wire/render_data.proto"; +import "components/feed/core/proto/v2/wire/templates.proto"; +import "components/feed/core/proto/v2/wire/token.proto"; + +// An extensible operation to change the state of data on the client. +message DataOperation { + // Next tag: 8 + + enum Operation { + UNKNOWN_OPERATION = 0; + // Remove all stored content of all types + CLEAR_ALL = 1; + // Update content if it exists, else append to the bottom of the feed + UPDATE_OR_APPEND = 2; + // Remove the item from the stream + REMOVE = 3; + } + + // The operation to perform on the data. + optional Operation operation = 1; + + // Data common to all payload types. + optional PayloadMetadata metadata = 2; + + // The actual data being supplied. + oneof payload { + // A stream UI level feature such as a cluster or card. + Feature feature = 3; + + // A token, capable of making a next page request. + Token next_page_token = 5; + + // Information to help render the content in the response. + RenderData render_data = 6; + + // A handle for updating one or more pieces of content in place. + InPlaceUpdateHandle in_place_update_handle = 8; + + // A collection of templates. + Templates templates = 4 [deprecated = true]; + } +} diff --git a/chromium/components/feed/core/proto/v2/wire/display_info.proto b/chromium/components/feed/core/proto/v2/wire/display_info.proto new file mode 100644 index 00000000000..f12bc749e8a --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/display_info.proto @@ -0,0 +1,24 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// The information about the client's screen. +// Next id: 4 +message DisplayInfo { + // Density of the screen in physical pixels per density independent pixel + // (DIP); see: + // http://developer.android.com/reference/android/util/DisplayMetrics.html#density + optional float screen_density = 1; + + // The width of the screen in pixels. + optional uint32 screen_width_in_pixels = 2; + + // The height of the screen in pixels. + optional uint32 screen_height_in_pixels = 3; +} diff --git a/chromium/components/feed/core/proto/v2/wire/duration.proto b/chromium/components/feed/core/proto/v2/wire/duration.proto new file mode 100644 index 00000000000..5f261ae32bc --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/duration.proto @@ -0,0 +1,15 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto3"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Copy of third_party/protobuf/src/google/protobuf/duration.proto. +message Duration { + int64 seconds = 1; + int32 nanos = 2; +} diff --git a/chromium/components/feed/core/proto/v2/wire/expiration_info.proto b/chromium/components/feed/core/proto/v2/wire/expiration_info.proto new file mode 100644 index 00000000000..326ceb86093 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/expiration_info.proto @@ -0,0 +1,26 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/duration.proto"; + +/* Information about whether and when a feature should expire and be removed + * from Discover. */ +message ExpirationInfo { + // Whether the feature can expire. + optional bool should_expire = 1; + + // Indicates how long after this response was received the client should wait + // before expiring (and hiding) this content. This expiration time is a best + // effort, and should not be done while the content is visible on screen. + // There are no penalties with showing the content after the expiry, though + // some uses of this API (ads in particular) do have SLA's about how often + // items can be shown after expiration. + optional feedwire.Duration expiration_duration = 2; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feature.proto b/chromium/components/feed/core/proto/v2/wire/feature.proto new file mode 100644 index 00000000000..775961cec7f --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feature.proto @@ -0,0 +1,47 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/content_id.proto"; +import "components/feed/core/proto/v2/wire/expiration_info.proto"; +import "components/feed/core/proto/v2/wire/stream_structure.proto"; + +// Features define both the structure and content found in the Stream. +message Feature { + // The ContentId identifying the parent feature for this feature. + optional ContentId parent_id = 1; + + // Enum denoting which extension containing the renderable data is associated + // with this Feature. + enum RenderableUnit { + UNKNOWN_RENDERABLE_UNIT = 0; + STREAM = 1; + CONTENT = 3; + CLUSTER = 4; + REDACTED_10 = 10; + REDACTED_11 = 11; + reserved 2, 5, 6, 7, 8, 9; + } + optional RenderableUnit renderable_unit = 2; + + // Indicates whether this feature should expire, and additional metadata + // necessariy to handle expiration. Note that clients may not support + // expiration of every type of feature. + optional ExpirationInfo expiration_info = 3; + + optional Stream stream_extension = 185431437; + optional Cluster cluster_extension = 190812910; + extensions 185431438; // Card + optional Content content_extension = 185431439; + extensions 194964015; // Token + extensions 286406442; // REDACTED + extensions 286406443; // REDACTED + + reserved 246375740, 274598443, 274598444, 277068786; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_action.proto b/chromium/components/feed/core/proto/v2/wire/feed_action.proto new file mode 100644 index 00000000000..62ae63efbc4 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_action.proto @@ -0,0 +1,38 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/action_payload.proto"; +import "components/feed/core/proto/v2/wire/content_id.proto"; + +message FeedAction { + // The Id for the content that this action was triggered on. + optional ContentId content_id = 1; + // Additional logging data that is on a per-action level + optional ActionPayload action_payload = 2; + // Client-generated data that pertains to the action. + optional ClientData client_data = 3; + // Next Id: 7 + + // The data the client provides to the server. + message ClientData { + // When the action was recorded on the client + optional int64 timestamp_seconds = 1; + + // A monotonically-increasing sequence number that increments per + // user + device. Used in experiments to measure action loss between client + // and server. + optional int64 sequence_number = 2; + + // The duration for the action in milliseconds. In case of view actions this + // is the duration for which the content is considered "viewed". + optional int64 duration_ms = 3; + } + reserved 4, 5, 6; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_action_request.proto b/chromium/components/feed/core/proto/v2/wire/feed_action_request.proto new file mode 100644 index 00000000000..f57f328fc0b --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_action_request.proto @@ -0,0 +1,20 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/consistency_token.proto"; +import "components/feed/core/proto/v2/wire/feed_action.proto"; + +// Request to upload new actions to the Actions Endpoint +message FeedActionRequest { + // Data related to recordable actions performed on the client. + repeated FeedAction feed_action = 1; + // Token used to write to the same storage. + optional ConsistencyToken consistency_token = 2; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_action_response.proto b/chromium/components/feed/core/proto/v2/wire/feed_action_response.proto new file mode 100644 index 00000000000..0cb782bd79a --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_action_response.proto @@ -0,0 +1,18 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/consistency_token.proto"; + +// A feed action response returns when an action has been successfully uploaded +// to the server. +message FeedActionResponse { + // Token used to read or write to the same storage. + optional ConsistencyToken consistency_token = 1; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_id.proto b/chromium/components/feed/core/proto/v2/wire/feed_id.proto new file mode 100644 index 00000000000..25afaa860b4 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_id.proto @@ -0,0 +1,20 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// This proto is used to uniquely identify a Feed of cards. +// The main use case is for the paginated feed, storing multiple +// feeds on the server, and needing to identify them individually. +// It is an empty extension holder because the client should not ever know +// or care what's in the FeedId, and so we can change the definition +// of what a FeedId for the server without worrying about users possibly +// looking into the implementation details. +message FeedId { + extensions 1 to 6; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_query.proto b/chromium/components/feed/core/proto/v2/wire/feed_query.proto new file mode 100644 index 00000000000..e69e919fae6 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_query.proto @@ -0,0 +1,52 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/token.proto"; + +message FeedQuery { + enum RequestReason { + // Bucket for any not listed. Should not be used (prefer adding a new + // request reason) + UNKNOWN_REQUEST_REASON = 0; + + // App is manually triggering a request, outside of scheduling a request. + // Should be used rarely. + MANUAL_REFRESH = 1; + + // Host wants a request to refresh content. + SCHEDULED_REFRESH = 2; + + // Host wants a request to load more content. + NEXT_PAGE_SCROLL = 3; + + REDACTED_4 = 4; + + // Host wants to update content in place. + IN_PLACE_UPDATE = 5; + } + + // The reason the query is being initiated. + optional RequestReason reason = 1; + + // A collection of Token messages, wrapped in a message so it can be used in a + // oneof. + message Tokens { repeated Token tokens = 1; } + + oneof token { + // The token for requesting the next page of Feed content, to be used with + // reason = NEXT_PAGE_SCROLL. + Token next_page_token = 3; + // Tokens from InPlaceUpdateHandle for content to update in place, if + // reason = IN_PLACE_UPDATE. + Tokens in_place_update_tokens = 5; + } + + reserved 2; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_request.proto b/chromium/components/feed/core/proto/v2/wire/feed_request.proto new file mode 100644 index 00000000000..b4ee427b6dc --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_request.proto @@ -0,0 +1,37 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/capability.proto"; +import "components/feed/core/proto/v2/wire/client_info.proto"; +import "components/feed/core/proto/v2/wire/consistency_token.proto"; +import "components/feed/core/proto/v2/wire/feed_id.proto"; +import "components/feed/core/proto/v2/wire/feed_query.proto"; + +// Request to fetch new data for the feed +message FeedRequest { + // Information about the client making the request. + optional ClientInfo client_info = 1; + + // Query parameters to fetch feed data. + optional FeedQuery feed_query = 2; + + // The list of client supported capabilities. + repeated Capability client_capability = 4; + + // Token used to read from/write to the same storage. + optional ConsistencyToken consistency_token = 5; + + // Created on the server and used by the client to identify the feed when + // clients will store multiple infinite feeds. + // See [INTERNAL LINK] + repeated FeedId feed_ids_to_preserve = 12; + + reserved 3, 13; +} diff --git a/chromium/components/feed/core/proto/v2/wire/feed_response.proto b/chromium/components/feed/core/proto/v2/wire/feed_response.proto new file mode 100644 index 00000000000..c408d59b387 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/feed_response.proto @@ -0,0 +1,48 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/capability.proto"; +import "components/feed/core/proto/v2/wire/feed_action_response.proto"; +import "components/feed/core/proto/v2/wire/data_operation.proto"; +import "components/feed/core/proto/v2/wire/feed_id.proto"; +import "components/feed/core/proto/v2/wire/response_status_code.proto"; + +// A feed response is a series of directives to manipulate backend storage, +// similar to database commands. Individual data operations contain all the +// necessary information to manipulate the client state. +message FeedResponse { + optional FeedActionResponse feed_response = 1000; + + // DataOperations are applied on the client in order in which they are + // received. + repeated DataOperation data_operation = 1; + // Metadata for the entire FeedResponse. + optional FeedResponseMetadata feed_response_metadata = 2; + + // The list of server-response supported capabilities. + repeated Capability server_capabilities = 3; + + // The status code for this response. + optional ResponseStatusCode response_status_code = 4; +} + +// Data which is relevant for the whole response made by the server. +message FeedResponseMetadata { + // Time at which the server fulfilled this response. This is needed as client + // cannot be the source of truth. + optional int64 response_time_ms = 1; + + // Created on the server and used by the client to identify the feed when + // clients will store multiple infinite feeds. + // See [INTERNAL LINK] + optional FeedId feed_id = 3; + + reserved 2; +} diff --git a/chromium/components/feed/core/proto/v2/wire/in_place_update_handle.proto b/chromium/components/feed/core/proto/v2/wire/in_place_update_handle.proto new file mode 100644 index 00000000000..382dcad03d3 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/in_place_update_handle.proto @@ -0,0 +1,26 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/duration.proto"; +import "components/feed/core/proto/v2/wire/token.proto"; + +// InPlaceUpdateHandle is a handle for the client to send to the server in order +// to update content in-place. +message InPlaceUpdateHandle { + // Indicates how long after this response ws received the client should wait + // before sending the token back to the server. It is not an error to send the + // token earlier, but in that case the server may just replace the handle with + // and an updated use_after and the same token. + optional feedwire.Duration use_after = 1; + + // Opaque token to use in a request for the server to send updated versions of + // its associated content. + optional Token token = 2; +} diff --git a/chromium/components/feed/core/proto/v2/wire/next_page_token.proto b/chromium/components/feed/core/proto/v2/wire/next_page_token.proto new file mode 100644 index 00000000000..37d532db4ef --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/next_page_token.proto @@ -0,0 +1,13 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +message NextPageToken { + optional bytes next_page_token = 1; +} diff --git a/chromium/components/feed/core/proto/v2/wire/payload_metadata.proto b/chromium/components/feed/core/proto/v2/wire/payload_metadata.proto new file mode 100644 index 00000000000..28a7695af40 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/payload_metadata.proto @@ -0,0 +1,19 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/content_id.proto"; + +// Metadata common to all payloads in a DataOperation. +message PayloadMetadata { + // The unique identifier of the payload. + optional ContentId content_id = 1; + + reserved 2, 3, 4, 5; +} diff --git a/chromium/components/feed/core/proto/v2/wire/render_data.proto b/chromium/components/feed/core/proto/v2/wire/render_data.proto new file mode 100644 index 00000000000..67bf2029043 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/render_data.proto @@ -0,0 +1,23 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Contains data to use during client-side rendering of the response, like +// templates and themes. +message RenderData { + // Enum denoting which extension contains the render data. + enum RenderDataType { + UNKNOWN_RENDER_DATA_TYPE = 0; + XSURFACE = 1; + } + optional RenderDataType render_data_type = 1; + + message XSurfaceContainer { optional bytes render_data = 1; } + optional XSurfaceContainer xsurface_container = 1000; +} diff --git a/chromium/components/feed/core/proto/v2/wire/request.proto b/chromium/components/feed/core/proto/v2/wire/request.proto new file mode 100644 index 00000000000..7add283ffd5 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/request.proto @@ -0,0 +1,25 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/feed_request.proto"; + +// Top level request message. +message Request { + // Supported versions of request messages are specified as extensions to + // the top-level request message. An enum is used to denote the expected + // extensions for this request. + enum RequestVersion { + UNKNOWN_REQUEST_VERSION = 0; + FEED_QUERY = 1; + } + optional RequestVersion request_version = 1; + + optional FeedRequest feed_request = 1000; +} diff --git a/chromium/components/feed/core/proto/v2/wire/response.proto b/chromium/components/feed/core/proto/v2/wire/response.proto new file mode 100644 index 00000000000..3b87e7e2085 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/response.proto @@ -0,0 +1,25 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/feed_response.proto"; + +// Top level response message. +message Response { + // Supported versions of response messages are specified as extensions to + // the top-level response message. An enum is used to denote the expected + // extensions for this response. + enum ResponseVersion { + UNKNOWN_RESPONSE_VERSION = 0; + FEED_RESPONSE = 1; + } + optional ResponseVersion response_version = 1; + + optional FeedResponse feed_response = 1000; +} diff --git a/chromium/components/feed/core/proto/v2/wire/response_status_code.proto b/chromium/components/feed/core/proto/v2/wire/response_status_code.proto new file mode 100644 index 00000000000..ec45adb7ff3 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/response_status_code.proto @@ -0,0 +1,23 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Status for the Feed response. +enum ResponseStatusCode { + UNKNOWN_STATUS_CODE = 0; + + // Eligible for feed and no errors encountered. + STATUS_OK = 1; + + // Ineligible for Feed. + STATUS_INELIGIBLE_FOR_FEED = 2; + + // No cards. + STATUS_NO_CONTENT_RETURNED = 3; +} diff --git a/chromium/components/feed/core/proto/v2/wire/stream_structure.proto b/chromium/components/feed/core/proto/v2/wire/stream_structure.proto new file mode 100644 index 00000000000..256aa6c89af --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/stream_structure.proto @@ -0,0 +1,43 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Top level feature which shows a stream of cards. Provides any UI information +// which may be needed in order to render the stream of cards. +message Stream { +} + +// Feature which represents a cluster in a Stream. May have a Card or Content +// as children. +// TODO Determine if Clusters can be removed. +message Cluster { + // Empty for now as we don't support any custom information. +} + +// Feature which is able to show actual content in a stream. This could be +// inside or outside a card. Actual data on what to display will be sent on an +// extension. +message Content { + enum Type { + UNKNOWN_CONTENT = 0; + XSURFACE = 1; + } + optional Type type = 1; + + optional bool is_ad = 3; + + optional XSurfaceContent xsurface_content = 1000; + + reserved 2; +} + +// Opaque data to for rendering a piece of content. +message XSurfaceContent { + optional bytes xsurface_output = 1; +} diff --git a/chromium/components/feed/core/proto/v2/wire/templates.proto b/chromium/components/feed/core/proto/v2/wire/templates.proto new file mode 100644 index 00000000000..ad8ec10b631 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/templates.proto @@ -0,0 +1,24 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Templates provide a way to separate formatting from content. +// Deprecated: use RenderData instead. +message Templates { + option deprecated = true; + + // Enum denoting which extension contains template data. + enum TemplateType { + UNKNOWN_TEMPLATE_TYPE = 0; + XSURFACE = 1; + } + optional TemplateType template_type = 1; + + extensions 264680549; // XSurfaceTemplates +} diff --git a/chromium/components/feed/core/proto/v2/wire/token.proto b/chromium/components/feed/core/proto/v2/wire/token.proto new file mode 100644 index 00000000000..37e91678636 --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/token.proto @@ -0,0 +1,25 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +import "components/feed/core/proto/v2/wire/content_id.proto"; +import "components/feed/core/proto/v2/wire/next_page_token.proto"; + +// A structure containing client-opaque data relating to a request. +message Token { + // The ContentId identifying the parent for this feature. Needed for tokens + // used in a data operation. + optional ContentId parent_id = 2; + + extensions 1001; // REDACTED + optional NextPageToken next_page_token = 1002; + extensions 1003; // InPlaceUpdateToken + + reserved 1, 194964015; +} diff --git a/chromium/components/feed/core/proto/v2/wire/version.proto b/chromium/components/feed/core/proto/v2/wire/version.proto new file mode 100644 index 00000000000..6ead96a0f5c --- /dev/null +++ b/chromium/components/feed/core/proto/v2/wire/version.proto @@ -0,0 +1,44 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +package feedwire; + +option optimize_for = LITE_RUNTIME; + +// Specification of an application or OS version. +// A version string typically looks like: 'major.minor.build.revision' +message Version { + optional int32 major = 1; + optional int32 minor = 2; + optional int32 build = 3; + optional int32 revision = 4; + + // The CPU architecture that the native libraries support + enum Architecture { + UNKNOWN_ARCHITECTURE = 0; + ARM = 1; + ARM64 = 2; + MIPS = 3; + MIPS64 = 4; + X86 = 5; + X86_64 = 6; + } + optional Architecture architecture = 5; + + // The release stage of the build + enum BuildType { + UNKNOWN_BUILD_TYPE = 0; + DEV = 1; + ALPHA = 2; + BETA = 3; + RELEASE = 4; + } + optional BuildType build_type = 6; + + // Specific to Android OS versions. Specifies the API version that the OS + // supports. + optional int32 api_version = 7; +} diff --git a/chromium/components/feed/core/proto/wire/action_payload.proto b/chromium/components/feed/core/proto/wire/action_payload.proto index e7923e73590..96c49df59ae 100644 --- a/chromium/components/feed/core/proto/wire/action_payload.proto +++ b/chromium/components/feed/core/proto/wire/action_payload.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/action_payload_for_test.proto b/chromium/components/feed/core/proto/wire/action_payload_for_test.proto index b382bc1c02d..0eef9277501 100644 --- a/chromium/components/feed/core/proto/wire/action_payload_for_test.proto +++ b/chromium/components/feed/core/proto/wire/action_payload_for_test.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; @@ -19,7 +19,7 @@ message ActionPayloadForTest { // The mid that represents the topic of the story on the card optional string id = 1; - extend components.feed.core.proto.wire.ActionPayload { + extend feedwire1.ActionPayload { optional ActionPayloadForTest action_payload_for_test_extension = 2; } } diff --git a/chromium/components/feed/core/proto/wire/action_request.proto b/chromium/components/feed/core/proto/wire/action_request.proto index 5e47d7ead47..78b000bb1f1 100644 --- a/chromium/components/feed/core/proto/wire/action_request.proto +++ b/chromium/components/feed/core/proto/wire/action_request.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/action_type.proto b/chromium/components/feed/core/proto/wire/action_type.proto index efc1ddf85df..f0e8fd69e7a 100644 --- a/chromium/components/feed/core/proto/wire/action_type.proto +++ b/chromium/components/feed/core/proto/wire/action_type.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/capability.proto b/chromium/components/feed/core/proto/wire/capability.proto index 512206469af..6bfb7860787 100644 --- a/chromium/components/feed/core/proto/wire/capability.proto +++ b/chromium/components/feed/core/proto/wire/capability.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; @@ -12,7 +12,7 @@ option java_package = "org.chromium.components.feed.core.proto.wire"; option java_outer_classname = "CapabilityProto"; // Feature capability of either the client or the server. -// Next ID: 12. +// Next ID: 14. enum Capability { UNKNOWN_CAPABILITY = 0; BASE_UI = 1; @@ -24,7 +24,8 @@ enum Capability { ARTICLE_SNIPPETS = 8; CAROUSELS = 9; ELEMENTS = 10; - CONTENT_ID_UNIFICATION = 11; + SEND_FEEDBACK = 12; + CLICK_ACTION = 13; - reserved 3; + reserved 3, 11; } diff --git a/chromium/components/feed/core/proto/wire/client_info.proto b/chromium/components/feed/core/proto/wire/client_info.proto index 4138682452e..c395cca3df9 100644 --- a/chromium/components/feed/core/proto/wire/client_info.proto +++ b/chromium/components/feed/core/proto/wire/client_info.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/consistency_token.proto b/chromium/components/feed/core/proto/wire/consistency_token.proto index f172b364bb2..4b7b209d942 100644 --- a/chromium/components/feed/core/proto/wire/consistency_token.proto +++ b/chromium/components/feed/core/proto/wire/consistency_token.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/content_id.proto b/chromium/components/feed/core/proto/wire/content_id.proto index f35d86518c7..51ba3334a71 100644 --- a/chromium/components/feed/core/proto/wire/content_id.proto +++ b/chromium/components/feed/core/proto/wire/content_id.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/data_operation.proto b/chromium/components/feed/core/proto/wire/data_operation.proto index bc7f178b8a4..a917619b2c0 100644 --- a/chromium/components/feed/core/proto/wire/data_operation.proto +++ b/chromium/components/feed/core/proto/wire/data_operation.proto @@ -8,7 +8,7 @@ import "components/feed/core/proto/wire/feature.proto"; import "components/feed/core/proto/wire/payload_metadata.proto"; import "components/feed/core/proto/ui/piet/piet.proto"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/display_info.proto b/chromium/components/feed/core/proto/wire/display_info.proto index ba2bd6b3434..7d0a2dbc262 100644 --- a/chromium/components/feed/core/proto/wire/display_info.proto +++ b/chromium/components/feed/core/proto/wire/display_info.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feature.proto b/chromium/components/feed/core/proto/wire/feature.proto index 9f0c4c01739..24741223b8e 100644 --- a/chromium/components/feed/core/proto/wire/feature.proto +++ b/chromium/components/feed/core/proto/wire/feature.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_action.proto b/chromium/components/feed/core/proto/wire/feed_action.proto index 8b04bc8ab45..dce68844591 100644 --- a/chromium/components/feed/core/proto/wire/feed_action.proto +++ b/chromium/components/feed/core/proto/wire/feed_action.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_action_query_data.proto b/chromium/components/feed/core/proto/wire/feed_action_query_data.proto index ce228c1168f..46cd59cf823 100644 --- a/chromium/components/feed/core/proto/wire/feed_action_query_data.proto +++ b/chromium/components/feed/core/proto/wire/feed_action_query_data.proto @@ -7,7 +7,7 @@ syntax = "proto2"; import "components/feed/core/proto/wire/action_type.proto"; import "components/feed/core/proto/wire/semantic_properties.proto"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_action_request.proto b/chromium/components/feed/core/proto/wire/feed_action_request.proto index 72fda0e2f41..a82e7ceb962 100644 --- a/chromium/components/feed/core/proto/wire/feed_action_request.proto +++ b/chromium/components/feed/core/proto/wire/feed_action_request.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_action_response.proto b/chromium/components/feed/core/proto/wire/feed_action_response.proto index 5ea0e391738..4e201632451 100644 --- a/chromium/components/feed/core/proto/wire/feed_action_response.proto +++ b/chromium/components/feed/core/proto/wire/feed_action_response.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_query.proto b/chromium/components/feed/core/proto/wire/feed_query.proto index 76a94142ec7..2620819c328 100644 --- a/chromium/components/feed/core/proto/wire/feed_query.proto +++ b/chromium/components/feed/core/proto/wire/feed_query.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_request.proto b/chromium/components/feed/core/proto/wire/feed_request.proto index 162b640c604..b67a006d69f 100644 --- a/chromium/components/feed/core/proto/wire/feed_request.proto +++ b/chromium/components/feed/core/proto/wire/feed_request.proto @@ -11,7 +11,7 @@ import "components/feed/core/proto/wire/feed_action_query_data.proto"; import "components/feed/core/proto/wire/feed_query.proto"; import "components/feed/core/proto/wire/request.proto"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/feed_response.proto b/chromium/components/feed/core/proto/wire/feed_response.proto index 74578002de4..ec101483f8b 100644 --- a/chromium/components/feed/core/proto/wire/feed_response.proto +++ b/chromium/components/feed/core/proto/wire/feed_response.proto @@ -8,7 +8,7 @@ import "components/feed/core/proto/wire/capability.proto"; import "components/feed/core/proto/wire/data_operation.proto"; import "components/feed/core/proto/wire/response.proto"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/mockserver/mock_server.proto b/chromium/components/feed/core/proto/wire/mockserver/mock_server.proto index 67a2c39f240..7fc41e4f923 100644 --- a/chromium/components/feed/core/proto/wire/mockserver/mock_server.proto +++ b/chromium/components/feed/core/proto/wire/mockserver/mock_server.proto @@ -14,7 +14,7 @@ option java_outer_classname = "MockServerProto"; message MockServer { // The initial response - optional components.feed.core.proto.wire.Response initial_response = 1; + optional feedwire1.Response initial_response = 1; // conditional responses represent responses for paged content repeated ConditionalResponse conditional_responses = 2; @@ -25,7 +25,7 @@ message MockServer { /** This represents a response providing updates to the stream. */ message MockUpdate { // The response with the push update - optional components.feed.core.proto.wire.Response response = 1; + optional feedwire1.Response response = 1; // The amount of time to wait, in milliseconds, before the push is triggered. // This is relative to the time the GCL file is loaded. @@ -38,5 +38,5 @@ message ConditionalResponse { optional bytes continuation_token = 1; // The response to use - optional components.feed.core.proto.wire.Response response = 2; + optional feedwire1.Response response = 2; } diff --git a/chromium/components/feed/core/proto/wire/payload_metadata.proto b/chromium/components/feed/core/proto/wire/payload_metadata.proto index e40e0c4c252..d695087a05e 100644 --- a/chromium/components/feed/core/proto/wire/payload_metadata.proto +++ b/chromium/components/feed/core/proto/wire/payload_metadata.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/piet_shared_state_item.proto b/chromium/components/feed/core/proto/wire/piet_shared_state_item.proto index c57a47cfcff..99f4d420998 100644 --- a/chromium/components/feed/core/proto/wire/piet_shared_state_item.proto +++ b/chromium/components/feed/core/proto/wire/piet_shared_state_item.proto @@ -7,7 +7,7 @@ syntax = "proto2"; import "components/feed/core/proto/ui/piet/piet.proto"; import "components/feed/core/proto/wire/content_id.proto"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/request.proto b/chromium/components/feed/core/proto/wire/request.proto index d26e8089eba..a02372025b1 100644 --- a/chromium/components/feed/core/proto/wire/request.proto +++ b/chromium/components/feed/core/proto/wire/request.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/response.proto b/chromium/components/feed/core/proto/wire/response.proto index 81e4f2658bf..11692309e9e 100644 --- a/chromium/components/feed/core/proto/wire/response.proto +++ b/chromium/components/feed/core/proto/wire/response.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/semantic_properties.proto b/chromium/components/feed/core/proto/wire/semantic_properties.proto index c6327b8c777..d478b7744a5 100644 --- a/chromium/components/feed/core/proto/wire/semantic_properties.proto +++ b/chromium/components/feed/core/proto/wire/semantic_properties.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/proto/wire/token.proto b/chromium/components/feed/core/proto/wire/token.proto index 8cdb8770e8a..e14edf2f643 100644 --- a/chromium/components/feed/core/proto/wire/token.proto +++ b/chromium/components/feed/core/proto/wire/token.proto @@ -6,7 +6,7 @@ syntax = "proto2"; import "components/feed/core/proto/wire/feature.proto"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; @@ -15,9 +15,7 @@ option java_outer_classname = "TokenProto"; // A continuation token (paging token). message Token { - extend components.feed.core.proto.wire.Feature { - optional Token token_extension = 194964015; - } + extend feedwire1.Feature { optional Token token_extension = 194964015; } // Indicates the last position of the current content for a parent. A request // can be made using the next_page_token to get additional features which will diff --git a/chromium/components/feed/core/proto/wire/version.proto b/chromium/components/feed/core/proto/wire/version.proto index 164753b7fe8..ba9ef172eef 100644 --- a/chromium/components/feed/core/proto/wire/version.proto +++ b/chromium/components/feed/core/proto/wire/version.proto @@ -4,7 +4,7 @@ syntax = "proto2"; -package components.feed.core.proto.wire; +package feedwire1; option optimize_for = LITE_RUNTIME; diff --git a/chromium/components/feed/core/shared_prefs/BUILD.gn b/chromium/components/feed/core/shared_prefs/BUILD.gn new file mode 100644 index 00000000000..9b0b637fc77 --- /dev/null +++ b/chromium/components/feed/core/shared_prefs/BUILD.gn @@ -0,0 +1,12 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +static_library("feed_shared_prefs") { + sources = [ + "pref_names.cc", + "pref_names.h", + ] + + deps = [ "//components/prefs" ] +} diff --git a/chromium/components/feed/core/shared_prefs/pref_names.cc b/chromium/components/feed/core/shared_prefs/pref_names.cc new file mode 100644 index 00000000000..da75d63d4c4 --- /dev/null +++ b/chromium/components/feed/core/shared_prefs/pref_names.cc @@ -0,0 +1,26 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_SHARED_PREFS_PREF_NAMES_CC_ +#define COMPONENTS_FEED_CORE_SHARED_PREFS_PREF_NAMES_CC_ + +#include "components/feed/core/shared_prefs/pref_names.h" + +#include "components/prefs/pref_registry_simple.h" + +namespace feed { +namespace prefs { + +const char kEnableSnippets[] = "ntp_snippets.enable"; +const char kArticlesListVisible[] = "ntp_snippets.list_visible"; + +void RegisterFeedSharedProfilePrefs(PrefRegistrySimple* registry) { + registry->RegisterBooleanPref(kEnableSnippets, true); + registry->RegisterBooleanPref(kArticlesListVisible, true); +} + +} // namespace prefs +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_SHARED_PREFS_PREF_NAMES_CC_ diff --git a/chromium/components/feed/core/shared_prefs/pref_names.h b/chromium/components/feed/core/shared_prefs/pref_names.h new file mode 100644 index 00000000000..27fa5e3f65b --- /dev/null +++ b/chromium/components/feed/core/shared_prefs/pref_names.h @@ -0,0 +1,27 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_SHARED_PREFS_PREF_NAMES_H_ +#define COMPONENTS_FEED_CORE_SHARED_PREFS_PREF_NAMES_H_ + +class PrefRegistrySimple; + +// These prefs are shared by Feed and Zine (ntp_snippets). + +namespace feed { +namespace prefs { + +// If set to false, remote suggestions are completely disabled. This is set by +// an enterprise policy. +extern const char kEnableSnippets[]; + +// Whether the list of NTP snippets is visible in UI. This is set to false when +// the user toggles the list off. +extern const char kArticlesListVisible[]; + +void RegisterFeedSharedProfilePrefs(PrefRegistrySimple* registry); +} // namespace prefs +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_SHARED_PREFS_PREF_NAMES_H_ diff --git a/chromium/components/feed/core/v2/BUILD.gn b/chromium/components/feed/core/v2/BUILD.gn new file mode 100644 index 00000000000..99d8d69a7c1 --- /dev/null +++ b/chromium/components/feed/core/v2/BUILD.gn @@ -0,0 +1,117 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//testing/test.gni") + +if (is_android) { + import("//build/config/android/rules.gni") +} + +source_set("feed_core_v2") { + sources = [ + "enums.cc", + "enums.h", + "feed_network.cc", + "feed_network.h", + "feed_network_impl.cc", + "feed_network_impl.h", + "feed_store.cc", + "feed_store.h", + "feed_stream.cc", + "feed_stream.h", + "prefs.cc", + "prefs.h", + "proto_util.cc", + "proto_util.h", + "public/feed_service.cc", + "public/feed_service.h", + "public/feed_stream_api.h", + "refresh_task_scheduler.h", + "request_throttler.cc", + "request_throttler.h", + "scheduling.cc", + "scheduling.h", + "stream_event_metrics.cc", + "stream_event_metrics.h", + "stream_model.cc", + "stream_model.h", + "stream_model/ephemeral_change.cc", + "stream_model/ephemeral_change.h", + "stream_model/feature_tree.cc", + "stream_model/feature_tree.h", + "stream_model_update_request.cc", + "stream_model_update_request.h", + "tasks/load_stream_from_store_task.cc", + "tasks/load_stream_from_store_task.h", + "tasks/load_stream_task.cc", + "tasks/load_stream_task.h", + "tasks/wait_for_store_initialize_task.cc", + "tasks/wait_for_store_initialize_task.h", + ] + deps = [ + "//components/feed/core:feed_core", + "//components/feed/core/common:feed_core_common", + "//components/offline_pages/task:task", + "//components/prefs", + "//components/signin/public/identity_manager", + "//components/variations", + "//components/variations/net", + "//components/web_resource:web_resource", + "//net", + "//services/network/public/cpp", + "//services/network/public/mojom", + "//third_party/zlib/google:compression_utils", + ] + + public_deps = [ + "//base", + "//components/feed/core/common:feed_core_common", + "//components/feed/core/proto:proto_v2", + ] +} + +source_set("core_unit_tests") { + testonly = true + sources = [ + "feed_network_impl_unittest.cc", + "feed_store_unittest.cc", + "feed_stream_unittest.cc", + "request_throttler_unittest.cc", + "stream_model_unittest.cc", + "stream_model_update_request_unittest.cc", + "test/callback_receiver.h", + "test/callback_receiver.h", + "test/proto_printer.cc", + "test/proto_printer.h", + "test/stream_builder.cc", + "test/stream_builder.h", + ] + + deps = [ + ":feed_core_v2", + ":unit_tests_bundle_data", + "//base", + "//base/test:test_support", + "//components/feed/core:feed_core", + "//components/feed/core/common:feed_core_common", + "//components/leveldb_proto:test_support", + "//components/prefs:test_support", + "//components/signin/public/identity_manager", + "//components/signin/public/identity_manager:test_support", + "//net:test_support", + "//services/network:test_support", + "//services/network/public/cpp", + "//services/network/public/mojom", + "//testing/gtest", + "//third_party/zlib/google:compression_utils", + ] +} + +bundle_data("unit_tests_bundle_data") { + visibility = [ ":core_unit_tests" ] + testonly = true + sources = [ "//components/test/data/feed/response.binarypb" ] + outputs = [ "{{bundle_resources_dir}}/" + + "{{source_root_relative_dir}}/{{source_file_part}}" ] +} diff --git a/chromium/components/feed/core/v2/README.md b/chromium/components/feed/core/v2/README.md new file mode 100644 index 00000000000..267bc51312b --- /dev/null +++ b/chromium/components/feed/core/v2/README.md @@ -0,0 +1 @@ +The next iteration of the feed component, in development. diff --git a/chromium/components/feed/core/v2/enums.cc b/chromium/components/feed/core/v2/enums.cc new file mode 100644 index 00000000000..ce8c0ce37ec --- /dev/null +++ b/chromium/components/feed/core/v2/enums.cc @@ -0,0 +1,52 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/enums.h" + +#include <ostream> + +namespace feed { + +// Included for debug builds only for reduced binary size. + +std::ostream& operator<<(std::ostream& out, LoadStreamStatus value) { +#ifndef NDEBUG + switch (value) { + case LoadStreamStatus::kNoStatus: + return out << "kNoStatus"; + case LoadStreamStatus::kLoadedFromStore: + return out << "kLoadedFromStore"; + case LoadStreamStatus::kLoadedFromNetwork: + return out << "kLoadedFromNetwork"; + case LoadStreamStatus::kFailedWithStoreError: + return out << "kFailedWithStoreError"; + case LoadStreamStatus::kNoStreamDataInStore: + return out << "kNoStreamDataInStore"; + case LoadStreamStatus::kModelAlreadyLoaded: + return out << "kModelAlreadyLoaded"; + case LoadStreamStatus::kNoResponseBody: + return out << "kNoResponseBody"; + case LoadStreamStatus::kProtoTranslationFailed: + return out << "kProtoTranslationFailed"; + case LoadStreamStatus::kDataInStoreIsStale: + return out << "kDataInStoreIsStale"; + case LoadStreamStatus::kDataInStoreIsStaleTimestampInFuture: + return out << "kDataInStoreIsStaleTimestampInFuture"; + case LoadStreamStatus::kCannotLoadFromNetworkSupressedForHistoryDelete: + return out << "kCannotLoadFromNetworkSupressedForHistoryDelete"; + case LoadStreamStatus::kCannotLoadFromNetworkOffline: + return out << "kCannotLoadFromNetworkOffline"; + case LoadStreamStatus::kCannotLoadFromNetworkThrottled: + return out << "kCannotLoadFromNetworkThrottled"; + case LoadStreamStatus::kLoadNotAllowedEulaNotAccepted: + return out << "kLoadNotAllowedEulaNotAccepted"; + case LoadStreamStatus::kLoadNotAllowedArticlesListHidden: + return out << "kLoadNotAllowedArticlesListHidden"; + } +#else + return out << (static_cast<int>(value)); +#endif // ifndef NDEBUG +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/enums.h b/chromium/components/feed/core/v2/enums.h new file mode 100644 index 00000000000..dc6d84e2e96 --- /dev/null +++ b/chromium/components/feed/core/v2/enums.h @@ -0,0 +1,45 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_ENUMS_H_ +#define COMPONENTS_FEED_CORE_V2_ENUMS_H_ + +#include <iosfwd> + +#include "components/feed/core/common/enums.h" + +namespace feed { + +enum NetworkRequestType : int { + kFeedQuery = 0, + kUploadActions = 1, +}; + +enum class LoadStreamStatus { + // Loading was not attempted. + kNoStatus = 0, + kLoadedFromStore = 1, + kLoadedFromNetwork = 2, + kFailedWithStoreError = 3, + kNoStreamDataInStore = 4, + kModelAlreadyLoaded = 5, + kNoResponseBody = 6, + // TODO(harringtond): Let's add more specific proto translation errors. + kProtoTranslationFailed = 7, + kDataInStoreIsStale = 8, + // The timestamp for stored data is in the future, so we're treating stored + // data as it it is stale. + kDataInStoreIsStaleTimestampInFuture = 9, + kCannotLoadFromNetworkSupressedForHistoryDelete = 10, + kCannotLoadFromNetworkOffline = 11, + kCannotLoadFromNetworkThrottled = 12, + kLoadNotAllowedEulaNotAccepted = 13, + kLoadNotAllowedArticlesListHidden = 14, +}; + +std::ostream& operator<<(std::ostream& out, LoadStreamStatus value); + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_ENUMS_H_ diff --git a/chromium/components/feed/core/v2/feed_network.cc b/chromium/components/feed/core/v2/feed_network.cc new file mode 100644 index 00000000000..5bca7bf3bba --- /dev/null +++ b/chromium/components/feed/core/v2/feed_network.cc @@ -0,0 +1,30 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_network.h" + +#include "components/feed/core/proto/v2/wire/action_request.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_response.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" + +namespace feed { + +FeedNetwork::QueryRequestResult::QueryRequestResult() = default; +FeedNetwork::QueryRequestResult::~QueryRequestResult() = default; +FeedNetwork::QueryRequestResult::QueryRequestResult(QueryRequestResult&&) = + default; +FeedNetwork::QueryRequestResult& FeedNetwork::QueryRequestResult::operator=( + QueryRequestResult&&) = default; + +FeedNetwork::ActionRequestResult::ActionRequestResult() = default; +FeedNetwork::ActionRequestResult::~ActionRequestResult() = default; +FeedNetwork::ActionRequestResult::ActionRequestResult(ActionRequestResult&&) = + default; +FeedNetwork::ActionRequestResult& FeedNetwork::ActionRequestResult::operator=( + ActionRequestResult&&) = default; + +FeedNetwork::~FeedNetwork() = default; + +} // namespace feed diff --git a/chromium/components/feed/core/v2/feed_network.h b/chromium/components/feed/core/v2/feed_network.h new file mode 100644 index 00000000000..8a25a238117 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_network.h @@ -0,0 +1,69 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_FEED_NETWORK_H_ +#define COMPONENTS_FEED_CORE_V2_FEED_NETWORK_H_ + +#include <memory> +#include "base/callback.h" + +namespace feedwire { +class ActionRequest; +class FeedActionResponse; +class Request; +class Response; +} // namespace feedwire + +namespace feed { + +class FeedNetwork { + public: + // Result of SendQueryRequest. + struct QueryRequestResult { + QueryRequestResult(); + ~QueryRequestResult(); + QueryRequestResult(QueryRequestResult&&); + QueryRequestResult& operator=(QueryRequestResult&&); + // HTTP status code if one was received, 0 otherwise. + int32_t status_code = 0; + // Response body if one was received. + std::unique_ptr<feedwire::Response> response_body; + }; + + // Result of SendActionRequest. + struct ActionRequestResult { + ActionRequestResult(); + ~ActionRequestResult(); + ActionRequestResult(ActionRequestResult&&); + ActionRequestResult& operator=(ActionRequestResult&&); + // HTTP status code if one was received, 0 otherwise. + int32_t status_code = 0; + // Response body if one was received. + std::unique_ptr<feedwire::FeedActionResponse> response_body; + }; + + virtual ~FeedNetwork(); + + // Send a feedwire::Request, and receive the response in |callback|. + // |callback| will be called unless the request is canceled with + // |CancelRequests()|. + virtual void SendQueryRequest( + const feedwire::Request& request, + base::OnceCallback<void(QueryRequestResult)> callback) = 0; + + // Send a feedwire::ActionRequest, and receive the response in |callback|. + // |callback| will be called unless the request is canceled with + // |CancelRequests()|. + virtual void SendActionRequest( + const feedwire::ActionRequest& request, + base::OnceCallback<void(ActionRequestResult)> callback) = 0; + + // Cancels all pending requests immediately. This could be used, for example, + // if there are pending requests for a user who just signed out. + virtual void CancelRequests() = 0; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_FEED_NETWORK_H_ diff --git a/chromium/components/feed/core/v2/feed_network_impl.cc b/chromium/components/feed/core/v2/feed_network_impl.cc new file mode 100644 index 00000000000..5231a84277f --- /dev/null +++ b/chromium/components/feed/core/v2/feed_network_impl.cc @@ -0,0 +1,458 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_network_impl.h" +#include "base/bind.h" +#include "base/containers/flat_set.h" +#include "base/metrics/histogram_functions.h" +#include "base/metrics/histogram_macros.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/proto/v2/wire/action_request.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_response.pb.h" +#include "components/feed/core/proto/v2/wire/feed_query.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/prefs/pref_service.h" +#include "components/signin/public/identity_manager/access_token_info.h" +#include "components/signin/public/identity_manager/identity_manager.h" +#include "components/signin/public/identity_manager/primary_account_access_token_fetcher.h" +#include "components/signin/public/identity_manager/scope_set.h" +#include "components/variations/net/variations_http_headers.h" +#include "net/base/load_flags.h" +#include "net/base/url_util.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_status_code.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "services/network/public/cpp/resource_request.h" +#include "services/network/public/cpp/resource_request_body.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/network/public/cpp/simple_url_loader.h" +#include "third_party/zlib/google/compression_utils.h" + +namespace feed { +namespace { +constexpr char kAuthenticationScope[] = + "https://www.googleapis.com/auth/googlenow"; +constexpr char kApplicationOctetStream[] = "application/octet-stream"; +constexpr base::TimeDelta kNetworkTimeout = base::TimeDelta::FromSeconds(30); + +constexpr char kFeedQueryUrl[] = + "https://www.google.com/httpservice/retry/InteractiveDiscoverAgaService/" + "FeedQuery"; +constexpr char kNextPageQueryUrl[] = + "https://www.google.com/httpservice/retry/InteractiveDiscoverAgaService/" + "NextPageQuery"; +constexpr char kBackgroundQueryUrl[] = + "https://www.google.com/httpservice/noretry/BackgroundDiscoverAgaService/" + "FeedQuery"; + +using RawResponse = FeedNetworkImpl::RawResponse; +} // namespace + +struct FeedNetworkImpl::RawResponse { + // A union of net::Error (if the request failed) and the http + // status code(if the request succeeded in reaching the server). + int32_t status_code; + // HTTP response body. + std::string response_bytes; +}; + +namespace { +template <typename RESULT> +void ParseAndForwardResponse(base::OnceCallback<void(RESULT)> result_callback, + RawResponse raw_response) { + RESULT result; + result.status_code = raw_response.status_code; + if (result.status_code == 200) { + auto response_message = std::make_unique<typename decltype( + result.response_body)::element_type>(); + if (response_message->ParseFromString(raw_response.response_bytes)) { + result.response_body = std::move(response_message); + } + } + std::move(result_callback).Run(std::move(result)); +} + +void AddMothershipPayloadQueryParams(bool is_post, + const std::string& payload, + const std::string& language_tag, + GURL* url) { + if (!is_post) + *url = net::AppendQueryParameter(*url, "reqpld", payload); + *url = net::AppendQueryParameter(*url, "fmt", "bin"); + if (!language_tag.empty()) + *url = net::AppendQueryParameter(*url, "hl", language_tag); +} + +} // namespace + +// Each NetworkFetch instance represents a single "logical" fetch that ends by +// calling the associated callback. Network fetches will actually attempt two +// fetches if there is a signed in user; the first to retrieve an access token, +// and the second to the specified url. +class FeedNetworkImpl::NetworkFetch { + public: + NetworkFetch(const GURL& url, + const std::string& request_type, + std::string request_body, + signin::IdentityManager* identity_manager, + network::SharedURLLoaderFactory* loader_factory, + const std::string& api_key, + const base::TickClock* tick_clock, + PrefService* pref_service) + : url_(url), + request_type_(request_type), + request_body_(std::move(request_body)), + identity_manager_(identity_manager), + loader_factory_(loader_factory), + api_key_(api_key), + tick_clock_(tick_clock), + entire_send_start_ticks_(tick_clock_->NowTicks()), + pref_service_(pref_service) { + // Apply the host override (from snippets-internals). + std::string host_override = + pref_service_->GetString(feed::prefs::kHostOverrideHost); + if (!host_override.empty()) { + GURL override_host_url(host_override); + if (override_host_url.is_valid()) { + GURL::Replacements replacements; + replacements.SetSchemeStr(override_host_url.scheme_piece()); + replacements.SetHostStr(override_host_url.host_piece()); + replacements.SetPortStr(override_host_url.port_piece()); + url_ = url_.ReplaceComponents(replacements); + host_overridden_ = true; + } + } + } + ~NetworkFetch() = default; + NetworkFetch(const NetworkFetch&) = delete; + NetworkFetch& operator=(const NetworkFetch&) = delete; + + void Start(base::OnceCallback<void(RawResponse)> done_callback) { + done_callback_ = std::move(done_callback); + + if (!identity_manager_->HasPrimaryAccount()) { + StartLoader(); + return; + } + + StartAccessTokenFetch(); + } + + private: + void StartAccessTokenFetch() { + signin::ScopeSet scopes{kAuthenticationScope}; + // It's safe to pass base::Unretained(this) since deleting the token fetcher + // will prevent the callback from being completed. + token_fetcher_ = std::make_unique<signin::PrimaryAccountAccessTokenFetcher>( + "feed", identity_manager_, scopes, + base::BindOnce(&NetworkFetch::AccessTokenFetchFinished, + base::Unretained(this), tick_clock_->NowTicks()), + signin::PrimaryAccountAccessTokenFetcher::Mode::kWaitUntilAvailable); + } + + void AccessTokenFetchFinished(base::TimeTicks token_start_ticks, + GoogleServiceAuthError error, + signin::AccessTokenInfo access_token_info) { + UMA_HISTOGRAM_ENUMERATION( + "ContentSuggestions.Feed.Network.TokenFetchStatus", error.state(), + GoogleServiceAuthError::NUM_STATES); + + base::TimeDelta token_duration = + tick_clock_->NowTicks() - token_start_ticks; + UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.TokenDuration", + token_duration); + + access_token_ = access_token_info.token; + StartLoader(); + } + + void StartLoader() { + loader_only_start_ticks_ = tick_clock_->NowTicks(); + simple_loader_ = MakeLoader(); + simple_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( + loader_factory_, base::BindOnce(&NetworkFetch::OnSimpleLoaderComplete, + base::Unretained(this))); + } + + std::unique_ptr<network::SimpleURLLoader> MakeLoader() { + // TODO(pnoland): Add data use measurement once it's supported for simple + // url loader. + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("interest_feedv2_send", R"( + semantics { + sender: "Feed Library" + description: "Chrome can show content suggestions (e.g. articles) " + "in the form of a feed. For signed-in users, these may be " + "personalized based on interest signals in the user's account." + trigger: "Triggered periodically in the background, or upon " + "explicit user request." + data: "The locale of the device and data describing the suggested " + "content that the user interacted with. For signed-in users " + "the request is authenticated. " + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: YES + cookies_store: "user" + setting: "This can be disabled from the New Tab Page by collapsing " + "the articles section." + chrome_policy { + NTPContentSuggestionsEnabled { + policy_options {mode: MANDATORY} + NTPContentSuggestionsEnabled: false + } + } + })"); + GURL url(url_); + if (access_token_.empty() && !api_key_.empty()) + url = net::AppendQueryParameter(url_, "key", api_key_); + + auto resource_request = std::make_unique<network::ResourceRequest>(); + resource_request->url = url; + + resource_request->load_flags = net::LOAD_BYPASS_CACHE; + resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit; + resource_request->method = request_type_; + + // Include credentials ONLY if the user has overridden the feed host through + // the internals page. This allows for some authentication workflows we need + // for testing. + if (host_overridden_) { + resource_request->credentials_mode = + network::mojom::CredentialsMode::kInclude; + resource_request->site_for_cookies = net::SiteForCookies::FromUrl(url); + } + + SetRequestHeaders(!request_body_.empty(), resource_request.get()); + + auto simple_loader = network::SimpleURLLoader::Create( + std::move(resource_request), traffic_annotation); + simple_loader->SetAllowHttpErrorResults(true); + simple_loader->SetTimeoutDuration(kNetworkTimeout); + PopulateRequestBody(simple_loader.get()); + return simple_loader; + } + + void SetRequestHeaders(bool has_request_body, + network::ResourceRequest* request) const { + if (has_request_body) { + request->headers.SetHeader(net::HttpRequestHeaders::kContentType, + kApplicationOctetStream); + request->headers.SetHeader("Content-Encoding", "gzip"); + } + + variations::SignedIn signed_in_status = variations::SignedIn::kNo; + if (!access_token_.empty()) { + request->headers.SetHeader(net::HttpRequestHeaders::kAuthorization, + "Bearer " + access_token_); + signed_in_status = variations::SignedIn::kYes; + } + + // Add X-Client-Data header with experiment IDs from field trials. + variations::AppendVariationsHeader(url_, variations::InIncognito::kNo, + signed_in_status, request); + } + + void PopulateRequestBody(network::SimpleURLLoader* loader) { + std::string compressed_request_body; + if (!request_body_.empty()) { + std::string uncompressed_request_body( + reinterpret_cast<const char*>(request_body_.data()), + request_body_.size()); + + compression::GzipCompress(uncompressed_request_body, + &compressed_request_body); + + loader->AttachStringForUpload(compressed_request_body, + kApplicationOctetStream); + } + + UMA_HISTOGRAM_COUNTS_1M( + "ContentSuggestions.Feed.Network.RequestSizeKB.Compressed", + static_cast<int>(compressed_request_body.size() / 1024)); + } + + void OnSimpleLoaderComplete(std::unique_ptr<std::string> response) { + int32_t status_code = simple_loader_->NetError(); + // If overriding the feed host, try to grab the Bless nonce. This is + // strictly informational, and only displayed in snippets-internals. + if (host_overridden_ && simple_loader_->ResponseInfo()) { + size_t iter = 0; + std::string value; + while (simple_loader_->ResponseInfo()->headers->EnumerateHeader( + &iter, "www-authenticate", &value)) { + size_t pos = value.find("nonce=\""); + if (pos != std::string::npos) { + std::string nonce = value.substr(pos + 7, 16); + if (nonce.size() == 16) { + pref_service_->SetString(feed::prefs::kHostOverrideBlessNonce, + nonce); + break; + } + } + } + } + + std::string response_body; + if (response) { + status_code = simple_loader_->ResponseInfo()->headers->response_code(); + response_body = std::move(*response); + + if (status_code == net::HTTP_UNAUTHORIZED) { + signin::ScopeSet scopes{kAuthenticationScope}; + CoreAccountId account_id = identity_manager_->GetPrimaryAccountId(); + if (!account_id.empty()) { + identity_manager_->RemoveAccessTokenFromCache(account_id, scopes, + access_token_); + } + } + } + + base::TimeDelta entire_send_duration = + tick_clock_->NowTicks() - entire_send_start_ticks_; + UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.Duration", + entire_send_duration); + + base::TimeDelta loader_only_duration = + tick_clock_->NowTicks() - loader_only_start_ticks_; + // This histogram purposefully matches name and bucket size used in + // RemoteSuggestionsFetcherImpl. + UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime", loader_only_duration); + + base::UmaHistogramSparse( + "ContentSuggestions.Feed.Network.RequestStatusCode", status_code); + + // The below is true even if there is a protocol error, so this will + // record response size as long as the request completed. + if (status_code >= 200) { + UMA_HISTOGRAM_COUNTS_1M("ContentSuggestions.Feed.Network.ResponseSizeKB", + static_cast<int>(response_body.size() / 1024)); + } + + std::move(done_callback_).Run({status_code, std::move(response_body)}); + } + + private: + GURL url_; + const std::string request_type_; + std::string access_token_; + const std::string request_body_; + signin::IdentityManager* const identity_manager_; + std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher> token_fetcher_; + std::unique_ptr<network::SimpleURLLoader> simple_loader_; + base::OnceCallback<void(RawResponse)> done_callback_; + network::SharedURLLoaderFactory* loader_factory_; + const std::string api_key_; + const base::TickClock* tick_clock_; + + // Set when the NetworkFetch is constructed, before token and article fetch. + const base::TimeTicks entire_send_start_ticks_; + + // Should be set right before the article fetch, and after the token fetch if + // there is one. + base::TimeTicks loader_only_start_ticks_; + PrefService* pref_service_; + bool host_overridden_ = false; +}; + +FeedNetworkImpl::FeedNetworkImpl( + Delegate* delegate, + signin::IdentityManager* identity_manager, + const std::string& api_key, + scoped_refptr<network::SharedURLLoaderFactory> loader_factory, + const base::TickClock* tick_clock, + PrefService* pref_service) + : delegate_(delegate), + identity_manager_(identity_manager), + api_key_(api_key), + loader_factory_(loader_factory), + tick_clock_(tick_clock), + pref_service_(pref_service) {} + +FeedNetworkImpl::~FeedNetworkImpl() = default; + +void FeedNetworkImpl::SendQueryRequest( + const feedwire::Request& request, + base::OnceCallback<void(QueryRequestResult)> callback) { + std::string binary_proto; + request.SerializeToString(&binary_proto); + + // TODO(harringtond): Decide how we want to override these URLs for testing. + // Should probably add a command-line flag. + GURL url; + switch (request.feed_request().feed_query().reason()) { + case feedwire::FeedQuery::SCHEDULED_REFRESH: + case feedwire::FeedQuery::IN_PLACE_UPDATE: + url = GURL(kBackgroundQueryUrl); + break; + case feedwire::FeedQuery::NEXT_PAGE_SCROLL: + url = GURL(kNextPageQueryUrl); + break; + case feedwire::FeedQuery::MANUAL_REFRESH: + url = GURL(kFeedQueryUrl); + break; + default: + std::move(callback).Run({}); + return; + } + + AddMothershipPayloadQueryParams(/*is_post=*/false, binary_proto, + delegate_->GetLanguageTag(), &url); + Send(url, "GET", /*request_body=*/std::string(), + base::BindOnce(&ParseAndForwardResponse<QueryRequestResult>, + std::move(callback))); +} + +void FeedNetworkImpl::SendActionRequest( + const feedwire::ActionRequest& request, + base::OnceCallback<void(ActionRequestResult)> callback) { + std::string binary_proto; + request.SerializeToString(&binary_proto); + + GURL url( + "https://www.google.com/httpservice/retry/ClankActionUploadService/" + "ClankActionUpload"); + AddMothershipPayloadQueryParams(/*is_post=*/true, /*payload=*/std::string(), + delegate_->GetLanguageTag(), &url); + + Send(url, "POST", std::move(binary_proto), + base::BindOnce(&ParseAndForwardResponse<ActionRequestResult>, + std::move(callback))); +} + +void FeedNetworkImpl::CancelRequests() { + pending_requests_.clear(); +} + +void FeedNetworkImpl::Send(const GURL& url, + const std::string& request_type, + std::string request_body, + base::OnceCallback<void(RawResponse)> callback) { + auto fetch = std::make_unique<NetworkFetch>( + url, request_type, std::move(request_body), identity_manager_, + loader_factory_.get(), api_key_, tick_clock_, pref_service_); + NetworkFetch* fetch_unowned = fetch.get(); + pending_requests_.emplace(std::move(fetch)); + + // It's safe to pass base::Unretained(this) since deleting the network fetch + // will prevent the callback from being completed. + fetch_unowned->Start(base::BindOnce(&FeedNetworkImpl::SendComplete, + base::Unretained(this), fetch_unowned, + std::move(callback))); +} + +void FeedNetworkImpl::SendComplete( + NetworkFetch* fetch, + base::OnceCallback<void(RawResponse)> callback, + RawResponse raw_response) { + DCHECK_EQ(1UL, pending_requests_.count(fetch)); + pending_requests_.erase(fetch); + + std::move(callback).Run(std::move(raw_response)); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/feed_network_impl.h b/chromium/components/feed/core/v2/feed_network_impl.h new file mode 100644 index 00000000000..b96f25d0b6c --- /dev/null +++ b/chromium/components/feed/core/v2/feed_network_impl.h @@ -0,0 +1,91 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_FEED_NETWORK_IMPL_H_ +#define COMPONENTS_FEED_CORE_V2_FEED_NETWORK_IMPL_H_ + +#include <string> +#include "base/callback.h" +#include "base/containers/flat_set.h" +#include "base/containers/unique_ptr_adapters.h" +#include "base/memory/scoped_refptr.h" +#include "components/feed/core/v2/feed_network.h" +#include "url/gurl.h" + +class PrefService; +namespace base { +class TickClock; +} +namespace signin { +class IdentityManager; +} +namespace network { +class SharedURLLoaderFactory; +} + +namespace feed { + +class FeedNetworkImpl : public FeedNetwork { + public: + class NetworkFetch; + struct RawResponse; + class Delegate { + public: + virtual ~Delegate() = default; + // Returns a string which represents the top locale and region of the + // device. + virtual std::string GetLanguageTag() = 0; + }; + + FeedNetworkImpl(Delegate* delegate, + signin::IdentityManager* identity_manager, + const std::string& api_key, + scoped_refptr<network::SharedURLLoaderFactory> loader_factory, + const base::TickClock* tick_clock, + PrefService* pref_service); + ~FeedNetworkImpl() override; + FeedNetworkImpl(const FeedNetworkImpl&) = delete; + FeedNetworkImpl& operator=(FeedNetworkImpl&) = delete; + + // FeedNetwork. + + void SendQueryRequest( + const feedwire::Request& request, + base::OnceCallback<void(QueryRequestResult)> callback) override; + + void SendActionRequest( + const feedwire::ActionRequest& request, + base::OnceCallback<void(ActionRequestResult)> callback) override; + + // Cancels all pending requests immediately. This could be used, for example, + // if there are pending requests for a user who just signed out. + void CancelRequests() override; + + private: + // Start a request to |url| of type |request_type| with body |request_body|. + // |callback| will be called when the response is received or if there is + // an error, including non-protocol errors. The contents of |request_body| + // will be gzipped. + void Send(const GURL& url, + const std::string& request_type, + std::string request_body, + base::OnceCallback<void(FeedNetworkImpl::RawResponse)> callback); + + void SendComplete(NetworkFetch* fetch, + base::OnceCallback<void(RawResponse)> callback, + RawResponse raw_response); + + Delegate* delegate_; + signin::IdentityManager* identity_manager_; + const std::string api_key_; + scoped_refptr<network::SharedURLLoaderFactory> loader_factory_; + const base::TickClock* tick_clock_; + PrefService* pref_service_; + base::flat_set<std::unique_ptr<NetworkFetch>, base::UniquePtrComparator> + pending_requests_; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_FEED_NETWORK_IMPL_H_ diff --git a/chromium/components/feed/core/v2/feed_network_impl_unittest.cc b/chromium/components/feed/core/v2/feed_network_impl_unittest.cc new file mode 100644 index 00000000000..7170cd5b8f1 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_network_impl_unittest.cc @@ -0,0 +1,413 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_network_impl.h" + +#include <memory> +#include <utility> + +#include "base/bind.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" +#include "base/test/bind_test_util.h" +#include "base/test/metrics/histogram_tester.h" +#include "base/test/simple_test_tick_clock.h" +#include "base/test/task_environment.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/proto/v2/wire/action_request.pb.h" +#include "components/feed/core/proto/v2/wire/feed_action_response.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/feed/core/v2/test/callback_receiver.h" +#include "components/prefs/testing_pref_service.h" +#include "components/signin/public/identity_manager/identity_test_environment.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_status_code.h" +#include "net/http/http_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" +#include "services/network/public/cpp/url_loader_completion_status.h" +#include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h" +#include "services/network/test/test_url_loader_factory.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/zlib/google/compression_utils.h" +#include "url/gurl.h" + +namespace feed { +namespace { + +using base::TimeDelta; +using testing::ElementsAre; +using ActionRequestResult = FeedNetwork::ActionRequestResult; +using QueryRequestResult = FeedNetwork::QueryRequestResult; + +feedwire::Request GetTestFeedRequest(feedwire::FeedQuery::RequestReason reason = + feedwire::FeedQuery::MANUAL_REFRESH) { + feedwire::Request request; + request.set_request_version(feedwire::Request::FEED_QUERY); + request.mutable_feed_request()->mutable_feed_query()->set_reason(reason); + return request; +} + +feedwire::Response GetTestFeedResponse() { + feedwire::Response response; + response.set_response_version(feedwire::Response::FEED_RESPONSE); + return response; +} + +feedwire::ActionRequest GetTestActionRequest() { + feedwire::ActionRequest request; + request.set_request_version(feedwire::ActionRequest::FEED_UPLOAD_ACTION); + return request; +} + +feedwire::FeedActionResponse GetTestActionResponse() { + feedwire::FeedActionResponse response; + response.mutable_consistency_token()->set_token("tok"); + return response; +} + +class TestDelegate : public FeedNetworkImpl::Delegate { + public: + std::string GetLanguageTag() override { return "en"; } +}; + +class FeedNetworkTest : public testing::Test { + public: + FeedNetworkTest() { + identity_test_env_.MakePrimaryAccountAvailable("example@gmail.com"); + identity_test_env_.SetAutomaticIssueOfAccessTokens(true); + } + FeedNetworkTest(FeedNetworkTest&) = delete; + FeedNetworkTest& operator=(const FeedNetworkTest&) = delete; + ~FeedNetworkTest() override = default; + + void SetUp() override { + feed::RegisterProfilePrefs(profile_prefs_.registry()); + + shared_url_loader_factory_ = + base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>( + &test_factory_); + feed_network_ = std::make_unique<FeedNetworkImpl>( + &delegate_, identity_test_env_.identity_manager(), "dummy_api_key", + shared_url_loader_factory_, task_environment_.GetMockTickClock(), + &profile_prefs_); + } + + FeedNetwork* feed_network() { return feed_network_.get(); } + + signin::IdentityTestEnvironment* identity_env() { + return &identity_test_env_; + } + + network::TestURLLoaderFactory* test_factory() { return &test_factory_; } + + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + + TestingPrefServiceSimple& profile_prefs() { return profile_prefs_; } + + void Respond(const GURL& url, + const std::string& response_string, + net::HttpStatusCode code = net::HTTP_OK, + network::URLLoaderCompletionStatus status = + network::URLLoaderCompletionStatus()) { + auto head = network::mojom::URLResponseHead::New(); + if (code >= 0) { + if (response_headers_) { + head->headers = response_headers_; + } else { + head->headers = base::MakeRefCounted<net::HttpResponseHeaders>( + "HTTP/1.1 " + base::NumberToString(code)); + } + status.decoded_body_length = response_string.length(); + } + + test_factory_.AddResponse(url, std::move(head), response_string, status); + } + + network::ResourceRequest RespondToQueryRequest( + const std::string& response_string, + net::HttpStatusCode code) { + task_environment_.RunUntilIdle(); + network::TestURLLoaderFactory::PendingRequest* pending_request = + test_factory()->GetPendingRequest(0); + CHECK(pending_request); + network::ResourceRequest resource_request = pending_request->request; + Respond(pending_request->request.url, response_string, code); + task_environment_.FastForwardUntilNoTasksRemain(); + return resource_request; + } + + network::ResourceRequest RespondToQueryRequest( + feedwire::Response response_message, + net::HttpStatusCode code) { + std::string binary_proto; + response_message.SerializeToString(&binary_proto); + return RespondToQueryRequest(binary_proto, code); + } + + network::ResourceRequest RespondToActionRequest( + feedwire::FeedActionResponse response_message, + net::HttpStatusCode code) { + std::string binary_proto; + response_message.SerializeToString(&binary_proto); + return RespondToQueryRequest(binary_proto, code); + } + + protected: + scoped_refptr<net::HttpResponseHeaders> response_headers_; + + private: + TestDelegate delegate_; + signin::IdentityTestEnvironment identity_test_env_; + std::unique_ptr<FeedNetwork> feed_network_; + network::TestURLLoaderFactory test_factory_; + scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory_; + base::SimpleTestTickClock test_tick_clock_; + TestingPrefServiceSimple profile_prefs_; +}; + +TEST_F(FeedNetworkTest, SendQueryRequestEmpty) { + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(feedwire::Request(), receiver.Bind()); + + ASSERT_TRUE(receiver.GetResult()); + const QueryRequestResult& result = *receiver.GetResult(); + EXPECT_EQ(0, result.status_code); + EXPECT_FALSE(result.response_body); +} + +TEST_F(FeedNetworkTest, SendQueryRequestSendsValidRequest) { + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + network::ResourceRequest resource_request = + RespondToQueryRequest("", net::HTTP_OK); + + EXPECT_EQ( + "https://www.google.com/httpservice/retry/InteractiveDiscoverAgaService/" + "FeedQuery?reqpld=%08%01%C2%3E%04%12%02%08%01&fmt=bin&hl=en", + resource_request.url); + EXPECT_EQ("GET", resource_request.method); + EXPECT_FALSE(resource_request.headers.HasHeader("content-encoding")); + std::string authorization; + EXPECT_TRUE( + resource_request.headers.GetHeader("Authorization", &authorization)); + EXPECT_EQ(authorization, "Bearer access_token"); +} + +TEST_F(FeedNetworkTest, SendQueryRequestInvalidResponse) { + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + RespondToQueryRequest("invalid", net::HTTP_OK); + + ASSERT_TRUE(receiver.GetResult()); + const QueryRequestResult& result = *receiver.GetResult(); + EXPECT_EQ(net::HTTP_OK, result.status_code); + EXPECT_FALSE(result.response_body); +} + +TEST_F(FeedNetworkTest, SendQueryRequestReceivesResponse) { + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_OK); + + ASSERT_TRUE(receiver.GetResult()); + const QueryRequestResult& result = *receiver.GetResult(); + EXPECT_EQ(net::HTTP_OK, result.status_code); + EXPECT_EQ(GetTestFeedResponse().response_version(), + result.response_body->response_version()); +} + +TEST_F(FeedNetworkTest, SendQueryRequestIgnoresBodyForNon200Response) { + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_FORBIDDEN); + + ASSERT_TRUE(receiver.GetResult()); + const QueryRequestResult& result = *receiver.GetResult(); + EXPECT_EQ(net::HTTP_FORBIDDEN, result.status_code); + EXPECT_FALSE(result.response_body); +} + +TEST_F(FeedNetworkTest, CancelRequest) { + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + feed_network()->CancelRequests(); + task_environment_.FastForwardUntilNoTasksRemain(); + + EXPECT_FALSE(receiver.GetResult()); +} + +TEST_F(FeedNetworkTest, RequestTimeout) { + base::HistogramTester histogram_tester; + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + task_environment_.FastForwardBy(TimeDelta::FromSeconds(30)); + + ASSERT_TRUE(receiver.GetResult()); + const QueryRequestResult& result = *receiver.GetResult(); + EXPECT_EQ(net::ERR_TIMED_OUT, result.status_code); + histogram_tester.ExpectTimeBucketCount( + "ContentSuggestions.Feed.Network.Duration", TimeDelta::FromSeconds(30), + 1); +} + +TEST_F(FeedNetworkTest, ParallelRequests) { + CallbackReceiver<QueryRequestResult> receiver1, receiver2; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver1.Bind()); + // Make another request with a different URL so Respond() won't affect both + // requests. + feed_network()->SendQueryRequest( + GetTestFeedRequest(feedwire::FeedQuery::NEXT_PAGE_SCROLL), + receiver2.Bind()); + + // Respond to both requests, avoiding FastForwardUntilNoTasksRemain until + // a response is added for both requests. + ASSERT_EQ(2, test_factory()->NumPending()); + for (int i = 0; i < 2; ++i) { + network::TestURLLoaderFactory::PendingRequest* pending_request = + test_factory()->GetPendingRequest(0); + ASSERT_TRUE(pending_request) << "for request #" << i; + std::string binary_proto; + GetTestFeedResponse().SerializeToString(&binary_proto); + Respond(pending_request->request.url, binary_proto, net::HTTP_OK); + } + task_environment_.FastForwardUntilNoTasksRemain(); + + EXPECT_TRUE(receiver1.GetResult()); + EXPECT_TRUE(receiver2.GetResult()); +} + +TEST_F(FeedNetworkTest, ShouldReportRequestStatusCode) { + CallbackReceiver<QueryRequestResult> receiver; + base::HistogramTester histogram_tester; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_FORBIDDEN); + EXPECT_THAT( + histogram_tester.GetAllSamples( + "ContentSuggestions.Feed.Network.RequestStatusCode"), + ElementsAre(base::Bucket(/*min=*/net::HTTP_FORBIDDEN, /*count=*/1))); +} + +TEST_F(FeedNetworkTest, ShouldIncludeAPIKeyForAuthError) { + identity_env()->SetAutomaticIssueOfAccessTokens(false); + CallbackReceiver<QueryRequestResult> receiver; + base::HistogramTester histogram_tester; + + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + identity_env()->WaitForAccessTokenRequestIfNecessaryAndRespondWithError( + GoogleServiceAuthError( + GoogleServiceAuthError::State::INVALID_GAIA_CREDENTIALS)); + + network::ResourceRequest resource_request = + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_OK); + + EXPECT_THAT(resource_request.url.spec(), + testing::HasSubstr("key=dummy_api_key")); + + EXPECT_THAT( + histogram_tester.GetAllSamples( + "ContentSuggestions.Feed.Network.TokenFetchStatus"), + testing::ElementsAre(base::Bucket( + /*min=*/GoogleServiceAuthError::State::INVALID_GAIA_CREDENTIALS, + /*count=*/1))); +} + +// Disabled for chromeos, which doesn't allow for there not to be a signed in +// user. +#if !defined(OS_CHROMEOS) +TEST_F(FeedNetworkTest, ShouldIncludeAPIKeyForNoSignedInUser) { + identity_env()->ClearPrimaryAccount(); + CallbackReceiver<QueryRequestResult> receiver; + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + + network::ResourceRequest resource_request = + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_OK); + + EXPECT_THAT(resource_request.url.spec(), + testing::HasSubstr("key=dummy_api_key")); +} +#endif + +TEST_F(FeedNetworkTest, TestDurationHistogram) { + base::HistogramTester histogram_tester; + CallbackReceiver<QueryRequestResult> receiver; + const TimeDelta kDuration = TimeDelta::FromMilliseconds(12345); + + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + task_environment_.FastForwardBy(kDuration); + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_OK); + + EXPECT_TRUE(receiver.GetResult()); + histogram_tester.ExpectTimeBucketCount( + "ContentSuggestions.Feed.Network.Duration", kDuration, 1); +} + +// Verify that the kHostOverrideHost pref overrides the feed host +// and updates the Bless nonce if one sent in the response. +TEST_F(FeedNetworkTest, TestHostOverrideWithAuthHeader) { + CallbackReceiver<QueryRequestResult> receiver; + profile_prefs().SetString(feed::prefs::kHostOverrideHost, + "http://www.newhost.com/"); + feed_network()->SendQueryRequest(GetTestFeedRequest(), receiver.Bind()); + + response_headers_ = base::MakeRefCounted<net::HttpResponseHeaders>( + net::HttpUtil::AssembleRawHeaders( + "HTTP/1.1 401 Unauthorized\nwww-authenticate: Foo " + "nonce=\"1234123412341234\"\n\n")); + RespondToQueryRequest(GetTestFeedResponse(), net::HTTP_FORBIDDEN); + + EXPECT_TRUE(receiver.GetResult()); + EXPECT_EQ("1234123412341234", + profile_prefs().GetString(feed::prefs::kHostOverrideBlessNonce)); +} + +TEST_F(FeedNetworkTest, SendActionRequest) { + CallbackReceiver<ActionRequestResult> receiver; + feed_network()->SendActionRequest(GetTestActionRequest(), receiver.Bind()); + RespondToActionRequest(GetTestActionResponse(), net::HTTP_OK); + + ASSERT_TRUE(receiver.GetResult()); + const ActionRequestResult& result = *receiver.GetResult(); + EXPECT_EQ(net::HTTP_OK, result.status_code); + EXPECT_TRUE(result.response_body); +} + +TEST_F(FeedNetworkTest, SendActionRequestSendsValidRequest) { + CallbackReceiver<ActionRequestResult> receiver; + feed_network()->SendActionRequest(GetTestActionRequest(), receiver.Bind()); + network::ResourceRequest resource_request = + RespondToActionRequest(GetTestActionResponse(), net::HTTP_OK); + + EXPECT_EQ( + GURL("https://www.google.com/httpservice/retry/ClankActionUploadService/" + "ClankActionUpload?fmt=bin&hl=en"), + resource_request.url); + + EXPECT_EQ("POST", resource_request.method); + std::string content_encoding; + EXPECT_TRUE(resource_request.headers.GetHeader("content-encoding", + &content_encoding)); + EXPECT_EQ("gzip", content_encoding); + std::string authorization; + EXPECT_TRUE( + resource_request.headers.GetHeader("Authorization", &authorization)); + EXPECT_EQ(authorization, "Bearer access_token"); + + // Check that the body content is correct. This requires some work to extract + // the bytes and unzip them. + auto* elements = resource_request.request_body->elements(); + ASSERT_TRUE(elements); + ASSERT_EQ(1UL, elements->size()); + std::string sent_body((*elements)[0].bytes(), (*elements)[0].length()); + std::string sent_body_uncompressed; + ASSERT_TRUE(compression::GzipUncompress(sent_body, &sent_body_uncompressed)); + std::string expected_body; + ASSERT_TRUE(GetTestActionRequest().SerializeToString(&expected_body)); + EXPECT_EQ(expected_body, sent_body_uncompressed); +} + +} // namespace +} // namespace feed diff --git a/chromium/components/feed/core/v2/feed_store.cc b/chromium/components/feed/core/v2/feed_store.cc new file mode 100644 index 00000000000..7321e84fa8f --- /dev/null +++ b/chromium/components/feed/core/v2/feed_store.cc @@ -0,0 +1,388 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_store.h" + +#include <utility> + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/containers/flat_set.h" +#include "base/files/file_path.h" +#include "base/logging.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_piece.h" +#include "base/task/post_task.h" +#include "base/task/thread_pool.h" +#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/leveldb_proto/public/proto_database_provider.h" + +namespace feed { +namespace { + +// Keys are defined as: +// 'S/<stream-id>' -> stream_data +// 'T/<stream-id>/<sequence-number>' -> stream_structures +// 'c/<content-id>' -> content +// 'a/<id>' -> action +// 's/<content-id>' -> shared_state +// 'N' -> next_stream_state +constexpr char kMainStreamId[] = "0"; +const char kStreamDataKey[] = "S/0"; +const char kLocalActionPrefix[] = "a/"; +const char kNextStreamStateKey[] = "N"; + +leveldb::ReadOptions CreateReadOptions() { + leveldb::ReadOptions opts; + opts.fill_cache = false; + return opts; +} + +std::string KeyForContentId(base::StringPiece prefix, + const feedwire::ContentId& content_id) { + return base::StrCat({prefix, content_id.content_domain(), ",", + base::NumberToString(content_id.type()), ",", + base::NumberToString(content_id.id())}); +} + +std::string ContentKey(const feedwire::ContentId& content_id) { + return KeyForContentId("c/", content_id); +} + +std::string SharedStateKey(const feedwire::ContentId& content_id) { + return KeyForContentId("s/", content_id); +} + +std::string KeyForRecord(const feedstore::Record& record) { + switch (record.data_case()) { + case feedstore::Record::kStreamData: + return kStreamDataKey; + case feedstore::Record::kStreamStructures: + return base::StrCat( + {"T/", record.stream_structures().stream_id(), "/", + base::NumberToString(record.stream_structures().sequence_number())}); + case feedstore::Record::kContent: + return ContentKey(record.content().content_id()); + case feedstore::Record::kLocalAction: + return kLocalActionPrefix + + base::NumberToString(record.local_action().id()); + case feedstore::Record::kSharedState: + return SharedStateKey(record.shared_state().content_id()); + case feedstore::Record::kNextStreamState: + return kNextStreamStateKey; + case feedstore::Record::DATA_NOT_SET: // fall through + NOTREACHED() << "Invalid record case " << record.data_case(); + return ""; + } +} + +bool FilterByKey(const base::flat_set<std::string>& key_set, + const std::string& key) { + return key_set.contains(key); +} + +feedstore::Record MakeRecord(feedstore::Content content) { + feedstore::Record record; + *record.mutable_content() = std::move(content); + return record; +} + +feedstore::Record MakeRecord( + feedstore::StreamStructureSet stream_structure_set) { + feedstore::Record record; + *record.mutable_stream_structures() = std::move(stream_structure_set); + return record; +} + +feedstore::Record MakeRecord(feedstore::StreamSharedState shared_state) { + feedstore::Record record; + *record.mutable_shared_state() = std::move(shared_state); + return record; +} + +feedstore::Record MakeRecord(feedstore::StreamData stream_data) { + feedstore::Record record; + *record.mutable_stream_data() = std::move(stream_data); + return record; +} + +template <typename T> +std::pair<std::string, feedstore::Record> MakeKeyAndRecord(T record_data) { + std::pair<std::string, feedstore::Record> result; + result.second = MakeRecord(std::move(record_data)); + result.first = KeyForRecord(result.second); + return result; +} + +} // namespace + +FeedStore::LoadStreamResult::LoadStreamResult() = default; +FeedStore::LoadStreamResult::~LoadStreamResult() = default; +FeedStore::LoadStreamResult::LoadStreamResult(LoadStreamResult&&) = default; +FeedStore::LoadStreamResult& FeedStore::LoadStreamResult::operator=( + LoadStreamResult&&) = default; + +FeedStore::FeedStore( + std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> database) + : database_status_(leveldb_proto::Enums::InitStatus::kNotInitialized), + database_(std::move(database)) { +} + +FeedStore::~FeedStore() = default; + +void FeedStore::Initialize(base::OnceClosure initialize_complete) { + if (IsInitialized()) { + std::move(initialize_complete).Run(); + } else { + initialize_callback_ = std::move(initialize_complete); + database_->Init(base::BindOnce(&FeedStore::OnDatabaseInitialized, + weak_ptr_factory_.GetWeakPtr())); + } +} + +void FeedStore::OnDatabaseInitialized(leveldb_proto::Enums::InitStatus status) { + database_status_ = status; + if (initialize_callback_) + std::move(initialize_callback_).Run(); +} + +bool FeedStore::IsInitialized() const { + return database_status_ == leveldb_proto::Enums::InitStatus::kOK; +} + +bool FeedStore::IsInitializedForTesting() const { + return IsInitialized(); +} + +void FeedStore::ReadSingle( + const std::string& key, + base::OnceCallback<void(bool, std::unique_ptr<feedstore::Record>)> + callback) { + if (!IsInitialized()) { + std::move(callback).Run(false, nullptr); + return; + } + + database_->GetEntry(key, std::move(callback)); +} + +void FeedStore::ReadMany( + const base::flat_set<std::string>& key_set, + base::OnceCallback< + void(bool, std::unique_ptr<std::vector<feedstore::Record>>)> callback) { + if (!IsInitialized()) { + std::move(callback).Run(false, nullptr); + return; + } + + database_->LoadEntriesWithFilter( + base::BindRepeating(&FilterByKey, std::move(key_set)), + CreateReadOptions(), + /*target_prefix=*/"", std::move(callback)); +} + +void FeedStore::LoadStream( + base::OnceCallback<void(LoadStreamResult)> callback) { + if (!IsInitialized()) { + LoadStreamResult result; + result.read_error = true; + std::move(callback).Run(std::move(result)); + return; + } + auto filter = [](const std::string& key) { + return key == "S/0" || (key.size() > 3 && key[0] == 'T' && key[1] == '/' && + key[2] == '0' && key[3] == '/'); + }; + database_->LoadEntriesWithFilter( + base::BindRepeating(filter), CreateReadOptions(), + /*target_prefix=*/"", + base::BindOnce(&FeedStore::OnLoadStreamFinished, base::Unretained(this), + std::move(callback))); +} + +void FeedStore::OnLoadStreamFinished( + base::OnceCallback<void(LoadStreamResult)> callback, + bool success, + std::unique_ptr<std::vector<feedstore::Record>> records) { + LoadStreamResult result; + if (!records || !success) { + result.read_error = true; + } else { + for (feedstore::Record& record : *records) { + if (record.has_stream_structures()) { + result.stream_structures.push_back( + std::move(*record.mutable_stream_structures())); + } else if (record.has_stream_data()) { + result.stream_data = std::move(*record.mutable_stream_data()); + } + } + } + std::move(callback).Run(std::move(result)); +} + +void FeedStore::SaveFullStream( + std::unique_ptr<StreamModelUpdateRequest> update_request, + base::OnceCallback<void(bool)> callback) { + auto updates = std::make_unique< + std::vector<std::pair<std::string, feedstore::Record>>>(); + updates->push_back(MakeKeyAndRecord(std::move(update_request->stream_data))); + for (feedstore::Content& content : update_request->content) { + updates->push_back(MakeKeyAndRecord(std::move(content))); + } + for (feedstore::StreamSharedState& shared_state : + update_request->shared_states) { + updates->push_back(MakeKeyAndRecord(std::move(shared_state))); + } + feedstore::StreamStructureSet stream_structure_set; + stream_structure_set.set_stream_id(kMainStreamId); + for (feedstore::StreamStructure& structure : + update_request->stream_structures) { + *stream_structure_set.add_structures() = std::move(structure); + } + updates->push_back(MakeKeyAndRecord(std::move(stream_structure_set))); + + // Set up a filter to delete all stream-related data. + // But we need to exclude keys being written right now. + std::vector<std::string> key_vector(updates->size()); + for (size_t i = 0; i < key_vector.size(); ++i) { + key_vector[i] = (*updates)[i].first; + } + base::flat_set<std::string> updated_keys(std::move(key_vector)); + + auto filter = [](const base::flat_set<std::string>& updated_keys, + const std::string& key) { + if (key.empty() || updated_keys.contains(key)) + return false; + return key[0] == 'S' || key[0] == 'T' || key[0] == 'c' || key[0] == 's' || + key[0] == 'N'; + }; + + database_->UpdateEntriesWithRemoveFilter( + std::move(updates), base::BindRepeating(filter, std::move(updated_keys)), + base::BindOnce(&FeedStore::OnSaveStreamEntriesUpdated, + base::Unretained(this), std::move(callback))); +} + +void FeedStore::OnSaveStreamEntriesUpdated( + base::OnceCallback<void(bool)> complete_callback, + bool ok) { + std::move(complete_callback).Run(ok); +} + +void FeedStore::WriteOperations( + int32_t sequence_number, + std::vector<feedstore::DataOperation> operations) { + std::vector<feedstore::Record> records; + feedstore::Record structures_record; + feedstore::StreamStructureSet& structure_set = + *structures_record.mutable_stream_structures(); + for (feedstore::DataOperation& operation : operations) { + *structure_set.add_structures() = std::move(*operation.mutable_structure()); + if (operation.has_content()) { + feedstore::Record record; + record.set_allocated_content(operation.release_content()); + records.push_back(std::move(record)); + } + } + structure_set.set_stream_id(kMainStreamId); + structure_set.set_sequence_number(sequence_number); + + records.push_back(std::move(structures_record)); + Write(std::move(records), base::DoNothing()); +} + +void FeedStore::ReadContent( + std::vector<feedwire::ContentId> content_ids, + std::vector<feedwire::ContentId> shared_state_ids, + base::OnceCallback<void(std::vector<feedstore::Content>, + std::vector<feedstore::StreamSharedState>)> + content_callback) { + std::vector<std::string> key_vector; + key_vector.reserve(content_ids.size() + shared_state_ids.size()); + for (const auto& content_id : content_ids) + key_vector.push_back(ContentKey(content_id)); + for (const auto& content_id : shared_state_ids) + key_vector.push_back(SharedStateKey(content_id)); + + for (const auto& shared_state_id : shared_state_ids) + key_vector.push_back(SharedStateKey(shared_state_id)); + + ReadMany(base::flat_set<std::string>(std::move(key_vector)), + base::BindOnce(&FeedStore::OnReadContentFinished, + weak_ptr_factory_.GetWeakPtr(), + std::move(content_callback))); +} + +void FeedStore::OnReadContentFinished( + base::OnceCallback<void(std::vector<feedstore::Content>, + std::vector<feedstore::StreamSharedState>)> + callback, + bool success, + std::unique_ptr<std::vector<feedstore::Record>> records) { + if (!success || !records) { + std::move(callback).Run({}, {}); + return; + } + + std::vector<feedstore::Content> content; + // Most of records will be content. + content.reserve(records->size()); + std::vector<feedstore::StreamSharedState> shared_states; + for (auto& record : *records) { + if (record.data_case() == feedstore::Record::kContent) + content.push_back(std::move(record.content())); + else if (record.data_case() == feedstore::Record::kSharedState) + shared_states.push_back(std::move(record.shared_state())); + } + + std::move(callback).Run(std::move(content), std::move(shared_states)); +} + +void FeedStore::ReadNextStreamState( + base::OnceCallback<void(std::unique_ptr<feedstore::StreamAndContentState>)> + callback) { + ReadSingle( + kNextStreamStateKey, + base::BindOnce(&FeedStore::OnReadNextStreamStateFinished, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void FeedStore::OnReadNextStreamStateFinished( + base::OnceCallback<void(std::unique_ptr<feedstore::StreamAndContentState>)> + callback, + bool success, + std::unique_ptr<feedstore::Record> record) { + if (!success || !record) { + std::move(callback).Run(nullptr); + return; + } + + std::move(callback).Run( + base::WrapUnique(record->release_next_stream_state())); +} + +void FeedStore::Write(std::vector<feedstore::Record> records, + base::OnceCallback<void(bool)> callback) { + auto entries_to_save = std::make_unique< + leveldb_proto::ProtoDatabase<feedstore::Record>::KeyEntryVector>(); + for (auto& record : records) { + std::string key = KeyForRecord(record); + if (!key.empty()) + entries_to_save->push_back({std::move(key), std::move(record)}); + } + + database_->UpdateEntries( + std::move(entries_to_save), + /*keys_to_remove=*/std::make_unique<leveldb_proto::KeyVector>(), + base::BindOnce(&FeedStore::OnWriteFinished, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); +} + +void FeedStore::OnWriteFinished(base::OnceCallback<void(bool)> callback, + bool success) { + std::move(callback).Run(success); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/feed_store.h b/chromium/components/feed/core/v2/feed_store.h new file mode 100644 index 00000000000..3ae93ead5b6 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_store.h @@ -0,0 +1,129 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_FEED_STORE_H_ +#define COMPONENTS_FEED_CORE_V2_FEED_STORE_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "base/containers/flat_set.h" +#include "base/memory/weak_ptr.h" +#include "base/sequenced_task_runner.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/leveldb_proto/public/proto_database.h" +#include "components/leveldb_proto/public/proto_database_provider.h" + +namespace feed { +struct StreamModelUpdateRequest; + +class FeedStore { + public: + struct LoadStreamResult { + LoadStreamResult(); + ~LoadStreamResult(); + LoadStreamResult(LoadStreamResult&&); + LoadStreamResult& operator=(LoadStreamResult&&); + LoadStreamResult(const LoadStreamResult&) = delete; + LoadStreamResult& operator=(const LoadStreamResult&) = delete; + + bool read_error = false; + feedstore::StreamData stream_data; + std::vector<feedstore::StreamStructureSet> stream_structures; + }; + + explicit FeedStore( + std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> + database); + ~FeedStore(); + FeedStore(const FeedStore&) = delete; + FeedStore& operator=(const FeedStore&) = delete; + + void Initialize(base::OnceClosure initialize_complete); + + void LoadStream(base::OnceCallback<void(LoadStreamResult)> callback); + + void SaveFullStream(std::unique_ptr<StreamModelUpdateRequest> update_request, + base::OnceCallback<void(bool)> callback); + + void WriteOperations(int32_t sequence_number, + std::vector<feedstore::DataOperation> operations); + + // Read StreamData and pass it to stream_data_callback, or nullptr on failure. + void ReadStreamData( + base::OnceCallback<void(std::unique_ptr<feedstore::StreamData>)> + stream_data_callback); + + // Read Content and StreamSharedStates and pass them to content_callback, or + // nullptrs on failure. + void ReadContent( + std::vector<feedwire::ContentId> content_ids, + std::vector<feedwire::ContentId> shared_state_ids, + base::OnceCallback<void(std::vector<feedstore::Content>, + std::vector<feedstore::StreamSharedState>)> + content_callback); + + void ReadNextStreamState( + base::OnceCallback< + void(std::unique_ptr<feedstore::StreamAndContentState>)> callback); + + // TODO(iwells): implement reading stored actions + + // TODO(iwells): implement this + // Deletes old records that are no longer needed + // void RemoveOldData(base::OnceCallback<void(bool)> callback); + + bool IsInitializedForTesting() const; + + private: + void OnDatabaseInitialized(leveldb_proto::Enums::InitStatus status); + bool IsInitialized() const; + + void Write(std::vector<feedstore::Record> records, + base::OnceCallback<void(bool)> callback); + + void ReadSingle( + const std::string& key, + base::OnceCallback<void(bool, std::unique_ptr<feedstore::Record>)> + callback); + void ReadMany(const base::flat_set<std::string>& key_set, + base::OnceCallback< + void(bool, std::unique_ptr<std::vector<feedstore::Record>>)> + callback); + void OnSaveStreamEntriesUpdated( + base::OnceCallback<void(bool)> complete_callback, + bool ok); + void OnLoadStreamFinished( + base::OnceCallback<void(LoadStreamResult)> callback, + bool success, + std::unique_ptr<std::vector<feedstore::Record>> records); + void OnReadContentFinished( + base::OnceCallback<void(std::vector<feedstore::Content>, + std::vector<feedstore::StreamSharedState>)> + callback, + bool success, + std::unique_ptr<std::vector<feedstore::Record>> records); + void OnReadNextStreamStateFinished( + base::OnceCallback< + void(std::unique_ptr<feedstore::StreamAndContentState>)> callback, + bool success, + std::unique_ptr<feedstore::Record> record); + + void OnWriteFinished(base::OnceCallback<void(bool)> callback, bool success); + + // TODO(iwells): implement + // bool OldRecordFilter(const std::string& key); + // void OnRemoveOldDataFinished(base::OnceCallback<void(bool)> callback, + // bool success); + + base::OnceClosure initialize_callback_; + leveldb_proto::Enums::InitStatus database_status_; + std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> database_; + base::WeakPtrFactory<FeedStore> weak_ptr_factory_{this}; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_FEED_STORE_H_ diff --git a/chromium/components/feed/core/v2/feed_store_unittest.cc b/chromium/components/feed/core/v2/feed_store_unittest.cc new file mode 100644 index 00000000000..28e2043fc12 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_store_unittest.cc @@ -0,0 +1,454 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_store.h" + +#include <map> +#include <set> +#include <utility> + +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_piece.h" +#include "base/test/bind_test_util.h" +#include "base/test/task_environment.h" +#include "components/feed/core/proto/v2/wire/content_id.pb.h" +#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/test/callback_receiver.h" +#include "components/feed/core/v2/test/proto_printer.h" +#include "components/feed/core/v2/test/stream_builder.h" +#include "components/leveldb_proto/testing/fake_db.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { + +const char kNextPageToken[] = "next page token"; +const char kConsistencyToken[] = "consistency token"; +const int64_t kLastAddedTimeMs = 100; + +using LoadStreamResult = FeedStore::LoadStreamResult; + +feedstore::StreamData MakeStreamData() { + feedstore::StreamData stream_data; + *stream_data.mutable_content_id() = MakeRootId(); + stream_data.set_next_page_token(kNextPageToken); + stream_data.set_consistency_token(kConsistencyToken); + stream_data.set_last_added_time_millis(kLastAddedTimeMs); + + return stream_data; +} + +std::string KeyForContentId(base::StringPiece prefix, + const feedwire::ContentId& content_id) { + return base::StrCat({prefix, content_id.content_domain(), ",", + base::NumberToString(content_id.type()), ",", + base::NumberToString(content_id.id())}); +} + +feedstore::Record RecordForContent(feedstore::Content content) { + feedstore::Record record; + *record.mutable_content() = std::move(content); + return record; +} + +feedstore::Record RecordForSharedState(feedstore::StreamSharedState shared) { + feedstore::Record record; + *record.mutable_shared_state() = std::move(shared); + return record; +} + +} // namespace + +class FeedStoreTest : public testing::Test { + protected: + void MakeFeedStore(std::map<std::string, feedstore::Record> entries, + leveldb_proto::Enums::InitStatus init_status = + leveldb_proto::Enums::InitStatus::kOK) { + db_entries_ = std::move(entries); + auto fake_db = + std::make_unique<leveldb_proto::test::FakeDB<feedstore::Record>>( + &db_entries_); + fake_db_ = fake_db.get(); + store_ = std::make_unique<FeedStore>(std::move(fake_db)); + store_->Initialize(base::DoNothing()); + fake_db_->InitStatusCallback(init_status); + } + + std::set<std::string> StoredKeys() { + std::set<std::string> result; + for (auto& entry : db_entries_) { + result.insert(entry.first); + } + return result; + } + + std::string StoreToString() { + std::stringstream ss; + for (auto& entry : db_entries_) { + ss << "[" << entry.first << "] " << entry.second; + } + return ss.str(); + } + + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::SYSTEM_TIME}; + std::unique_ptr<FeedStore> store_; + std::map<std::string, feedstore::Record> db_entries_; + leveldb_proto::test::FakeDB<feedstore::Record>* fake_db_; +}; + +TEST_F(FeedStoreTest, InitSuccess) { + MakeFeedStore({}); + EXPECT_TRUE(store_->IsInitializedForTesting()); +} + +TEST_F(FeedStoreTest, InitFailure) { + std::map<std::string, feedstore::Record> entries; + auto fake_db = + std::make_unique<leveldb_proto::test::FakeDB<feedstore::Record>>( + &entries); + leveldb_proto::test::FakeDB<feedstore::Record>* fake_db_raw = fake_db.get(); + auto store = std::make_unique<FeedStore>(std::move(fake_db)); + + store->Initialize(base::DoNothing()); + EXPECT_FALSE(store->IsInitializedForTesting()); + + fake_db_raw->InitStatusCallback(leveldb_proto::Enums::InitStatus::kError); + EXPECT_FALSE(store->IsInitializedForTesting()); +} + +TEST_F(FeedStoreTest, SaveFullStream) { + MakeFeedStore({}); + CallbackReceiver<bool> receiver; + store_->SaveFullStream(MakeTypicalInitialModelState(), receiver.Bind()); + fake_db_->UpdateCallback(true); + + ASSERT_TRUE(receiver.GetResult()); + + EXPECT_EQ(StoreToString(), R"([S/0] { + stream_data { + content_id { + content_domain: "root" + } + shared_state_id { + content_domain: "render_data" + } + } +} +[T/0/0] { + stream_structures { + stream_id: "0" + structures { + operation: 1 + } + structures { + operation: 2 + content_id { + content_domain: "root" + } + type: 1 + } + structures { + operation: 2 + content_id { + content_domain: "content" + type: 3 + } + parent_id { + content_domain: "root" + } + type: 4 + } + structures { + operation: 2 + content_id { + content_domain: "stories" + type: 4 + } + parent_id { + content_domain: "content" + type: 3 + } + type: 3 + } + structures { + operation: 2 + content_id { + content_domain: "content" + type: 3 + id: 1 + } + parent_id { + content_domain: "root" + } + type: 4 + } + structures { + operation: 2 + content_id { + content_domain: "stories" + type: 4 + id: 1 + } + parent_id { + content_domain: "content" + type: 3 + id: 1 + } + type: 3 + } + } +} +[c/stories,4,0] { + content { + content_id { + content_domain: "stories" + type: 4 + } + frame: "f:0" + } +} +[c/stories,4,1] { + content { + content_id { + content_domain: "stories" + type: 4 + id: 1 + } + frame: "f:1" + } +} +[s/render_data,0,0] { + shared_state { + content_id { + content_domain: "render_data" + } + shared_state_data: "ss:0" + } +} +)"); +} + +TEST_F(FeedStoreTest, SaveFullStreamOverwritesData) { + MakeFeedStore({}); + // Insert some junk that should be removed. + db_entries_["S/0"].mutable_local_action()->set_id(6); + db_entries_["T/0/0"].mutable_local_action()->set_id(6); + db_entries_["T/0/73"].mutable_local_action()->set_id(6); + db_entries_["c/stories,4,0"].mutable_local_action()->set_id(6); + db_entries_["c/stories,4,1"].mutable_local_action()->set_id(6); + db_entries_["c/garbage"].mutable_local_action()->set_id(6); + db_entries_["s/render_data,0,0"].mutable_local_action()->set_id(6); + db_entries_["s/garbage,0,0"].mutable_local_action()->set_id(6); + + CallbackReceiver<bool> receiver; + store_->SaveFullStream(MakeTypicalInitialModelState(), receiver.Bind()); + fake_db_->UpdateCallback(true); + + ASSERT_TRUE(receiver.GetResult()); + ASSERT_EQ(std::set<std::string>({ + "S/0", + "T/0/0", + "c/stories,4,0", + "c/stories,4,1", + "s/render_data,0,0", + }), + StoredKeys()); + + for (std::string key : StoredKeys()) { + EXPECT_FALSE(db_entries_[key].has_local_action()) + << "Found local action at key " << key + << ", did SaveFullStream erase everything?"; + } +} + +TEST_F(FeedStoreTest, LoadStreamSuccess) { + MakeFeedStore({}); + store_->SaveFullStream(MakeTypicalInitialModelState(), base::DoNothing()); + fake_db_->UpdateCallback(true); + + CallbackReceiver<LoadStreamResult> receiver; + store_->LoadStream(receiver.Bind()); + fake_db_->LoadCallback(true); + + ASSERT_TRUE(receiver.GetResult()); + EXPECT_FALSE(receiver.GetResult()->read_error); + EXPECT_EQ(ToTextProto(MakeRootId()), + ToTextProto(receiver.GetResult()->stream_data.content_id())); +} + +TEST_F(FeedStoreTest, LoadStreamFail) { + MakeFeedStore({}); + store_->SaveFullStream(MakeTypicalInitialModelState(), base::DoNothing()); + fake_db_->UpdateCallback(true); + + CallbackReceiver<LoadStreamResult> receiver; + store_->LoadStream(receiver.Bind()); + fake_db_->LoadCallback(false); + + ASSERT_TRUE(receiver.GetResult()); + EXPECT_TRUE(receiver.GetResult()->read_error); +} + +TEST_F(FeedStoreTest, LoadStreamNoData) { + MakeFeedStore({}); + + CallbackReceiver<LoadStreamResult> receiver; + store_->LoadStream(receiver.Bind()); + fake_db_->LoadCallback(true); + + ASSERT_TRUE(receiver.GetResult()); + EXPECT_FALSE(receiver.GetResult()->stream_data.has_content_id()); +} + +TEST_F(FeedStoreTest, WriteOperations) { + MakeFeedStore({}); + CallbackReceiver<LoadStreamResult> receiver; + store_->WriteOperations(5, {MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeCluster(6, MakeRootId()))}); + fake_db_->UpdateCallback(true); + + EXPECT_EQ(StoreToString(), R"([T/0/5] { + stream_structures { + stream_id: "0" + sequence_number: 5 + structures { + operation: 2 + content_id { + content_domain: "content" + type: 3 + id: 2 + } + parent_id { + content_domain: "root" + } + type: 4 + } + structures { + operation: 2 + content_id { + content_domain: "content" + type: 3 + id: 6 + } + parent_id { + content_domain: "root" + } + type: 4 + } + } +} +)"); +} + +TEST_F(FeedStoreTest, ReadNonexistentContentAndSharedStates) { + MakeFeedStore({}); + + bool did_read = false; + store_->ReadContent( + {MakeContentContentId(0)}, {MakeSharedStateContentId(0)}, + base::BindLambdaForTesting( + [&](std::vector<feedstore::Content> content, + std::vector<feedstore::StreamSharedState> shared_states) { + did_read = true; + EXPECT_EQ(content.size(), 0ul); + EXPECT_EQ(shared_states.size(), 0ul); + })); + fake_db_->LoadCallback(true); + EXPECT_TRUE(did_read); +} + +TEST_F(FeedStoreTest, ReadContentAndSharedStates) { + feedstore::Content content1 = MakeContent(1); + feedstore::Content content2 = MakeContent(2); + feedstore::StreamSharedState shared1 = MakeSharedState(1); + feedstore::StreamSharedState shared2 = MakeSharedState(2); + + MakeFeedStore({{KeyForContentId("c/", content1.content_id()), + RecordForContent(content1)}, + {KeyForContentId("c/", content2.content_id()), + RecordForContent(content2)}, + {KeyForContentId("s/", shared1.content_id()), + RecordForSharedState(shared1)}, + {KeyForContentId("s/", shared2.content_id()), + RecordForSharedState(shared2)}}); + + std::vector<feedwire::ContentId> content_ids = {content1.content_id(), + content2.content_id()}; + std::vector<feedwire::ContentId> shared_state_ids = {shared1.content_id(), + shared2.content_id()}; + + // Successful read + bool did_successful_read = false; + store_->ReadContent( + content_ids, shared_state_ids, + base::BindLambdaForTesting( + [&](std::vector<feedstore::Content> content, + std::vector<feedstore::StreamSharedState> shared_states) { + did_successful_read = true; + ASSERT_EQ(content.size(), 2ul); + EXPECT_EQ(ToTextProto(content[0].content_id()), + ToTextProto(content1.content_id())); + EXPECT_EQ(content[0].frame(), content1.frame()); + + ASSERT_EQ(shared_states.size(), 2ul); + EXPECT_EQ(ToTextProto(shared_states[0].content_id()), + ToTextProto(shared1.content_id())); + EXPECT_EQ(shared_states[0].shared_state_data(), + shared1.shared_state_data()); + })); + fake_db_->LoadCallback(true); + EXPECT_TRUE(did_successful_read); + + // Failed read + bool did_failed_read = false; + store_->ReadContent( + content_ids, shared_state_ids, + base::BindLambdaForTesting( + [&](std::vector<feedstore::Content> content, + std::vector<feedstore::StreamSharedState> shared_states) { + did_failed_read = true; + EXPECT_EQ(content.size(), 0ul); + EXPECT_EQ(shared_states.size(), 0ul); + })); + fake_db_->LoadCallback(false); + EXPECT_TRUE(did_failed_read); +} + +TEST_F(FeedStoreTest, ReadNextStreamState) { + feedstore::Record record; + feedstore::StreamAndContentState* next_stream_state = + record.mutable_next_stream_state(); + *next_stream_state->mutable_stream_data() = MakeStreamData(); + *next_stream_state->add_content() = MakeContent(0); + *next_stream_state->add_shared_state() = MakeSharedState(0); + + MakeFeedStore({{"N", record}}); + + // Successful read + bool did_successful_read = false; + store_->ReadNextStreamState(base::BindLambdaForTesting( + [&](std::unique_ptr<feedstore::StreamAndContentState> result) { + did_successful_read = true; + ASSERT_TRUE(result); + EXPECT_TRUE(result->has_stream_data()); + EXPECT_EQ(result->content_size(), 1); + EXPECT_EQ(result->shared_state_size(), 1); + })); + fake_db_->GetCallback(true); + EXPECT_TRUE(did_successful_read); + + // Failed read + bool did_failed_read = false; + store_->ReadNextStreamState(base::BindLambdaForTesting( + [&](std::unique_ptr<feedstore::StreamAndContentState> result) { + did_failed_read = true; + EXPECT_FALSE(result); + })); + fake_db_->GetCallback(false); + EXPECT_TRUE(did_failed_read); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/feed_stream.cc b/chromium/components/feed/core/v2/feed_stream.cc new file mode 100644 index 00000000000..fe1e13c33f6 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_stream.cc @@ -0,0 +1,448 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_stream.h" + +#include <set> +#include <string> +#include <utility> + +#include "base/bind.h" +#include "base/metrics/histogram_macros.h" +#include "base/time/clock.h" +#include "base/time/tick_clock.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/ui.pb.h" +#include "components/feed/core/shared_prefs/pref_names.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/refresh_task_scheduler.h" +#include "components/feed/core/v2/scheduling.h" +#include "components/feed/core/v2/stream_model.h" +#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/tasks/load_stream_task.h" +#include "components/feed/core/v2/tasks/wait_for_store_initialize_task.h" +#include "components/prefs/pref_service.h" + +namespace feed { + +// Tracks UI changes in |StreamModel| and forwards them to |SurfaceInterface|s. +// TODO(harringtond): implement spinner slice. +class FeedStream::SurfaceUpdater : public StreamModel::Observer { + public: + using ContentRevision = ContentRevision; + explicit SurfaceUpdater(const base::ObserverList<SurfaceInterface>* surfaces) + : surfaces_(surfaces) {} + ~SurfaceUpdater() override = default; + SurfaceUpdater(const SurfaceUpdater&) = delete; + SurfaceUpdater& operator=(const SurfaceUpdater&) = delete; + + void SetModel(StreamModel* model) { + if (model_ == model) + return; + if (model_) + model_->SetObserver(nullptr); + model_ = model; + if (model_) { + model_->SetObserver(this); + + const std::vector<ContentRevision>& content_list = + model_->GetContentList(); + current_content_set_.insert(content_list.begin(), content_list.end()); + for (SurfaceInterface& surface : *surfaces_) { + surface.StreamUpdate(GetUpdateForNewSurface(*model_)); + } + } + } + + // StreamModel::Observer. + void OnUiUpdate(const StreamModel::UiUpdate& update) override { + DCHECK(model_); // The update comes from the model. + + if (!update.content_list_changed) + return; + feedui::StreamUpdate stream_update; + const std::vector<ContentRevision>& content_list = model_->GetContentList(); + for (ContentRevision content_revision : content_list) { + AddSliceUpdate(*model_, content_revision, + current_content_set_.count(content_revision) == 0, + &stream_update); + } + for (const StreamModel::UiUpdate::SharedStateInfo& info : + update.shared_states) { + if (info.updated) + AddSharedState(*model_, info.shared_state_id, &stream_update); + } + + current_content_set_.clear(); + current_content_set_.insert(content_list.begin(), content_list.end()); + + for (SurfaceInterface& surface : *surfaces_) { + surface.StreamUpdate(stream_update); + } + } + + // Sends the initial stream state to a newly connected surface. + void SurfaceAdded(SurfaceInterface* surface) { + if (model_) { + surface->StreamUpdate(GetUpdateForNewSurface(*model_)); + } + } + + void LoadStreamFailed(LoadStreamStatus load_stream_status) { + auto zero_state_type = feedui::ZeroStateSlice::NO_CARDS_AVAILABLE; + switch (load_stream_status) { + case LoadStreamStatus::kProtoTranslationFailed: + case LoadStreamStatus::kNoResponseBody: + case LoadStreamStatus::kCannotLoadFromNetworkOffline: + case LoadStreamStatus::kCannotLoadFromNetworkThrottled: + zero_state_type = feedui::ZeroStateSlice::CANT_REFRESH; + break; + default: + break; + } + // Note that with multiple surface, it's possible that we send a zero-state + // to a single surface multiple times. + for (SurfaceInterface& surface : *surfaces_) { + SendZeroStateUpdate(zero_state_type, &surface); + } + } + + private: + static std::string ToSliceId(ContentRevision content_revision) { + auto integer_value = content_revision.value(); + return std::string(reinterpret_cast<char*>(&integer_value), + sizeof(integer_value)); + } + + static feedui::StreamUpdate GetUpdateForNewSurface(const StreamModel& model) { + feedui::StreamUpdate result; + for (ContentRevision content_revision : model.GetContentList()) { + AddSliceUpdate(model, content_revision, /*is_content_new=*/true, &result); + } + for (std::string& id : model.GetSharedStateIds()) { + AddSharedState(model, id, &result); + } + + return result; + } + + static void SendZeroStateUpdate(feedui::ZeroStateSlice::Type zero_state_type, + SurfaceInterface* surface) { + feedui::StreamUpdate update; + feedui::Slice* slice = update.add_updated_slices()->mutable_slice(); + slice->mutable_zero_state_slice()->set_type(zero_state_type); + slice->set_slice_id("zero-state"); + surface->StreamUpdate(update); + } + + static void AddSharedState(const StreamModel& model, + const std::string& shared_state_id, + feedui::StreamUpdate* stream_update) { + const std::string* shared_state_data = + model.FindSharedStateData(shared_state_id); + if (!shared_state_data) + return; + feedui::SharedState* added_shared_state = + stream_update->add_new_shared_states(); + added_shared_state->set_id(shared_state_id); + added_shared_state->set_xsurface_shared_state(*shared_state_data); + } + + static void AddSliceUpdate(const StreamModel& model, + ContentRevision content_revision, + bool is_content_new, + feedui::StreamUpdate* stream_update) { + if (is_content_new) { + feedui::Slice* slice = + stream_update->add_updated_slices()->mutable_slice(); + slice->set_slice_id(ToSliceId(content_revision)); + const feedstore::Content* content = model.FindContent(content_revision); + DCHECK(content); + slice->mutable_xsurface_slice()->set_xsurface_frame(content->frame()); + } else { + stream_update->add_updated_slices()->set_slice_id( + ToSliceId(content_revision)); + } + } + + // Owned by |FeedStream|. + // Warning!: Null when the model is not yet loaded. + StreamModel* model_ = nullptr; + const base::ObserverList<SurfaceInterface>* surfaces_; + + std::set<ContentRevision> current_content_set_; +}; + +std::unique_ptr<StreamModelUpdateRequest> +FeedStream::WireResponseTranslator::TranslateWireResponse( + feedwire::Response response, + base::TimeDelta response_time, + base::Time current_time) { + return ::feed::TranslateWireResponse(std::move(response), response_time, + current_time); +} + +FeedStream::FeedStream( + RefreshTaskScheduler* refresh_task_scheduler, + EventObserver* stream_event_observer, + Delegate* delegate, + PrefService* profile_prefs, + FeedNetwork* feed_network, + FeedStore* feed_store, + const base::Clock* clock, + const base::TickClock* tick_clock, + scoped_refptr<base::SequencedTaskRunner> background_task_runner) + : refresh_task_scheduler_(refresh_task_scheduler), + stream_event_observer_(stream_event_observer), + delegate_(delegate), + profile_prefs_(profile_prefs), + feed_network_(feed_network), + store_(feed_store), + clock_(clock), + tick_clock_(tick_clock), + background_task_runner_(background_task_runner), + task_queue_(this), + user_classifier_(std::make_unique<UserClassifier>(profile_prefs, clock)), + request_throttler_(profile_prefs, clock) { + static WireResponseTranslator default_translator; + wire_response_translator_ = &default_translator; + + surface_updater_ = std::make_unique<SurfaceUpdater>(&surfaces_); + + // Inserting this task first ensures that |store_| is initialized before + // it is used. + task_queue_.AddTask(std::make_unique<WaitForStoreInitializeTask>(store_)); +} + +void FeedStream::InitializeScheduling() { + if (!IsArticlesListVisible()) { + refresh_task_scheduler_->Cancel(); + return; + } + + refresh_task_scheduler_->EnsureScheduled( + GetUserClassTriggerThreshold(GetUserClass(), TriggerType::kFixedTimer)); +} + +FeedStream::~FeedStream() = default; + +void FeedStream::TriggerStreamLoad() { + if (model_ || model_loading_in_progress_) + return; + + // If we should not load the stream, abort and send a zero-state update. + if (!IsArticlesListVisible()) { + LoadStreamTaskComplete(LoadStreamTask::Result( + LoadStreamStatus::kLoadNotAllowedArticlesListHidden)); + return; + } + if (!delegate_->IsEulaAccepted()) { + LoadStreamTaskComplete(LoadStreamTask::Result( + LoadStreamStatus::kLoadNotAllowedEulaNotAccepted)); + return; + } + + model_loading_in_progress_ = true; + task_queue_.AddTask(std::make_unique<LoadStreamTask>( + this, base::BindOnce(&FeedStream::LoadStreamTaskComplete, + base::Unretained(this)))); +} + +void FeedStream::LoadStreamTaskComplete(LoadStreamTask::Result result) { + stream_event_observer_->OnLoadStream(result.load_from_store_status, + result.final_status); + DVLOG(1) << "LoadStreamTaskComplete load_from_store_status=" + << result.load_from_store_status + << " final_status=" << result.final_status; + model_loading_in_progress_ = false; + + // If loading failed, update surfaces with an appropriate zero-state error. + if (!model_) { + surface_updater_->LoadStreamFailed(result.final_status); + } +} + +void FeedStream::AttachSurface(SurfaceInterface* surface) { + surfaces_.AddObserver(surface); + surface_updater_->SurfaceAdded(surface); + TriggerStreamLoad(); +} + +void FeedStream::DetachSurface(SurfaceInterface* surface) { + surfaces_.RemoveObserver(surface); +} + +void FeedStream::SetArticlesListVisible(bool is_visible) { + profile_prefs_->SetBoolean(prefs::kArticlesListVisible, is_visible); +} + +bool FeedStream::IsArticlesListVisible() { + return profile_prefs_->GetBoolean(prefs::kArticlesListVisible); +} + +void FeedStream::ExecuteOperations( + std::vector<feedstore::DataOperation> operations) { + if (!model_) { + DLOG(ERROR) << "Calling ExecuteOperations before the model is loaded"; + return; + } + return model_->ExecuteOperations(std::move(operations)); +} + +EphemeralChangeId FeedStream::CreateEphemeralChange( + std::vector<feedstore::DataOperation> operations) { + if (!model_) { + DLOG(ERROR) << "Calling CreateEphemeralChange before the model is loaded"; + return {}; + } + return model_->CreateEphemeralChange(std::move(operations)); +} + +bool FeedStream::CommitEphemeralChange(EphemeralChangeId id) { + if (!model_) + return false; + return model_->CommitEphemeralChange(id); +} + +bool FeedStream::RejectEphemeralChange(EphemeralChangeId id) { + if (!model_) + return false; + return model_->RejectEphemeralChange(id); +} + +UserClass FeedStream::GetUserClass() { + return user_classifier_->GetUserClass(); +} + +base::Time FeedStream::GetLastFetchTime() { + const base::Time fetch_time = + profile_prefs_->GetTime(feed::prefs::kLastFetchAttemptTime); + // Ignore impossible time values. + if (fetch_time > clock_->Now()) + return base::Time(); + return fetch_time; +} + +void FeedStream::LoadModelForTesting(std::unique_ptr<StreamModel> model) { + LoadModel(std::move(model)); +} +offline_pages::TaskQueue* FeedStream::GetTaskQueueForTesting() { + return &task_queue_; +} + +void FeedStream::OnTaskQueueIsIdle() { + if (idle_callback_) + idle_callback_.Run(); +} + +void FeedStream::SetIdleCallbackForTesting( + base::RepeatingClosure idle_callback) { + idle_callback_ = idle_callback; +} + +void FeedStream::SetUserClassifierForTesting( + std::unique_ptr<UserClassifier> user_classifier) { + user_classifier_ = std::move(user_classifier); +} + +void FeedStream::OnStoreChange(const StreamModel::StoreUpdate& update) { + store_->WriteOperations(update.sequence_number, update.operations); +} + +LoadStreamStatus FeedStream::ShouldMakeFeedQueryRequest() { + // TODO(harringtond): |suppress_refreshes_until_| was historically used for + // privacy purposes after clearing data to make sure sync data made it to the + // server. I'm not sure we need this now. But also, it was documented as not + // affecting manually triggered refreshes, but coded in a way that it does. + // I've tried to keep the same functionality as the old feed code, but we + // should revisit this. + if (tick_clock_->NowTicks() < suppress_refreshes_until_) { + return LoadStreamStatus::kCannotLoadFromNetworkSupressedForHistoryDelete; + } + + if (delegate_->IsOffline()) { + return LoadStreamStatus::kCannotLoadFromNetworkOffline; + } + + if (!request_throttler_.RequestQuota(NetworkRequestType::kFeedQuery)) { + return LoadStreamStatus::kCannotLoadFromNetworkThrottled; + } + + return LoadStreamStatus::kNoStatus; +} + +void FeedStream::OnEulaAccepted() { + MaybeTriggerRefresh(TriggerType::kForegrounded); +} + +void FeedStream::OnHistoryDeleted() { + // Due to privacy, we should not fetch for a while (unless the user + // explicitly asks for new suggestions) to give sync the time to propagate + // the changes in history to the server. + suppress_refreshes_until_ = + tick_clock_->NowTicks() + kSuppressRefreshDuration; + ClearAll(); +} + +void FeedStream::OnCacheDataCleared() { + ClearAll(); +} + +void FeedStream::OnSignedIn() { + ClearAll(); +} + +void FeedStream::OnSignedOut() { + ClearAll(); +} + +void FeedStream::OnEnterForeground() { + MaybeTriggerRefresh(TriggerType::kForegrounded); +} + +void FeedStream::ExecuteRefreshTask() { + if (!IsArticlesListVisible()) { + // While the check and cancel isn't strictly necessary, a long lived session + // could be issuing refreshes due to the background trigger while articles + // are not visible. + refresh_task_scheduler_->Cancel(); + return; + } + MaybeTriggerRefresh(TriggerType::kFixedTimer); +} + +void FeedStream::ClearAll() { + // TODO(harringtond): How should we handle in-progress tasks. + stream_event_observer_->OnClearAll(clock_->Now() - GetLastFetchTime()); + + // TODO(harringtond): This should result in clearing feed data + // and _maybe_ triggering refresh with TriggerType::kNtpShown. + // That work should be embedded in a task. +} + +void FeedStream::MaybeTriggerRefresh(TriggerType trigger, + bool clear_all_before_refresh) { + stream_event_observer_->OnMaybeTriggerRefresh(trigger, + clear_all_before_refresh); + // TODO(harringtond): Implement refresh (with LoadStreamTask). +} + +void FeedStream::LoadModel(std::unique_ptr<StreamModel> model) { + DCHECK(!model_); + model_ = std::move(model); + model_->SetStoreObserver(this); + surface_updater_->SetModel(model_.get()); +} + +void FeedStream::UnloadModel() { + if (!model_) + return; + surface_updater_->SetModel(nullptr); + model_.reset(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/feed_stream.h b/chromium/components/feed/core/v2/feed_stream.h new file mode 100644 index 00000000000..5c636822809 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_stream.h @@ -0,0 +1,225 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_FEED_STREAM_H_ +#define COMPONENTS_FEED_CORE_V2_FEED_STREAM_H_ + +#include <memory> +#include <vector> + +#include "base/memory/scoped_refptr.h" +#include "base/observer_list.h" +#include "base/sequenced_task_runner.h" +#include "base/task_runner_util.h" +#include "components/feed/core/common/enums.h" +#include "components/feed/core/common/user_classifier.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/request_throttler.h" +#include "components/feed/core/v2/stream_model.h" +#include "components/feed/core/v2/tasks/load_stream_task.h" +#include "components/offline_pages/task/task_queue.h" + +class PrefService; + +namespace base { +class Clock; +class TickClock; +} // namespace base + +namespace feed { +class FeedStore; +class StreamModel; +class FeedNetwork; +class RefreshTaskScheduler; +struct StreamModelUpdateRequest; + +// Implements FeedStreamApi. |FeedStream| additionally exposes functionality +// needed by other classes within the Feed component. +class FeedStream : public FeedStreamApi, + public offline_pages::TaskQueue::Delegate, + public StreamModel::StoreObserver { + public: + class Delegate { + public: + virtual ~Delegate() = default; + // Returns true if Chrome's EULA has been accepted. + virtual bool IsEulaAccepted() = 0; + // Returns true if the device is offline. + virtual bool IsOffline() = 0; + }; + + // An observer of stream events for testing and for tracking metrics. + // Concrete implementation should have no observable effects on the Feed. + class EventObserver { + public: + virtual void OnLoadStream(LoadStreamStatus load_from_store_status, + LoadStreamStatus final_status) = 0; + virtual void OnMaybeTriggerRefresh(TriggerType trigger, + bool clear_all_before_refresh) = 0; + virtual void OnClearAll(base::TimeDelta time_since_last_clear) = 0; + }; + + // Forwards to |feed::TranslateWireResponse()| by default. Can be overridden + // for testing. + class WireResponseTranslator { + public: + WireResponseTranslator() = default; + ~WireResponseTranslator() = default; + virtual std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( + feedwire::Response response, + base::TimeDelta response_time, + base::Time current_time); + }; + + FeedStream(RefreshTaskScheduler* refresh_task_scheduler, + EventObserver* stream_event_observer, + Delegate* delegate, + PrefService* profile_prefs, + FeedNetwork* feed_network, + FeedStore* feed_store, + const base::Clock* clock, + const base::TickClock* tick_clock, + scoped_refptr<base::SequencedTaskRunner> background_task_runner); + ~FeedStream() override; + + FeedStream(const FeedStream&) = delete; + FeedStream& operator=(const FeedStream&) = delete; + + // Initializes scheduling. This should be called at startup. + void InitializeScheduling(); + + // FeedStreamApi. + + void AttachSurface(SurfaceInterface*) override; + void DetachSurface(SurfaceInterface*) override; + void SetArticlesListVisible(bool is_visible) override; + bool IsArticlesListVisible() override; + void ExecuteOperations( + std::vector<feedstore::DataOperation> operations) override; + EphemeralChangeId CreateEphemeralChange( + std::vector<feedstore::DataOperation> operations) override; + bool CommitEphemeralChange(EphemeralChangeId id) override; + bool RejectEphemeralChange(EphemeralChangeId id) override; + + // offline_pages::TaskQueue::Delegate. + void OnTaskQueueIsIdle() override; + + // StreamModel::StoreObserver. + void OnStoreChange(const StreamModel::StoreUpdate& update) override; + + // Event indicators. These functions are called from an external source + // to indicate an event. + + // Called when Chrome's EULA has been accepted. This should happen when + // Delegate::IsEulaAccepted() changes from false to true. + void OnEulaAccepted(); + // Invoked when Chrome is foregrounded. + void OnEnterForeground(); + // The user signed in to Chrome. + void OnSignedIn(); + // The user signed out of Chrome. + void OnSignedOut(); + // The user has deleted their Chrome history. + void OnHistoryDeleted(); + // Chrome's cached data was cleared. + void OnCacheDataCleared(); + // Invoked by RefreshTaskScheduler's scheduled task. + void ExecuteRefreshTask(); + + // State shared for the sake of implementing FeedStream. Typically these + // functions are used by tasks. + + void LoadModel(std::unique_ptr<StreamModel> model); + + FeedNetwork* GetNetwork() { return feed_network_; } + FeedStore* GetStore() { return store_; } + + // Returns the computed UserClass for the active user. + UserClass GetUserClass(); + + // Returns the time of the last content fetch. + base::Time GetLastFetchTime(); + + // Determines if a FeedQuery request can be made. If successful, + // returns |LoadStreamStatus::kNoStatus| and acquires throttler quota. + // Otherwise returns the reason. + LoadStreamStatus ShouldMakeFeedQueryRequest(); + + // Loads |model|. Should be used for testing in place of typical model + // loading from network or storage. + void LoadModelForTesting(std::unique_ptr<StreamModel> model); + offline_pages::TaskQueue* GetTaskQueueForTesting(); + void UnloadModelForTesting() { UnloadModel(); } + + // Returns the model if it is loaded, or null otherwise. + StreamModel* GetModel() { return model_.get(); } + + const base::Clock* GetClock() { return clock_; } + + WireResponseTranslator* GetWireResponseTranslator() const { + return wire_response_translator_; + } + + void SetWireResponseTranslatorForTesting( + WireResponseTranslator* wire_response_translator) { + wire_response_translator_ = wire_response_translator; + } + + void SetIdleCallbackForTesting(base::RepeatingClosure idle_callback); + void SetUserClassifierForTesting( + std::unique_ptr<UserClassifier> user_classifier); + + private: + class SurfaceUpdater; + class ModelStoreChangeMonitor; + void MaybeTriggerRefresh(TriggerType trigger, + bool clear_all_before_refresh = false); + void TriggerStreamLoad(); + void UnloadModel(); + + void LoadStreamTaskComplete(LoadStreamTask::Result result); + + void ClearAll(); + + // Unowned. + + RefreshTaskScheduler* refresh_task_scheduler_; + EventObserver* stream_event_observer_; + Delegate* delegate_; + PrefService* profile_prefs_; + FeedNetwork* feed_network_; + FeedStore* store_; + const base::Clock* clock_; + const base::TickClock* tick_clock_; + WireResponseTranslator* wire_response_translator_; + + scoped_refptr<base::SequencedTaskRunner> background_task_runner_; + + offline_pages::TaskQueue task_queue_; + // Whether the model is being loaded. Used to prevent multiple simultaneous + // attempts to load the model. + bool model_loading_in_progress_ = false; + std::unique_ptr<SurfaceUpdater> surface_updater_; + // The stream model. Null if not yet loaded. + // Internally, this should only be changed by |LoadModel()| and + // |UnloadModel()|. + std::unique_ptr<StreamModel> model_; + + // Set of (unowned) attached surfaces. + base::ObserverList<SurfaceInterface> surfaces_; + + // Mutable state. + std::unique_ptr<UserClassifier> user_classifier_; + RequestThrottler request_throttler_; + base::TimeTicks suppress_refreshes_until_; + + // To allow tests to wait on task queue idle. + base::RepeatingClosure idle_callback_; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_FEED_STREAM_H_ diff --git a/chromium/components/feed/core/v2/feed_stream_unittest.cc b/chromium/components/feed/core/v2/feed_stream_unittest.cc new file mode 100644 index 00000000000..a514fd7cc63 --- /dev/null +++ b/chromium/components/feed/core/v2/feed_stream_unittest.cc @@ -0,0 +1,713 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/feed_stream.h" + +#include <memory> +#include <string> +#include <utility> + +#include "base/optional.h" +#include "base/strings/string_number_conversions.h" +#include "base/test/bind_test_util.h" +#include "base/test/scoped_run_loop_timeout.h" +#include "base/test/simple_test_clock.h" +#include "base/test/simple_test_tick_clock.h" +#include "base/test/task_environment.h" +#include "base/threading/sequenced_task_runner_handle.h" +#include "components/feed/core/common/pref_names.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/ui.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/shared_prefs/pref_names.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/refresh_task_scheduler.h" +#include "components/feed/core/v2/scheduling.h" +#include "components/feed/core/v2/stream_model.h" +#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/tasks/load_stream_from_store_task.h" +#include "components/feed/core/v2/test/stream_builder.h" +#include "components/leveldb_proto/public/proto_database_provider.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/testing_pref_service.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { + +std::unique_ptr<StreamModel> LoadModelFromStore(FeedStore* store) { + LoadStreamFromStoreTask::Result result; + auto complete = [&](LoadStreamFromStoreTask::Result task_result) { + result = std::move(task_result); + }; + LoadStreamFromStoreTask load_task( + store, /*clock=*/nullptr, + UserClass::kActiveSuggestionsConsumer, // Has no effect. + base::BindLambdaForTesting(complete)); + // We want to load the data no matter how stale. + load_task.IgnoreStalenessForTesting(); + + base::RunLoop run_loop; + load_task.Execute(run_loop.QuitClosure()); + run_loop.Run(); + + if (result.status == LoadStreamStatus::kLoadedFromStore) { + auto model = std::make_unique<StreamModel>(); + model->Update(std::move(result.update_request)); + return model; + } + LOG(WARNING) << "LoadModelFromStore failed with " << result.status; + return nullptr; +} + +// Returns the model state string (|StreamModel::DumpStateForTesting()|), +// given a model initialized with |update_request| and having |operations| +// applied. +std::string ModelStateFor( + std::unique_ptr<StreamModelUpdateRequest> update_request, + std::vector<feedstore::DataOperation> operations = {}, + std::vector<feedstore::DataOperation> more_operations = {}) { + StreamModel model; + model.Update(std::move(update_request)); + model.ExecuteOperations(operations); + model.ExecuteOperations(more_operations); + return model.DumpStateForTesting(); +} + +// Returns the model state string (|StreamModel::DumpStateForTesting()|), +// given a model initialized with |store|. +std::string ModelStateFor(FeedStore* store) { + auto model = LoadModelFromStore(store); + if (model) { + return model->DumpStateForTesting(); + } + return "{Failed to load model from store}"; +} + +// This is EXPECT_EQ, but also dumps the string values for ease of reading. +#define EXPECT_STRINGS_EQUAL(WANT, GOT) \ + { \ + std::string want = (WANT), got = (GOT); \ + EXPECT_EQ(want, got) << "Wanted:\n" << (want) << "\nBut got:\n" << (got); \ + } + +class TestSurface : public FeedStream::SurfaceInterface { + public: + // FeedStream::SurfaceInterface. + void StreamUpdate(const feedui::StreamUpdate& stream_update) override { + if (!initial_state) + initial_state = stream_update; + update = stream_update; + ++update_count_; + } + + // Test functions. + + void Clear() { + initial_state = base::nullopt; + update = base::nullopt; + update_count_ = 0; + } + + // Describe what is shown on the surface in a format that can be easily + // asserted against. + std::string Describe() { + if (!initial_state) + return "empty"; + + if (update->updated_slices().size() == 1 && + update->updated_slices()[0].has_slice() && + update->updated_slices()[0].slice().has_zero_state_slice()) { + return "zero-state"; + } + + std::stringstream ss; + ss << update->updated_slices().size() << " slices"; + // If there's more than one update, we want to know that. + if (update_count_ > 1) { + ss << " " << update_count_ << " updates"; + } + return ss.str(); + } + + base::Optional<feedui::StreamUpdate> initial_state; + base::Optional<feedui::StreamUpdate> update; + + private: + int update_count_ = 0; +}; + +class TestUserClassifier : public UserClassifier { + public: + TestUserClassifier(PrefService* pref_service, const base::Clock* clock) + : UserClassifier(pref_service, clock) {} + // UserClassifier. + UserClass GetUserClass() const override { + return overridden_user_class_.value_or(UserClassifier::GetUserClass()); + } + + // Test use. + void OverrideUserClass(UserClass user_class) { + overridden_user_class_ = user_class; + } + + private: + base::Optional<UserClass> overridden_user_class_; +}; + +class TestFeedNetwork : public FeedNetwork { + public: + // FeedNetwork implementation. + void SendQueryRequest( + const feedwire::Request& request, + base::OnceCallback<void(QueryRequestResult)> callback) override { + ++send_query_call_count; + // Emulate a successful response. + // The response body is currently an empty message, because most of the + // time we want to inject a translated response for ease of test-writing. + query_request_sent = request; + QueryRequestResult result; + result.status_code = 200; + result.response_body = std::make_unique<feedwire::Response>(); + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, base::BindOnce(std::move(callback), std::move(result))); + } + void SendActionRequest( + const feedwire::ActionRequest& request, + base::OnceCallback<void(ActionRequestResult)> callback) override { + NOTIMPLEMENTED(); + } + void CancelRequests() override { NOTIMPLEMENTED(); } + + base::Optional<feedwire::Request> query_request_sent; + int send_query_call_count = 0; +}; + +// Forwards to |FeedStream::WireResponseTranslator| unless a response is +// injected. +class TestWireResponseTranslator : public FeedStream::WireResponseTranslator { + public: + std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( + feedwire::Response response, + base::TimeDelta response_time, + base::Time current_time) override { + if (injected_response_) { + return std::move(injected_response_); + } + return FeedStream::WireResponseTranslator::TranslateWireResponse( + std::move(response), response_time, current_time); + } + void InjectResponse(std::unique_ptr<StreamModelUpdateRequest> response) { + injected_response_ = std::move(response); + } + bool InjectedResponseConsumed() const { return !injected_response_; } + + private: + std::unique_ptr<StreamModelUpdateRequest> injected_response_; +}; + +class FakeRefreshTaskScheduler : public RefreshTaskScheduler { + public: + // RefreshTaskScheduler implementation. + void EnsureScheduled(base::TimeDelta period) override { + scheduled_period = period; + } + void Cancel() override { canceled = true; } + void RefreshTaskComplete() override { refresh_task_complete = true; } + + base::Optional<base::TimeDelta> scheduled_period; + bool canceled = false; + bool refresh_task_complete = false; +}; + +class TestEventObserver : public FeedStream::EventObserver { + public: + // FeedStreamUnittest::StreamEventObserver. + void OnLoadStream(LoadStreamStatus load_from_store_status, + LoadStreamStatus final_status) override { + load_stream_status = final_status; + LOG(INFO) << "OnLoadStream: " << final_status + << " (store status: " << load_from_store_status << ")"; + } + void OnMaybeTriggerRefresh(TriggerType trigger, + bool clear_all_before_refresh) override { + refresh_trigger_type = trigger; + } + void OnClearAll(base::TimeDelta time_since_last_clear) override { + this->time_since_last_clear = time_since_last_clear; + } + + // Test access. + + base::Optional<LoadStreamStatus> load_stream_status; + base::Optional<base::TimeDelta> time_since_last_clear; + base::Optional<TriggerType> refresh_trigger_type; +}; + +class FeedStreamTest : public testing::Test, public FeedStream::Delegate { + public: + void SetUp() override { + feed::prefs::RegisterFeedSharedProfilePrefs(profile_prefs_.registry()); + feed::RegisterProfilePrefs(profile_prefs_.registry()); + CHECK_EQ(kTestTimeEpoch, task_environment_.GetMockClock()->Now()); + stream_ = std::make_unique<FeedStream>( + &refresh_scheduler_, &event_observer_, this, &profile_prefs_, &network_, + store_.get(), task_environment_.GetMockClock(), + task_environment_.GetMockTickClock(), + task_environment_.GetMainThreadTaskRunner()); + + // Set the user classifier. + auto user_classifier = std::make_unique<TestUserClassifier>( + &profile_prefs_, task_environment_.GetMockClock()); + user_classifier_ = user_classifier.get(); + stream_->SetUserClassifierForTesting(std::move(user_classifier)); + + WaitForIdleTaskQueue(); // Wait for any initialization. + + stream_->SetWireResponseTranslatorForTesting(&response_translator_); + } + + void TearDown() override { + // Ensure the task queue can return to idle. Failure to do so may be due + // to a stuck task that never called |TaskComplete()|. + WaitForIdleTaskQueue(); + // Store requires PostTask to clean up. + store_.reset(); + task_environment_.RunUntilIdle(); + } + + // FeedStream::Delegate. + bool IsEulaAccepted() override { return is_eula_accepted_; } + bool IsOffline() override { return is_offline_; } + + // For tests. + + bool IsTaskQueueIdle() const { + return !stream_->GetTaskQueueForTesting()->HasPendingTasks() && + !stream_->GetTaskQueueForTesting()->HasRunningTask(); + } + + void WaitForIdleTaskQueue() { + if (IsTaskQueueIdle()) + return; + base::test::ScopedRunLoopTimeout run_timeout( + FROM_HERE, base::TimeDelta::FromSeconds(1)); + base::RunLoop run_loop; + stream_->SetIdleCallbackForTesting(run_loop.QuitClosure()); + run_loop.Run(); + } + + void UnloadModel() { + WaitForIdleTaskQueue(); + stream_->UnloadModelForTesting(); + } + + protected: + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; + TestUserClassifier* user_classifier_; + TestEventObserver event_observer_; + TestingPrefServiceSimple profile_prefs_; + TestFeedNetwork network_; + TestWireResponseTranslator response_translator_; + + std::unique_ptr<FeedStore> store_ = std::make_unique<FeedStore>( + leveldb_proto::ProtoDatabaseProvider::GetUniqueDB<feedstore::Record>( + leveldb_proto::ProtoDbType::FEED_STREAM_DATABASE, + /*file_path=*/{}, + task_environment_.GetMainThreadTaskRunner())); + FakeRefreshTaskScheduler refresh_scheduler_; + std::unique_ptr<FeedStream> stream_; + bool is_eula_accepted_ = true; + bool is_offline_ = false; +}; + +TEST_F(FeedStreamTest, IsArticlesListVisibleByDefault) { + EXPECT_TRUE(stream_->IsArticlesListVisible()); +} + +TEST_F(FeedStreamTest, SetArticlesListVisible) { + EXPECT_TRUE(stream_->IsArticlesListVisible()); + stream_->SetArticlesListVisible(false); + EXPECT_FALSE(stream_->IsArticlesListVisible()); + stream_->SetArticlesListVisible(true); + EXPECT_TRUE(stream_->IsArticlesListVisible()); +} + +TEST_F(FeedStreamTest, RefreshIsScheduledOnInitialize) { + stream_->InitializeScheduling(); + EXPECT_TRUE(refresh_scheduler_.scheduled_period); +} + +TEST_F(FeedStreamTest, ScheduledRefreshTriggersRefresh) { + stream_->InitializeScheduling(); + stream_->ExecuteRefreshTask(); + + EXPECT_EQ(TriggerType::kFixedTimer, event_observer_.refresh_trigger_type); + // TODO(harringtond): Once we actually perform the refresh, make sure + // RefreshTaskComplete() is called. + // EXPECT_TRUE(refresh_scheduler_.refresh_task_complete); +} + +TEST_F(FeedStreamTest, DoNotRefreshIfArticlesListIsHidden) { + stream_->SetArticlesListVisible(false); + stream_->InitializeScheduling(); + stream_->ExecuteRefreshTask(); + + EXPECT_TRUE(refresh_scheduler_.canceled); + EXPECT_FALSE(event_observer_.refresh_trigger_type); +} + +TEST_F(FeedStreamTest, SurfaceReceivesInitialContent) { + { + auto model = std::make_unique<StreamModel>(); + model->Update(MakeTypicalInitialModelState()); + stream_->LoadModelForTesting(std::move(model)); + } + TestSurface surface; + stream_->AttachSurface(&surface); + ASSERT_TRUE(surface.initial_state); + const feedui::StreamUpdate& initial_state = surface.initial_state.value(); + ASSERT_EQ(2, initial_state.updated_slices().size()); + EXPECT_NE("", initial_state.updated_slices(0).slice().slice_id()); + EXPECT_EQ("f:0", initial_state.updated_slices(0) + .slice() + .xsurface_slice() + .xsurface_frame()); + EXPECT_NE("", initial_state.updated_slices(1).slice().slice_id()); + EXPECT_EQ("f:1", initial_state.updated_slices(1) + .slice() + .xsurface_slice() + .xsurface_frame()); + ASSERT_EQ(1, initial_state.new_shared_states().size()); + EXPECT_EQ("ss:0", + initial_state.new_shared_states()[0].xsurface_shared_state()); +} + +TEST_F(FeedStreamTest, SurfaceReceivesInitialContentLoadedAfterAttach) { + TestSurface surface; + stream_->AttachSurface(&surface); + ASSERT_FALSE(surface.initial_state); + { + auto model = std::make_unique<StreamModel>(); + model->Update(MakeTypicalInitialModelState()); + stream_->LoadModelForTesting(std::move(model)); + } + + ASSERT_EQ("2 slices", surface.Describe()); + const feedui::StreamUpdate& initial_state = surface.initial_state.value(); + + EXPECT_NE("", initial_state.updated_slices(0).slice().slice_id()); + EXPECT_EQ("f:0", initial_state.updated_slices(0) + .slice() + .xsurface_slice() + .xsurface_frame()); + EXPECT_NE("", initial_state.updated_slices(1).slice().slice_id()); + EXPECT_EQ("f:1", initial_state.updated_slices(1) + .slice() + .xsurface_slice() + .xsurface_frame()); + ASSERT_EQ(1, initial_state.new_shared_states().size()); + EXPECT_EQ("ss:0", + initial_state.new_shared_states()[0].xsurface_shared_state()); +} + +TEST_F(FeedStreamTest, SurfaceReceivesUpdatedContent) { + { + auto model = std::make_unique<StreamModel>(); + model->ExecuteOperations(MakeTypicalStreamOperations()); + stream_->LoadModelForTesting(std::move(model)); + } + TestSurface surface; + stream_->AttachSurface(&surface); + // Remove #1, add #2. + stream_->ExecuteOperations({ + MakeOperation(MakeRemove(MakeClusterId(1))), + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }); + ASSERT_TRUE(surface.update); + const feedui::StreamUpdate& initial_state = surface.initial_state.value(); + const feedui::StreamUpdate& update = surface.update.value(); + + ASSERT_EQ("2 slices 2 updates", surface.Describe()); + // First slice is just an ID that matches the old 1st slice ID. + EXPECT_EQ(initial_state.updated_slices(0).slice().slice_id(), + update.updated_slices(0).slice_id()); + // Second slice is a new xsurface slice. + EXPECT_NE("", update.updated_slices(1).slice().slice_id()); + EXPECT_EQ("f:2", + update.updated_slices(1).slice().xsurface_slice().xsurface_frame()); +} + +TEST_F(FeedStreamTest, SurfaceReceivesSecondUpdatedContent) { + { + auto model = std::make_unique<StreamModel>(); + model->ExecuteOperations(MakeTypicalStreamOperations()); + stream_->LoadModelForTesting(std::move(model)); + } + TestSurface surface; + stream_->AttachSurface(&surface); + // Add #2. + stream_->ExecuteOperations({ + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }); + + // Clear the last update and add #3. + stream_->ExecuteOperations({ + MakeOperation(MakeCluster(3, MakeRootId())), + MakeOperation(MakeContentNode(3, MakeClusterId(3))), + MakeOperation(MakeContent(3)), + }); + + // The last update should have only one new piece of content. + // This verifies the current content set is tracked properly. + ASSERT_EQ("4 slices 3 updates", surface.Describe()); + + ASSERT_EQ(4, surface.update->updated_slices().size()); + EXPECT_FALSE(surface.update->updated_slices(0).has_slice()); + EXPECT_FALSE(surface.update->updated_slices(1).has_slice()); + EXPECT_FALSE(surface.update->updated_slices(2).has_slice()); + EXPECT_EQ("f:3", surface.update->updated_slices(3) + .slice() + .xsurface_slice() + .xsurface_frame()); +} + +TEST_F(FeedStreamTest, DetachSurface) { + { + auto model = std::make_unique<StreamModel>(); + model->ExecuteOperations(MakeTypicalStreamOperations()); + stream_->LoadModelForTesting(std::move(model)); + } + TestSurface surface; + stream_->AttachSurface(&surface); + EXPECT_TRUE(surface.initial_state); + stream_->DetachSurface(&surface); + surface.Clear(); + + // Arbitrary stream change. Surface should not see the update. + stream_->ExecuteOperations({ + MakeOperation(MakeRemove(MakeClusterId(1))), + }); + EXPECT_FALSE(surface.update); +} + +TEST_F(FeedStreamTest, LoadFromNetwork) { + // Store is empty, so we should fallback to a network request. + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_TRUE(network_.query_request_sent); + EXPECT_TRUE(response_translator_.InjectedResponseConsumed()); + EXPECT_EQ("2 slices", surface.Describe()); + // Verify the model is filled correctly. + EXPECT_STRINGS_EQUAL(ModelStateFor(MakeTypicalInitialModelState()), + stream_->GetModel()->DumpStateForTesting()); + // Verify the data was written to the store. + EXPECT_STRINGS_EQUAL(ModelStateFor(store_.get()), + ModelStateFor(MakeTypicalInitialModelState())); +} + +TEST_F(FeedStreamTest, LoadFromNetworkBecauseStoreIsStale) { + // Fill the store with stream data that is just barely stale, and verify we + // fetch new data over the network. + user_classifier_->OverrideUserClass(UserClass::kActiveSuggestionsConsumer); + store_->SaveFullStream(MakeTypicalInitialModelState( + + kTestTimeEpoch - base::TimeDelta::FromHours(12) - + base::TimeDelta::FromMinutes(1)), + base::DoNothing()); + + // Store is stale, so we should fallback to a network request. + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_TRUE(network_.query_request_sent); + EXPECT_TRUE(response_translator_.InjectedResponseConsumed()); + ASSERT_TRUE(surface.initial_state); +} + +TEST_F(FeedStreamTest, LoadFromNetworkFailsDueToProtoTranslation) { + // No data in the store, so we should fetch from the network. + // The network will respond with an empty response, which should fail proto + // translation. + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ(LoadStreamStatus::kProtoTranslationFailed, + event_observer_.load_stream_status); +} + +TEST_F(FeedStreamTest, DoNotLoadFromNetworkWhenOffline) { + is_offline_ = true; + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ(LoadStreamStatus::kCannotLoadFromNetworkOffline, + event_observer_.load_stream_status); + EXPECT_EQ("zero-state", surface.Describe()); +} + +TEST_F(FeedStreamTest, DoNotLoadStreamWhenArticleListIsHidden) { + stream_->SetArticlesListVisible(false); + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ(LoadStreamStatus::kLoadNotAllowedArticlesListHidden, + event_observer_.load_stream_status); + EXPECT_EQ("zero-state", surface.Describe()); +} + +TEST_F(FeedStreamTest, DoNotLoadStreamWhenEulaIsNotAccepted) { + is_eula_accepted_ = false; + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ(LoadStreamStatus::kLoadNotAllowedEulaNotAccepted, + event_observer_.load_stream_status); + EXPECT_EQ("zero-state", surface.Describe()); +} + +TEST_F(FeedStreamTest, DoNotLoadFromNetworkAfterHistoryIsDeleted) { + stream_->OnHistoryDeleted(); + task_environment_.FastForwardBy(kSuppressRefreshDuration - + base::TimeDelta::FromSeconds(1)); + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ("zero-state", surface.Describe()); + + EXPECT_EQ(LoadStreamStatus::kCannotLoadFromNetworkSupressedForHistoryDelete, + event_observer_.load_stream_status); + + stream_->DetachSurface(&surface); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(2)); + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ("2 slices 2 updates", surface.Describe()); +} + +TEST_F(FeedStreamTest, ShouldMakeFeedQueryRequestConsumesQuota) { + LoadStreamStatus status = LoadStreamStatus::kNoStatus; + for (; status == LoadStreamStatus::kNoStatus; + status = stream_->ShouldMakeFeedQueryRequest()) { + } + + ASSERT_EQ(LoadStreamStatus::kCannotLoadFromNetworkThrottled, status); +} + +TEST_F(FeedStreamTest, LoadStreamFromStore) { + // Fill the store with stream data that is just barely fresh, and verify it + // loads. + user_classifier_->OverrideUserClass(UserClass::kActiveSuggestionsConsumer); + store_->SaveFullStream(MakeTypicalInitialModelState( + kTestTimeEpoch - base::TimeDelta::FromHours(12) + + base::TimeDelta::FromMinutes(1)), + base::DoNothing()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + + ASSERT_EQ("2 slices", surface.Describe()); + EXPECT_FALSE(network_.query_request_sent); + // Verify the model is filled correctly. + EXPECT_STRINGS_EQUAL(ModelStateFor(MakeTypicalInitialModelState()), + stream_->GetModel()->DumpStateForTesting()); +} + +TEST_F(FeedStreamTest, DetachSurfaceWhileLoadingModel) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + stream_->AttachSurface(&surface); + stream_->DetachSurface(&surface); + WaitForIdleTaskQueue(); + + EXPECT_EQ("empty", surface.Describe()); + EXPECT_TRUE(network_.query_request_sent); +} + +TEST_F(FeedStreamTest, AttachMultipleSurfacesLoadsModelOnce) { + response_translator_.InjectResponse(MakeTypicalInitialModelState()); + TestSurface surface; + TestSurface other_surface; + stream_->AttachSurface(&surface); + stream_->AttachSurface(&other_surface); + WaitForIdleTaskQueue(); + + ASSERT_EQ(1, network_.send_query_call_count); + + // After load, another surface doesn't trigger any tasks. + TestSurface later_surface; + stream_->AttachSurface(&later_surface); + + EXPECT_TRUE(IsTaskQueueIdle()); +} + +TEST_F(FeedStreamTest, ModelChangesAreSavedToStorage) { + store_->SaveFullStream(MakeTypicalInitialModelState(), base::DoNothing()); + TestSurface surface; + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + ASSERT_TRUE(surface.initial_state); + + // Remove #1, add #2. + const std::vector<feedstore::DataOperation> operations = { + MakeOperation(MakeRemove(MakeClusterId(1))), + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }; + stream_->ExecuteOperations(operations); + + WaitForIdleTaskQueue(); + + // Verify changes are applied to storage. + EXPECT_STRINGS_EQUAL( + ModelStateFor(MakeTypicalInitialModelState(), operations), + ModelStateFor(store_.get())); + + // Unload and reload the model from the store, and verify we can still apply + // operations correctly. + stream_->DetachSurface(&surface); + surface.Clear(); + UnloadModel(); + stream_->AttachSurface(&surface); + WaitForIdleTaskQueue(); + ASSERT_TRUE(surface.initial_state); + + // Remove #2, add #3. + const std::vector<feedstore::DataOperation> operations2 = { + MakeOperation(MakeRemove(MakeClusterId(2))), + MakeOperation(MakeCluster(3, MakeRootId())), + MakeOperation(MakeContentNode(3, MakeClusterId(3))), + MakeOperation(MakeContent(3)), + }; + stream_->ExecuteOperations(operations2); + + WaitForIdleTaskQueue(); + EXPECT_STRINGS_EQUAL( + ModelStateFor(MakeTypicalInitialModelState(), operations, operations2), + ModelStateFor(store_.get())); +} + +} // namespace +} // namespace feed diff --git a/chromium/components/feed/core/v2/prefs.cc b/chromium/components/feed/core/v2/prefs.cc new file mode 100644 index 00000000000..d3d5a0b39f2 --- /dev/null +++ b/chromium/components/feed/core/v2/prefs.cc @@ -0,0 +1,56 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/prefs.h" + +#include <utility> + +#include "base/value_conversions.h" +#include "base/values.h" +#include "components/prefs/pref_registry_simple.h" +#include "components/prefs/pref_service.h" + +namespace feed { +namespace { +const char kThrottlerRequestCountListPrefName[] = + "feedv2.request_throttler.request_counts"; +const char kThrottlerLastRequestTime[] = + "feedv2.request_throttler.last_request_time"; + +} // namespace + +namespace prefs { + +std::vector<int> GetThrottlerRequestCounts(PrefService* pref_service) { + std::vector<int> result; + const auto& value_list = + pref_service->GetList(kThrottlerRequestCountListPrefName)->GetList(); + for (const base::Value& value : value_list) { + result.push_back(value.is_int() ? value.GetInt() : 0); + } + return result; +} + +void SetThrottlerRequestCounts(std::vector<int> request_counts, + PrefService* pref_service) { + std::vector<base::Value> value_list; + for (int count : request_counts) { + value_list.push_back(base::Value(count)); + } + + pref_service->Set(kThrottlerRequestCountListPrefName, + base::Value(std::move(value_list))); +} + +base::Time GetLastRequestTime(PrefService* pref_service) { + return pref_service->GetTime(kThrottlerLastRequestTime); +} + +void SetLastRequestTime(base::Time request_time, PrefService* pref_service) { + return pref_service->SetTime(kThrottlerLastRequestTime, request_time); +} + +} // namespace prefs + +} // namespace feed diff --git a/chromium/components/feed/core/v2/prefs.h b/chromium/components/feed/core/v2/prefs.h new file mode 100644 index 00000000000..29da3c9c6cf --- /dev/null +++ b/chromium/components/feed/core/v2/prefs.h @@ -0,0 +1,33 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_PREFS_H_ +#define COMPONENTS_FEED_CORE_V2_PREFS_H_ + +#include <vector> + +#include "base/time/time.h" + +class PrefService; + +namespace feed { +namespace prefs { + +// Functions for accessing prefs. + +// For counting previously made requests, one integer for each +// |NetworkRequestType|. +std::vector<int> GetThrottlerRequestCounts(PrefService* pref_service); +void SetThrottlerRequestCounts(std::vector<int> request_counts, + PrefService* pref_service); + +// Time of the last request. For determining whether the next day's quota should +// be released. +base::Time GetLastRequestTime(PrefService* pref_service); +void SetLastRequestTime(base::Time request_time, PrefService* pref_service); + +} // namespace prefs +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_PREFS_H_ diff --git a/chromium/components/feed/core/v2/proto_util.cc b/chromium/components/feed/core/v2/proto_util.cc new file mode 100644 index 00000000000..90029710307 --- /dev/null +++ b/chromium/components/feed/core/v2/proto_util.cc @@ -0,0 +1,49 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/proto_util.h" + +#include <tuple> + +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "components/feed/core/proto/v2/store.pb.h" + +namespace feed { + +std::string ContentIdString(const feedwire::ContentId& content_id) { + return base::StrCat({content_id.content_domain(), ",", + base::NumberToString(content_id.type()), ",", + base::NumberToString(content_id.id())}); +} + +bool Equal(const feedwire::ContentId& a, const feedwire::ContentId& b) { + return a.content_domain() == b.content_domain() && a.id() == b.id() && + a.type() == b.type(); +} + +bool CompareContentId(const feedwire::ContentId& a, + const feedwire::ContentId& b) { + // Local variables because tie() needs l-values. + const int a_id = a.id(); + const int b_id = b.id(); + const feedwire::ContentId::Type a_type = a.type(); + const feedwire::ContentId::Type b_type = b.type(); + return std::tie(a.content_domain(), a_id, a_type) < + std::tie(b.content_domain(), b_id, b_type); +} + +} // namespace feed + +namespace feedstore { +void SetLastAddedTime(base::Time t, feedstore::StreamData* data) { + data->set_last_added_time_millis( + (t - base::Time::UnixEpoch()).InMilliseconds()); +} + +base::Time GetLastAddedTime(const feedstore::StreamData& data) { + return base::Time::UnixEpoch() + + base::TimeDelta::FromMilliseconds(data.last_added_time_millis()); +} +} // namespace feedstore diff --git a/chromium/components/feed/core/v2/proto_util.h b/chromium/components/feed/core/v2/proto_util.h new file mode 100644 index 00000000000..c7700625eb7 --- /dev/null +++ b/chromium/components/feed/core/v2/proto_util.h @@ -0,0 +1,44 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_PROTO_UTIL_H_ +#define COMPONENTS_FEED_CORE_V2_PROTO_UTIL_H_ + +#include <string> + +#include "base/time/time.h" + +#include "components/feed/core/proto/v2/wire/content_id.pb.h" + +namespace feedstore { +class StreamData; +} + +// Helper functions/classes for dealing with feed proto messages. + +namespace feed { + +std::string ContentIdString(const feedwire::ContentId&); +bool Equal(const feedwire::ContentId& a, const feedwire::ContentId& b); +bool CompareContentId(const feedwire::ContentId& a, + const feedwire::ContentId& b); + +class ContentIdCompareFunctor { + public: + bool operator()(const feedwire::ContentId& a, + const feedwire::ContentId& b) const { + return CompareContentId(a, b); + } +}; + +} // namespace feed + +namespace feedstore { + +void SetLastAddedTime(base::Time t, feedstore::StreamData* data); +base::Time GetLastAddedTime(const feedstore::StreamData& data); + +} // namespace feedstore + +#endif // COMPONENTS_FEED_CORE_V2_PROTO_UTIL_H_ diff --git a/chromium/components/feed/core/v2/public/feed_service.cc b/chromium/components/feed/core/v2/public/feed_service.cc new file mode 100644 index 00000000000..d1fb36f1fd9 --- /dev/null +++ b/chromium/components/feed/core/v2/public/feed_service.cc @@ -0,0 +1,113 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/public/feed_service.h" + +#include <utility> + +#include "base/time/default_clock.h" +#include "base/time/default_tick_clock.h" +#include "components/feed/core/v2/feed_network_impl.h" +#include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/refresh_task_scheduler.h" +#include "net/base/network_change_notifier.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" + +namespace feed { +namespace { + +class EulaObserver : public web_resource::EulaAcceptedNotifier::Observer { + public: + explicit EulaObserver(FeedStream* feed_stream) : feed_stream_(feed_stream) {} + EulaObserver(EulaObserver&) = delete; + EulaObserver& operator=(const EulaObserver&) = delete; + + // web_resource::EulaAcceptedNotifier::Observer. + void OnEulaAccepted() override { feed_stream_->OnEulaAccepted(); } + + private: + FeedStream* feed_stream_; +}; + +} // namespace + +class FeedService::NetworkDelegateImpl : public FeedNetworkImpl::Delegate { + public: + explicit NetworkDelegateImpl(FeedService::Delegate* service_delegate) + : service_delegate_(service_delegate) {} + NetworkDelegateImpl(const NetworkDelegateImpl&) = delete; + NetworkDelegateImpl& operator=(const NetworkDelegateImpl&) = delete; + + // FeedNetworkImpl::Delegate. + std::string GetLanguageTag() override { + return service_delegate_->GetLanguageTag(); + } + + private: + FeedService::Delegate* service_delegate_; +}; + +class FeedService::StreamDelegateImpl : public FeedStream::Delegate { + public: + explicit StreamDelegateImpl(PrefService* local_state) + : eula_notifier_(local_state) {} + StreamDelegateImpl(const StreamDelegateImpl&) = delete; + StreamDelegateImpl& operator=(const StreamDelegateImpl&) = delete; + + void Initialize(FeedStream* feed_stream) { + eula_observer_ = std::make_unique<EulaObserver>(feed_stream); + eula_notifier_.Init(eula_observer_.get()); + } + + // FeedStream::Delegate. + bool IsEulaAccepted() override { return eula_notifier_.IsEulaAccepted(); } + bool IsOffline() override { return net::NetworkChangeNotifier::IsOffline(); } + + private: + web_resource::EulaAcceptedNotifier eula_notifier_; + std::unique_ptr<EulaObserver> eula_observer_; +}; + +FeedService::FeedService(std::unique_ptr<FeedStreamApi> stream) + : stream_(std::move(stream)) {} + +FeedService::FeedService( + std::unique_ptr<Delegate> delegate, + std::unique_ptr<RefreshTaskScheduler> refresh_task_scheduler, + PrefService* profile_prefs, + PrefService* local_state, + std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> database, + signin::IdentityManager* identity_manager, + scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory, + scoped_refptr<base::SequencedTaskRunner> background_task_runner, + const std::string& api_key) + : delegate_(std::move(delegate)), + refresh_task_scheduler_(std::move(refresh_task_scheduler)) { + stream_delegate_ = std::make_unique<StreamDelegateImpl>(local_state); + network_delegate_ = std::make_unique<NetworkDelegateImpl>(delegate_.get()); + feed_network_ = std::make_unique<FeedNetworkImpl>( + network_delegate_.get(), identity_manager, api_key, url_loader_factory, + base::DefaultTickClock::GetInstance(), profile_prefs); + store_ = std::make_unique<FeedStore>(std::move(database)); + + stream_ = std::make_unique<FeedStream>( + refresh_task_scheduler_.get(), + nullptr, // TODO(harringtond): Implement EventObserver. + stream_delegate_.get(), profile_prefs, feed_network_.get(), store_.get(), + base::DefaultClock::GetInstance(), base::DefaultTickClock::GetInstance(), + background_task_runner); + + stream_delegate_->Initialize(static_cast<FeedStream*>(stream_.get())); + + // TODO(harringtond): Call FeedStream::OnSignedIn() + // TODO(harringtond): Call FeedStream::OnSignedOut() + // TODO(harringtond): Call FeedStream::OnHistoryDeleted() + // TODO(harringtond): Call FeedStream::OnCacheDataCleared() + // TODO(harringtond): Call FeedStream::OnEnterForeground() +} + +FeedService::~FeedService() = default; + +} // namespace feed diff --git a/chromium/components/feed/core/v2/public/feed_service.h b/chromium/components/feed/core/v2/public/feed_service.h new file mode 100644 index 00000000000..2d24ad0d0f6 --- /dev/null +++ b/chromium/components/feed/core/v2/public/feed_service.h @@ -0,0 +1,85 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_SERVICE_H_ +#define COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_SERVICE_H_ + +#include <memory> +#include <string> + +#include "base/files/file_path.h" +#include "base/memory/scoped_refptr.h" +#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/keyed_service/core/keyed_service.h" +#include "components/leveldb_proto/public/proto_database.h" +#include "components/web_resource/eula_accepted_notifier.h" + +namespace base { +class SequencedTaskRunner; +} +namespace feedstore { +class Record; +} +namespace network { +class SharedURLLoaderFactory; +} +namespace signin { +class IdentityManager; +} + +namespace feed { +class RefreshTaskScheduler; +class FeedNetwork; +class FeedStore; + +class FeedService : public KeyedService { + public: + class Delegate { + public: + virtual ~Delegate() = default; + // Returns a string which represents the top locale and region of the + // device. + virtual std::string GetLanguageTag() = 0; + }; + + // Construct a FeedService given an already constructed FeedStreamApi. + // Used for testing only. + explicit FeedService(std::unique_ptr<FeedStreamApi> stream); + + // Construct a new FeedStreamApi along with FeedService. + FeedService( + std::unique_ptr<Delegate> delegate, + std::unique_ptr<RefreshTaskScheduler> refresh_task_scheduler, + PrefService* profile_prefs, + PrefService* local_state, + std::unique_ptr<leveldb_proto::ProtoDatabase<feedstore::Record>> database, + signin::IdentityManager* identity_manager, + scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory, + scoped_refptr<base::SequencedTaskRunner> background_task_runner, + const std::string& api_key); + ~FeedService() override; + FeedService(const FeedService&) = delete; + FeedService& operator=(const FeedService&) = delete; + + FeedStreamApi* GetStream() { return stream_.get(); } + + private: + class StreamDelegateImpl; + class NetworkDelegateImpl; + + // These components are owned for construction of |FeedStreamApi|. These will + // be null if |FeedStreamApi| is created externally. + std::unique_ptr<Delegate> delegate_; + std::unique_ptr<StreamDelegateImpl> stream_delegate_; + std::unique_ptr<NetworkDelegateImpl> network_delegate_; + std::unique_ptr<FeedNetwork> feed_network_; + std::unique_ptr<FeedStore> store_; + std::unique_ptr<RefreshTaskScheduler> refresh_task_scheduler_; + + std::unique_ptr<FeedStreamApi> stream_; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_SERVICE_H_ diff --git a/chromium/components/feed/core/v2/public/feed_stream_api.h b/chromium/components/feed/core/v2/public/feed_stream_api.h new file mode 100644 index 00000000000..46a025c71fe --- /dev/null +++ b/chromium/components/feed/core/v2/public/feed_stream_api.h @@ -0,0 +1,66 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_STREAM_API_H_ +#define COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_STREAM_API_H_ + +#include <vector> + +#include "base/observer_list_types.h" +#include "base/util/type_safety/id_type.h" +#include "components/feed/core/proto/v2/wire/content_id.pb.h" + +namespace feedui { +class StreamUpdate; +} +namespace feedstore { +class DataOperation; +} + +namespace feed { +using ContentId = feedwire::ContentId; +// Uniquely identifies a revision of a |feedstore::Content|. If Content changes, +// it is assigned a new revision number. +using ContentRevision = util::IdTypeU32<class ContentRevisionClass>; +// A unique ID for an ephemeral change. +using EphemeralChangeId = util::IdTypeU32<class EphemeralChangeIdClass>; + +// This is the public access point for interacting with the Feed stream +// contents. +class FeedStreamApi { + public: + class SurfaceInterface : public base::CheckedObserver { + public: + // Called after registering the observer to provide the full stream state. + // Also called whenever the stream changes. + virtual void StreamUpdate(const feedui::StreamUpdate&) = 0; + }; + + FeedStreamApi() = default; + virtual ~FeedStreamApi() = default; + + virtual void AttachSurface(SurfaceInterface*) = 0; + virtual void DetachSurface(SurfaceInterface*) = 0; + + virtual void SetArticlesListVisible(bool is_visible) = 0; + virtual bool IsArticlesListVisible() = 0; + + // Apply |operations| to the stream model. Does nothing if the model is not + // yet loaded. + virtual void ExecuteOperations( + std::vector<feedstore::DataOperation> operations) = 0; + + // Create a temporary change that may be undone or committed later. Does + // nothing if the model is not yet loaded. + virtual EphemeralChangeId CreateEphemeralChange( + std::vector<feedstore::DataOperation> operations) = 0; + // Commits a change. Returns false if the change does not exist. + virtual bool CommitEphemeralChange(EphemeralChangeId id) = 0; + // Rejects a change. Returns false if the change does not exist. + virtual bool RejectEphemeralChange(EphemeralChangeId id) = 0; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_PUBLIC_FEED_STREAM_API_H_ diff --git a/chromium/components/feed/core/v2/refresh_task_scheduler.h b/chromium/components/feed/core/v2/refresh_task_scheduler.h new file mode 100644 index 00000000000..6f27dd57161 --- /dev/null +++ b/chromium/components/feed/core/v2/refresh_task_scheduler.h @@ -0,0 +1,32 @@ + +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_REFRESH_TASK_SCHEDULER_H_ +#define COMPONENTS_FEED_CORE_V2_REFRESH_TASK_SCHEDULER_H_ + +#include "base/time/time.h" + +namespace feed { + +// Schedules a repeating background task for refreshing the Feed. +// When the scheduled task executes, it calls FeedStream::ExecuteRefreshTask(). +class RefreshTaskScheduler { + public: + RefreshTaskScheduler() = default; + virtual ~RefreshTaskScheduler() = default; + + // Schedules the task if it is not yet scheduled, or if the scheduling + // period changes. + virtual void EnsureScheduled(base::TimeDelta period) = 0; + // Cancel the task if it was previously scheduled. + virtual void Cancel() = 0; + // After FeedStream::ExecuteRefreshTask is called, the callee must call this + // function to indicate the work is complete. + virtual void RefreshTaskComplete() = 0; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_REFRESH_TASK_SCHEDULER_H_ diff --git a/chromium/components/feed/core/v2/request_throttler.cc b/chromium/components/feed/core/v2/request_throttler.cc new file mode 100644 index 00000000000..95d4d8b5edd --- /dev/null +++ b/chromium/components/feed/core/v2/request_throttler.cc @@ -0,0 +1,76 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/request_throttler.h" + +#include <vector> + +#include "base/time/clock.h" +#include "components/feed/core/v2/prefs.h" +#include "components/prefs/pref_service.h" + +namespace feed { +namespace { +int GetMaxRequestsPerDay(NetworkRequestType request_type) { + // TODO(harringtond): Decide what launchable values are. + switch (request_type) { + case NetworkRequestType::kFeedQuery: + return 20; + case NetworkRequestType::kUploadActions: + return 20; + } +} + +int DaysSinceOrigin(const base::Time& time_value) { + // |LocalMidnight()| DCHECKs on some platforms if |time_value| is too small + // (like zero). So if time is before the unix epoch, return 0. + return time_value < base::Time::UnixEpoch() + ? 0 + : time_value.LocalMidnight().since_origin().InDays(); +} + +} // namespace + +RequestThrottler::RequestThrottler(PrefService* pref_service, + const base::Clock* clock) + : pref_service_(pref_service), clock_(clock) { + DCHECK(pref_service); + DCHECK(clock); +} + +bool RequestThrottler::RequestQuota(NetworkRequestType request_type) { + ResetCountersIfDayChanged(); + + const int max_requests_per_day = GetMaxRequestsPerDay(request_type); + + // Fetch request counts from prefs. There's an entry for each request type. + // We may need to resize the list. + std::vector<int> request_counts = + feed::prefs::GetThrottlerRequestCounts(pref_service_); + const size_t request_count_index = static_cast<size_t>(request_type); + if (request_counts.size() <= request_count_index) + request_counts.resize(request_count_index + 1); + + int& requests_already_made = request_counts[request_count_index]; + if (requests_already_made >= max_requests_per_day) + return false; + requests_already_made++; + feed::prefs::SetThrottlerRequestCounts(request_counts, pref_service_); + return true; +} + +void RequestThrottler::ResetCountersIfDayChanged() { + // Grant new quota on local midnight to spread out when clients that start + // making un-throttled requests to server. + const base::Time now = clock_->Now(); + const bool day_changed = + DaysSinceOrigin(feed::prefs::GetLastRequestTime(pref_service_)) != + DaysSinceOrigin(now); + feed::prefs::SetLastRequestTime(now, pref_service_); + + if (day_changed) + feed::prefs::SetThrottlerRequestCounts({}, pref_service_); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/request_throttler.h b/chromium/components/feed/core/v2/request_throttler.h new file mode 100644 index 00000000000..344d600f24f --- /dev/null +++ b/chromium/components/feed/core/v2/request_throttler.h @@ -0,0 +1,41 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_REQUEST_THROTTLER_H_ +#define COMPONENTS_FEED_CORE_V2_REQUEST_THROTTLER_H_ + +#include "components/feed/core/v2/enums.h" + +class PrefService; +namespace base { +class Clock; +} + +namespace feed { + +// Limits number of network requests that can be made each day. +class RequestThrottler { + public: + RequestThrottler(PrefService* pref_service, const base::Clock* clock); + + RequestThrottler(const RequestThrottler&) = delete; + RequestThrottler& operator=(const RequestThrottler&) = delete; + + // Returns whether quota is available for another request, persists the usage + // of said quota, and reports this information to UMA. + bool RequestQuota(NetworkRequestType request_type); + + private: + void ResetCountersIfDayChanged(); + + // Provides durable storage. + PrefService* pref_service_; + + // Used to access current time, injected for testing. + const base::Clock* clock_; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_REQUEST_THROTTLER_H_ diff --git a/chromium/components/feed/core/v2/request_throttler_unittest.cc b/chromium/components/feed/core/v2/request_throttler_unittest.cc new file mode 100644 index 00000000000..0cf0e98f3b0 --- /dev/null +++ b/chromium/components/feed/core/v2/request_throttler_unittest.cc @@ -0,0 +1,55 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/request_throttler.h" + +#include <memory> + +#include "base/test/simple_test_clock.h" +#include "components/feed/core/common/pref_names.h" +#include "components/prefs/testing_pref_service.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { + +const int kMaximumQueryRequestsPerDay = 20; + +class FeedRequestThrottlerTest : public testing::Test { + public: + FeedRequestThrottlerTest() { + RegisterProfilePrefs(test_prefs_.registry()); + + base::Time now; + EXPECT_TRUE(base::Time::FromString("2018-06-11 12:01AM", &now)); + test_clock_.SetNow(now); + } + + protected: + TestingPrefServiceSimple test_prefs_; + base::SimpleTestClock test_clock_; + RequestThrottler throttler_{&test_prefs_, &test_clock_}; +}; + +TEST_F(FeedRequestThrottlerTest, RequestQuotaAllAtOnce) { + for (int i = 0; i < kMaximumQueryRequestsPerDay; ++i) { + EXPECT_TRUE(throttler_.RequestQuota(NetworkRequestType::kFeedQuery)); + } + EXPECT_FALSE(throttler_.RequestQuota(NetworkRequestType::kFeedQuery)); +} + +TEST_F(FeedRequestThrottlerTest, QuotaIsPerDay) { + for (int i = 0; i < kMaximumQueryRequestsPerDay; ++i) { + EXPECT_TRUE(throttler_.RequestQuota(NetworkRequestType::kUploadActions)); + } + // Because we started at 12:01AM, we need to advance 24 hours before making + // another successful request. + test_clock_.Advance(base::TimeDelta::FromHours(23)); + EXPECT_FALSE(throttler_.RequestQuota(NetworkRequestType::kUploadActions)); + test_clock_.Advance(base::TimeDelta::FromHours(1)); + EXPECT_TRUE(throttler_.RequestQuota(NetworkRequestType::kUploadActions)); +} + +} // namespace +} // namespace feed diff --git a/chromium/components/feed/core/v2/scheduling.cc b/chromium/components/feed/core/v2/scheduling.cc new file mode 100644 index 00000000000..21511470cf2 --- /dev/null +++ b/chromium/components/feed/core/v2/scheduling.cc @@ -0,0 +1,51 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/scheduling.h" +#include "base/time/time.h" + +namespace feed { + +base::TimeDelta GetUserClassTriggerThreshold(UserClass user_class, + TriggerType trigger) { + switch (user_class) { + case UserClass::kRareSuggestionsViewer: + switch (trigger) { + case TriggerType::kNtpShown: + return base::TimeDelta::FromHours(4); + case TriggerType::kForegrounded: + return base::TimeDelta::FromHours(24); + case TriggerType::kFixedTimer: + return base::TimeDelta::FromHours(96); + } + case UserClass::kActiveSuggestionsViewer: + switch (trigger) { + case TriggerType::kNtpShown: + return base::TimeDelta::FromHours(4); + case TriggerType::kForegrounded: + return base::TimeDelta::FromHours(24); + case TriggerType::kFixedTimer: + return base::TimeDelta::FromHours(48); + } + case UserClass::kActiveSuggestionsConsumer: + switch (trigger) { + case TriggerType::kNtpShown: + return base::TimeDelta::FromHours(1); + case TriggerType::kForegrounded: + return base::TimeDelta::FromHours(12); + case TriggerType::kFixedTimer: + return base::TimeDelta::FromHours(24); + } + } +} + +bool ShouldWaitForNewContent(UserClass user_class, + bool has_content, + base::TimeDelta content_age) { + return !has_content || + content_age > GetUserClassTriggerThreshold(user_class, + TriggerType::kForegrounded); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/scheduling.h b/chromium/components/feed/core/v2/scheduling.h new file mode 100644 index 00000000000..3361fc66c74 --- /dev/null +++ b/chromium/components/feed/core/v2/scheduling.h @@ -0,0 +1,30 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_SCHEDULING_H_ +#define COMPONENTS_FEED_CORE_V2_SCHEDULING_H_ + +#include "base/time/time.h" +#include "components/feed/core/v2/enums.h" + +namespace feed { +constexpr base::TimeDelta kSuppressRefreshDuration = + base::TimeDelta::FromMinutes(30); + +// Returns a duration, T, depending on the UserClass and TriggerType. +// The following should be true: +// - At most one fetch is attempted per T. +// - Content is considered stale if time since last fetch is > T. We'll prefer +// to refresh stale content before showing it. +// - For TriggerType::kFixedTimer, T is the time between scheduled fetches. +base::TimeDelta GetUserClassTriggerThreshold(UserClass user_class, + TriggerType trigger); + +// Returns whether we should wait for new content before showing stream content. +bool ShouldWaitForNewContent(UserClass user_class, + bool has_content, + base::TimeDelta content_age); +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_SCHEDULING_H_ diff --git a/chromium/components/feed/core/v2/stream_event_metrics.cc b/chromium/components/feed/core/v2/stream_event_metrics.cc new file mode 100644 index 00000000000..ffce8fa1198 --- /dev/null +++ b/chromium/components/feed/core/v2/stream_event_metrics.cc @@ -0,0 +1,28 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +#include "components/feed/core/v2/stream_event_metrics.h" + +#include "base/metrics/histogram_macros.h" + +namespace feed { + +void StreamEventMetrics::OnLoadStream(LoadStreamStatus load_from_store_status, + LoadStreamStatus final_status) { + // TODO(harringtond): Add UMA for this, or record it with another histogram. +} + +void StreamEventMetrics::OnMaybeTriggerRefresh(TriggerType trigger, + bool clear_all_before_refresh) { + // TODO(harringtond): Either add UMA for this or remove it. +} + +void StreamEventMetrics::OnClearAll(base::TimeDelta time_since_last_clear) { + UMA_HISTOGRAM_CUSTOM_TIMES( + "ContentSuggestions.Feed.Scheduler.TimeSinceLastFetchOnClear", + time_since_last_clear, base::TimeDelta::FromSeconds(1), + base::TimeDelta::FromDays(7), + /*bucket_count=*/50); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_event_metrics.h b/chromium/components/feed/core/v2/stream_event_metrics.h new file mode 100644 index 00000000000..3940c2f428c --- /dev/null +++ b/chromium/components/feed/core/v2/stream_event_metrics.h @@ -0,0 +1,24 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_STREAM_EVENT_METRICS_H_ +#define COMPONENTS_FEED_CORE_V2_STREAM_EVENT_METRICS_H_ + +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_stream.h" + +namespace feed { + +// Reports UMA metrics for stream events. +class StreamEventMetrics : public FeedStream::EventObserver { + public: + void OnLoadStream(LoadStreamStatus load_from_store_status, + LoadStreamStatus final_status) override; + void OnMaybeTriggerRefresh(TriggerType trigger, + bool clear_all_before_refresh) override; + void OnClearAll(base::TimeDelta time_since_last_clear) override; +}; +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_STREAM_EVENT_METRICS_H_ diff --git a/chromium/components/feed/core/v2/stream_model.cc b/chromium/components/feed/core/v2/stream_model.cc new file mode 100644 index 00000000000..74381e4516e --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model.cc @@ -0,0 +1,229 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/stream_model.h" + +#include <algorithm> +#include <utility> + +#include "base/logging.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/content_id.pb.h" +#include "components/feed/core/v2/stream_model_update_request.h" + +namespace feed { +namespace { +bool HasClearAll(const std::vector<feedstore::StreamStructure>& structures) { + for (const feedstore::StreamStructure& data : structures) { + if (data.operation() == feedstore::StreamStructure::CLEAR_ALL) + return true; + } + return false; +} +} // namespace +StreamModel::UiUpdate::UiUpdate() = default; +StreamModel::UiUpdate::~UiUpdate() = default; +StreamModel::UiUpdate::UiUpdate(const UiUpdate&) = default; +StreamModel::UiUpdate& StreamModel::UiUpdate::operator=(const UiUpdate&) = + default; +StreamModel::StoreUpdate::StoreUpdate() = default; +StreamModel::StoreUpdate::~StoreUpdate() = default; +StreamModel::StoreUpdate::StoreUpdate(const StoreUpdate&) = default; +StreamModel::StoreUpdate& StreamModel::StoreUpdate::operator=( + const StoreUpdate&) = default; +StreamModel::StoreUpdate::StoreUpdate(StoreUpdate&&) = default; +StreamModel::StoreUpdate& StreamModel::StoreUpdate::operator=(StoreUpdate&&) = + default; + +StreamModel::StreamModel() = default; +StreamModel::~StreamModel() = default; + +void StreamModel::SetStoreObserver(StoreObserver* store_observer) { + DCHECK(!store_observer || !store_observer_) + << "Attempting to set store_observer multiple times"; + store_observer_ = store_observer; +} + +void StreamModel::SetObserver(Observer* observer) { + DCHECK(!observer || !observer_) + << "Attempting to set the observer multiple times"; + observer_ = observer; +} + +const feedstore::Content* StreamModel::FindContent( + ContentRevision revision) const { + return GetFinalFeatureTree()->FindContent(revision); +} +const std::string* StreamModel::FindSharedStateData( + const std::string& id) const { + auto iter = shared_states_.find(id); + if (iter != shared_states_.end()) { + return &iter->second.data; + } + return nullptr; +} + +std::vector<std::string> StreamModel::GetSharedStateIds() const { + std::vector<std::string> ids; + for (auto& entry : shared_states_) { + ids.push_back(entry.first); + } + return ids; +} + +void StreamModel::Update( + std::unique_ptr<StreamModelUpdateRequest> update_request) { + feedstore::StreamData& stream_data = update_request->stream_data; + std::vector<feedstore::StreamStructure>& stream_structures = + update_request->stream_structures; + if (HasClearAll(stream_structures)) { + shared_states_.clear(); + } + + // Update the feature tree. + for (const feedstore::StreamStructure& structure : stream_structures) { + base_feature_tree_.ApplyStreamStructure(structure); + } + for (feedstore::Content& content : update_request->content) { + base_feature_tree_.AddContent(std::move(content)); + } + + // Update non-tree data. + next_page_token_ = stream_data.next_page_token(); + last_added_time_ = + base::Time::UnixEpoch() + + base::TimeDelta::FromMilliseconds(stream_data.last_added_time_millis()); + consistency_token_ = stream_data.consistency_token(); + + for (feedstore::StreamSharedState& shared_state : + update_request->shared_states) { + std::string id = ContentIdString(shared_state.content_id()); + if (!shared_states_.contains(id)) { + shared_states_[id].data = + std::move(*shared_state.mutable_shared_state_data()); + } + } + + // Set next_structure_sequence_number_ when doing the initial load. + if (update_request->source == + StreamModelUpdateRequest::Source::kInitialLoadFromStore) { + next_structure_sequence_number_ = + update_request->max_structure_sequence_number + 1; + } + + // TODO(harringtond): Some StreamData fields not yet used. + // next_action_id - do we need to load the model before uploading + // actions? If not, we probably will want to move this out of + // StreamData. + // content_id - probably just ignore for now + + UpdateFlattenedTree(); +} + +EphemeralChangeId StreamModel::CreateEphemeralChange( + std::vector<feedstore::DataOperation> operations) { + const EphemeralChangeId id = + ephemeral_changes_.AddEphemeralChange(std::move(operations))->id(); + + UpdateFlattenedTree(); + + return id; +} + +void StreamModel::ExecuteOperations( + std::vector<feedstore::DataOperation> operations) { + for (const feedstore::DataOperation& operation : operations) { + if (operation.has_structure()) { + base_feature_tree_.ApplyStreamStructure(operation.structure()); + } + if (operation.has_content()) { + base_feature_tree_.AddContent(operation.content()); + } + } + + if (store_observer_) { + StoreUpdate store_update; + store_update.operations = std::move(operations); + store_update.sequence_number = next_structure_sequence_number_++; + store_observer_->OnStoreChange(std::move(store_update)); + } + + UpdateFlattenedTree(); +} + +bool StreamModel::CommitEphemeralChange(EphemeralChangeId id) { + std::unique_ptr<stream_model::EphemeralChange> change = + ephemeral_changes_.Remove(id); + if (!change) + return false; + + // Note: it's possible that the does change even upon commit because it + // may change the order that operations are applied. ExecuteOperations + // will ensure observers are updated. + ExecuteOperations(change->GetOperations()); + return true; +} + +bool StreamModel::RejectEphemeralChange(EphemeralChangeId id) { + if (ephemeral_changes_.Remove(id)) { + UpdateFlattenedTree(); + return true; + } + return false; +} + +void StreamModel::UpdateFlattenedTree() { + if (ephemeral_changes_.GetChangeList().empty()) { + feature_tree_after_changes_.reset(); + } else { + feature_tree_after_changes_ = + ApplyEphemeralChanges(base_feature_tree_, ephemeral_changes_); + } + // Update list of visible content. + std::vector<ContentRevision> new_state = + GetFinalFeatureTree()->GetVisibleContent(); + const bool content_list_changed = content_list_ != new_state; + content_list_ = std::move(new_state); + + // Pack and send UiUpdate. + UiUpdate update; + update.content_list_changed = content_list_changed; + for (auto& entry : shared_states_) { + SharedState& shared_state = entry.second; + UiUpdate::SharedStateInfo info; + info.shared_state_id = entry.first; + info.updated = shared_state.updated; + update.shared_states.push_back(std::move(info)); + + shared_state.updated = false; + } + + if (observer_) + observer_->OnUiUpdate(update); +} + +stream_model::FeatureTree* StreamModel::GetFinalFeatureTree() { + return feature_tree_after_changes_ ? feature_tree_after_changes_.get() + : &base_feature_tree_; +} +const stream_model::FeatureTree* StreamModel::GetFinalFeatureTree() const { + return const_cast<StreamModel*>(this)->GetFinalFeatureTree(); +} + +std::string StreamModel::DumpStateForTesting() { + std::stringstream ss; + ss << "StreamModel{\n"; + ss << "next_page_token='" << next_page_token_ << "'\n"; + ss << "consistency_token='" << consistency_token_ << "'\n"; + for (auto& entry : shared_states_) { + ss << "shared_state[" << entry.first << "]='" << entry.second.data << "'\n"; + } + ss << GetFinalFeatureTree()->DumpStateForTesting(); + ss << "}StreamModel\n"; + return ss.str(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model.h b/chromium/components/feed/core/v2/stream_model.h new file mode 100644 index 00000000000..2595ba228d3 --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model.h @@ -0,0 +1,152 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_STREAM_MODEL_H_ +#define COMPONENTS_FEED_CORE_V2_STREAM_MODEL_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "base/containers/flat_map.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/content_id.pb.h" +#include "components/feed/core/v2/proto_util.h" +#include "components/feed/core/v2/stream_model/ephemeral_change.h" +#include "components/feed/core/v2/stream_model/feature_tree.h" + +namespace feedwire { +class DataOperation; +} // namespace feedwire + +namespace feed { +struct StreamModelUpdateRequest; + +// An in-memory stream model. +class StreamModel { + public: + // Information about an update to the model. + struct UiUpdate { + struct SharedStateInfo { + // The shared state's unique ID. + std::string shared_state_id; + // Whether the shared state was just modified or added. + bool updated = false; + }; + UiUpdate(); + ~UiUpdate(); + UiUpdate(const UiUpdate&); + UiUpdate& operator=(const UiUpdate&); + // Whether the list of content has changed. Use + // |StreamModel::GetContentList()| to get the updated list of content. + bool content_list_changed = false; + // The list of shared states in the model. + std::vector<SharedStateInfo> shared_states; + }; + + struct StoreUpdate { + StoreUpdate(); + ~StoreUpdate(); + StoreUpdate(const StoreUpdate&); + StoreUpdate(StoreUpdate&&); + StoreUpdate& operator=(const StoreUpdate&); + StoreUpdate& operator=(StoreUpdate&&); + + int32_t sequence_number = 0; + std::vector<feedstore::DataOperation> operations; + }; + + class Observer { + public: + virtual ~Observer() = default; + // Called when the UI model changes. + virtual void OnUiUpdate(const UiUpdate& update) = 0; + }; + class StoreObserver { + public: + // Called when the peristent store should be modified to reflect a model + // change. + virtual void OnStoreChange(const StoreUpdate& update) = 0; + }; + + StreamModel(); + ~StreamModel(); + + StreamModel(const StreamModel& src) = delete; + StreamModel& operator=(const StreamModel&) = delete; + + void SetObserver(Observer* observer); + void SetStoreObserver(StoreObserver* store_observer); + + // Data access. + + // Returns the full list of content in the order it should be presented. + const std::vector<ContentRevision>& GetContentList() const { + return content_list_; + } + // Returns a list of all shared state IDs. + std::vector<std::string> GetSharedStateIds() const; + + // Apply an update from the network or storage. + void Update(std::unique_ptr<StreamModelUpdateRequest> update_request); + + // Returns the content identified by |ContentRevision|. + const feedstore::Content* FindContent(ContentRevision revision) const; + + // Returns the shared state data identified by |id|. + const std::string* FindSharedStateData(const std::string& id) const; + + // Apply |operations| to the model. + void ExecuteOperations(std::vector<feedstore::DataOperation> operations); + + // Create a temporary change that may be undone or committed later. + EphemeralChangeId CreateEphemeralChange( + std::vector<feedstore::DataOperation> operations); + // Commits a change. Returns false if the change does not exist. + bool CommitEphemeralChange(EphemeralChangeId id); + // Rejects a change. Returns false if the change does not exist. + bool RejectEphemeralChange(EphemeralChangeId id); + + // Outputs a string representing the model state for debugging or testing. + std::string DumpStateForTesting(); + + private: + struct SharedState { + // Whether the data has been changed since the last call to |OnUiUpdate()|. + bool updated = true; + std::string data; + }; + // The final feature tree after applying any ephemeral changes. + // May link directly to |base_feature_tree_|. + stream_model::FeatureTree* GetFinalFeatureTree(); + const stream_model::FeatureTree* GetFinalFeatureTree() const; + + void UpdateFlattenedTree(); + + Observer* observer_ = nullptr; // Unowned. + StoreObserver* store_observer_ = nullptr; // Unowned. + stream_model::ContentIdMap id_map_; + stream_model::FeatureTree base_feature_tree_{&id_map_}; + // |base_feature_tree_| with |ephemeral_changes_| applied. + // Null if there are no ephemeral changes. + std::unique_ptr<stream_model::FeatureTree> feature_tree_after_changes_; + stream_model::EphemeralChangeList ephemeral_changes_; + + // The following data is associated with the stream, but lives outside of the + // tree. + + std::string next_page_token_; // TODO(harringtond): use this value. + std::string consistency_token_; // TODO(harringtond): use this value. + base::Time last_added_time_; // TODO(harringtond): use this value. + base::flat_map<std::string, SharedState> shared_states_; + int32_t next_structure_sequence_number_ = 0; + + // Current state of the flattened tree. + // Updated after each tree change. + std::vector<ContentRevision> content_list_; +}; + +} // namespace feed +#endif // COMPONENTS_FEED_CORE_V2_STREAM_MODEL_H_ diff --git a/chromium/components/feed/core/v2/stream_model/README.md b/chromium/components/feed/core/v2/stream_model/README.md new file mode 100644 index 00000000000..4f3f75bbe9b --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model/README.md @@ -0,0 +1 @@ +This directory contains implementation details for StreamModel. diff --git a/chromium/components/feed/core/v2/stream_model/ephemeral_change.cc b/chromium/components/feed/core/v2/stream_model/ephemeral_change.cc new file mode 100644 index 00000000000..96e8a65a3ef --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model/ephemeral_change.cc @@ -0,0 +1,64 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/stream_model/ephemeral_change.h" + +namespace feed { +namespace stream_model { + +EphemeralChange::EphemeralChange( + EphemeralChangeId id, + std::vector<feedstore::DataOperation> operations) + : id_(id), operations_(std::move(operations)) {} +EphemeralChange::~EphemeralChange() = default; + +EphemeralChangeList::EphemeralChangeList() = default; +EphemeralChangeList::~EphemeralChangeList() = default; +EphemeralChange* EphemeralChangeList::AddEphemeralChange( + std::vector<feedstore::DataOperation> operations) { + change_list_.push_back(std::make_unique<EphemeralChange>( + id_generator_.GenerateNextId(), operations)); + return change_list_.back().get(); +} +EphemeralChange* EphemeralChangeList::Find(EphemeralChangeId id) { + for (std::unique_ptr<EphemeralChange>& change : change_list_) { + if (change->id() == id) + return change.get(); + } + return nullptr; +} + +std::unique_ptr<FeatureTree> ApplyEphemeralChanges( + const FeatureTree& tree, + const EphemeralChangeList& changes) { + auto tree_with_changes = std::make_unique<FeatureTree>(&tree); + + for (const std::unique_ptr<EphemeralChange>& change : + changes.GetChangeList()) { + for (const feedstore::DataOperation& operation : change->GetOperations()) { + if (operation.has_structure()) { + tree_with_changes->ApplyStreamStructure(operation.structure()); + } + if (operation.has_content()) { + tree_with_changes->AddContent(operation.content()); + } + } + } + return tree_with_changes; +} + +std::unique_ptr<EphemeralChange> EphemeralChangeList::Remove( + EphemeralChangeId id) { + for (size_t i = 0; i < change_list_.size(); ++i) { + if (change_list_[i]->id() == id) { + std::unique_ptr<EphemeralChange> result = std::move(change_list_[i]); + change_list_.erase(change_list_.begin() + i); + return result; + } + } + return nullptr; +} + +} // namespace stream_model +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model/ephemeral_change.h b/chromium/components/feed/core/v2/stream_model/ephemeral_change.h new file mode 100644 index 00000000000..fe293b3cb7b --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model/ephemeral_change.h @@ -0,0 +1,66 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_STREAM_MODEL_EPHEMERAL_CHANGE_H_ +#define COMPONENTS_FEED_CORE_V2_STREAM_MODEL_EPHEMERAL_CHANGE_H_ + +#include <memory> +#include <vector> +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/stream_model/feature_tree.h" + +namespace feed { +namespace stream_model { + +// A sequence of data operations that may be reverted. +class EphemeralChange { + public: + EphemeralChange(EphemeralChangeId id, + std::vector<feedstore::DataOperation> operations); + ~EphemeralChange(); + EphemeralChange(const EphemeralChange&) = delete; + EphemeralChange& operator=(const EphemeralChange&) = delete; + + EphemeralChangeId id() const { return id_; } + const std::vector<feedstore::DataOperation>& GetOperations() const { + return operations_; + } + std::vector<feedstore::DataOperation>& GetOperations() { return operations_; } + + private: + EphemeralChangeId id_; + std::vector<feedstore::DataOperation> operations_; +}; + +// A list of |EphemeralChange| objects. +class EphemeralChangeList { + public: + EphemeralChangeList(); + ~EphemeralChangeList(); + EphemeralChangeList(const EphemeralChangeList&) = delete; + EphemeralChangeList& operator=(const EphemeralChangeList&) = delete; + + const std::vector<std::unique_ptr<EphemeralChange>>& GetChangeList() const { + return change_list_; + } + EphemeralChange* Find(EphemeralChangeId id); + EphemeralChange* AddEphemeralChange( + std::vector<feedstore::DataOperation> operations); + std::unique_ptr<EphemeralChange> Remove(EphemeralChangeId id); + + private: + EphemeralChangeId::Generator id_generator_; + std::vector<std::unique_ptr<EphemeralChange>> change_list_; +}; + +// Return a new |FeatureTree| by applying |changes| to |tree|. +std::unique_ptr<FeatureTree> ApplyEphemeralChanges( + const FeatureTree& tree, + const EphemeralChangeList& changes); + +} // namespace stream_model +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_STREAM_MODEL_EPHEMERAL_CHANGE_H_ diff --git a/chromium/components/feed/core/v2/stream_model/feature_tree.cc b/chromium/components/feed/core/v2/stream_model/feature_tree.cc new file mode 100644 index 00000000000..5c8373a77e3 --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model/feature_tree.cc @@ -0,0 +1,223 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/stream_model/feature_tree.h" + +#include <algorithm> +#include <sstream> + +#include "base/logging.h" + +namespace feed { +namespace stream_model { + +ContentIdMap::ContentIdMap() = default; +ContentIdMap::~ContentIdMap() = default; + +ContentTag ContentIdMap::GetContentTag(const feedwire::ContentId& id) { + auto iter = mapping_.find(id); + if (iter != mapping_.end()) + return iter->second; + ContentTag tag = tag_generator_.GenerateNextId(); + mapping_[id] = tag; + return tag; +} + +ContentRevision ContentIdMap::NextContentRevision() { + return revision_generator_.GenerateNextId(); +} + +StreamNode::StreamNode() = default; +StreamNode::~StreamNode() = default; +StreamNode::StreamNode(const StreamNode&) = default; +StreamNode& StreamNode::operator=(const StreamNode&) = default; + +FeatureTree::FeatureTree(ContentIdMap* id_map) : id_map_(id_map) {} + +FeatureTree::FeatureTree(const FeatureTree* base) + : base_(base), + id_map_(base->id_map_), + computed_root_(base->computed_root_), + root_tag_(base->root_tag_), + nodes_(base->nodes_) {} +FeatureTree::~FeatureTree() = default; + +StreamNode* FeatureTree::GetOrMakeNode(ContentTag id) { + ResizeNodesIfNeeded(id); + return &nodes_[id.value()]; +} + +const StreamNode* FeatureTree::FindNode(ContentTag id) const { + return const_cast<FeatureTree*>(this)->FindNode(id); +} + +StreamNode* FeatureTree::FindNode(ContentTag id) { + if (!id.is_null() && nodes_.size() > id.value()) + return &nodes_[id.value()]; + return nullptr; +} + +const feedstore::Content* FeatureTree::FindContent(ContentRevision id) const { + auto iter = content_.find(id); + if (iter != content_.end()) + return &iter->second; + return base_ ? base_->FindContent(id) : nullptr; +} + +void FeatureTree::ApplyStreamStructure( + const feedstore::StreamStructure& structure) { + switch (structure.operation()) { + case feedstore::StreamStructure::CLEAR_ALL: + nodes_.clear(); + content_.clear(); + computed_root_ = false; + break; + case feedstore::StreamStructure::UPDATE_OR_APPEND: { + const ContentTag child_id = GetContentTag(structure.content_id()); + const bool is_stream = + structure.type() == feedstore::StreamStructure::STREAM; + ContentTag parent_id; + if (structure.has_parent_id()) { + parent_id = GetContentTag(structure.parent_id()); + } + ResizeNodesIfNeeded(std::max(child_id, parent_id)); + StreamNode& child = nodes_[child_id.value()]; + StreamNode* parent = FindNode(parent_id); + + // If a node already has a parent, treat this as an update, not an append + // operation. + child.is_stream = is_stream; + child.tombstoned = false; + if (root_tag_ == child_id) { + computed_root_ = false; + } + + if (parent && !child.has_parent) { + // The child doesn't yet have a parent, but it should. Link to the + // parent now. If the child already has a parent, we will never change + // the parent even if requested by UPDATE_OR_APPEND. + child.has_parent = true; + child.previous_sibling = parent->last_child; + parent->last_child = child_id; + } else if (!parent && is_stream) { + // The node meets the criteria for root. + computed_root_ = true; + root_tag_ = child_id; + } + } break; + case feedstore::StreamStructure::REMOVE: { + // Removal is just unlinking the node from the tree. + // If it's added back again later, it retains its old children. + ContentTag tag = GetContentTag(structure.content_id()); + if (root_tag_ == tag) { + computed_root_ = false; + } + GetOrMakeNode(tag)->tombstoned = true; + } break; + default: + break; + } +} // namespace stream_model + +void FeatureTree::ResizeNodesIfNeeded(ContentTag id) { + if (nodes_.size() <= id.value()) + nodes_.resize(id.value() + 1); +} + +void FeatureTree::AddContent(feedstore::Content content) { + AddContent(id_map_->NextContentRevision(), std::move(content)); +} + +void FeatureTree::AddContent(ContentRevision revision_id, + feedstore::Content content) { + // TODO(harringtond): Consider de-duping content. + // Currently, we copy content for ephemeral changes. Both when the ephemeral + // change is created, and when it is committed. We should consider eliminating + // these copies. + const ContentTag tag = GetContentTag(content.content_id()); + DCHECK(!content_.count(revision_id)); + GetOrMakeNode(tag)->content_revision = revision_id; + content_[revision_id] = std::move(content); +} + +void FeatureTree::ResolveRoot() { + if (computed_root_) { + DCHECK(!FindNode(root_tag_) || FindNode(root_tag_)->is_stream) << root_tag_; + DCHECK(!FindNode(root_tag_) || !FindNode(root_tag_)->tombstoned); + DCHECK(!FindNode(root_tag_) || !FindNode(root_tag_)->has_parent); + return; + } + root_tag_ = ContentTag(); + for (size_t i = 0; i < nodes_.size(); ++i) { + const StreamNode& node = nodes_[i]; + if (node.is_stream && !node.tombstoned && !node.has_parent) { + root_tag_ = ContentTag(i); + } + } + computed_root_ = true; +} + +std::vector<ContentRevision> FeatureTree::GetVisibleContent() { + ResolveRoot(); + std::vector<ContentRevision> result; + std::vector<ContentTag> stack; + + // Node: Cycles are impossible here. The root node is guaranteed to + // not be a child. All other nodes have exactly one parent. + // It is possible for nodes to cycle, like A->B->A, but in this case there can + // be no valid root because all nodes have a parent. + stack.push_back(root_tag_); + while (!stack.empty()) { + const ContentTag tag = stack.back(); + stack.pop_back(); + const StreamNode* node = FindNode(tag); + if (!node || node->tombstoned) + continue; + if (!node->last_child.is_null()) { + for (ContentTag child_id = node->last_child; !child_id.is_null(); + child_id = nodes_[child_id.value()].previous_sibling) { + stack.push_back(child_id); + } + } + if (!node->content_revision.is_null()) { + result.push_back(node->content_revision); + } + } + return result; +} + +std::string FeatureTree::DumpStateForTesting() { + std::stringstream ss; + ss << "FeatureTree{\n"; + ResolveRoot(); + std::vector<std::pair<int, ContentTag>> stack; + + stack.push_back({1, root_tag_}); + while (!stack.empty()) { + const ContentTag tag = stack.back().second; + const int depth = stack.back().first; + stack.pop_back(); + const StreamNode* node = FindNode(tag); + if (!node || node->tombstoned) + continue; + ss << std::string(depth, ' ') << "|-"; + ss << (node->is_stream ? "ROOT" : "node"); + if (!node->last_child.is_null()) { + for (ContentTag child_id = node->last_child; !child_id.is_null(); + child_id = nodes_[child_id.value()].previous_sibling) { + stack.push_back({depth + 1, child_id}); + } + } + if (!node->content_revision.is_null()) { + const feedstore::Content* content = FindContent(node->content_revision); + ss << " content.frame=" << content->frame(); + } + ss << '\n'; + } + ss << "}FeatureTree\n"; + return ss.str(); +} + +} // namespace stream_model +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model/feature_tree.h b/chromium/components/feed/core/v2/stream_model/feature_tree.h new file mode 100644 index 00000000000..2aa79e8bd6c --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model/feature_tree.h @@ -0,0 +1,137 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_STREAM_MODEL_FEATURE_TREE_H_ +#define COMPONENTS_FEED_CORE_V2_STREAM_MODEL_FEATURE_TREE_H_ + +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "base/util/type_safety/id_type.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/v2/proto_util.h" +#include "components/feed/core/v2/public/feed_stream_api.h" + +namespace feed { +namespace stream_model { + +// Uniquely identifies a feedwire::ContentId. Provided by |ContentIdMap|. +using ContentTag = util::IdTypeU32<class ContentTagClass>; +using ContentRevision = feed::ContentRevision; + +// Maps ContentId into ContentTag, and generates ContentRevision IDs. +class ContentIdMap { + public: + ContentIdMap(); + ~ContentIdMap(); + ContentIdMap(const ContentIdMap&) = delete; + ContentIdMap& operator=(const ContentIdMap&) = delete; + + ContentTag GetContentTag(const feedwire::ContentId& id); + ContentRevision NextContentRevision(); + + private: + ContentTag::Generator tag_generator_; + ContentRevision::Generator revision_generator_; + std::map<feedwire::ContentId, ContentTag, ContentIdCompareFunctor> mapping_; +}; + +// A node in FeatureTree. +struct StreamNode { + StreamNode(); + ~StreamNode(); + StreamNode(const StreamNode&); + StreamNode& operator=(const StreamNode&); + // If true, this nodes has been removed and should be ignored. + bool tombstoned = false; + // Whether this is a STREAM node. + bool is_stream = false; + // Whether this node has a parent. + bool has_parent = false; + // If this node has content, this identifies it. + ContentRevision content_revision; + // Child relations are stored in linked-list fashion. + // The ID of the last child, or null. + ContentTag last_child; + // The ID of the sibling before this one. + ContentTag previous_sibling; +}; + +// The feature tree which underlies StreamModel. +// This tree is different that most, the rules are as follows: +// * A node may or may not have a parent, so this is more of a forest than a +// tree. +// * When nodes are removed, their set of children are remembered. If the node +// is added again, it retains its old children. +// * A node can be added multiple times, but subsequent adds will not change +// the node's parent. +// * There is only one 'stream root' acknowledged, even though there can be many +// roots. The stream root is the last root node added of type STREAM. The +// stream root identifies the tree whose nodes are used to compute +// |GetVisibleContent()|. +// * A tree can be constructed with a base tree. This copies features from base, +// but refers to content stored in base by reference. +class FeatureTree { + public: + // Constructor. |id_map| is retained by FeatureTree, and must have a greater + // scope than FeatureTree. + explicit FeatureTree(ContentIdMap* id_map); + // Create a |FeatureTree| which starts as a copy of |base|. + // Copies structure from |base|, and keeps a reference for content access. + explicit FeatureTree(const FeatureTree* base); + ~FeatureTree(); + + FeatureTree(const FeatureTree& src) = delete; + FeatureTree& operator=(const FeatureTree& src) = delete; + + // Mutations. + + void ApplyStreamStructure(const feedstore::StreamStructure& structure); + void AddContent(feedstore::Content content); + void AddContent(ContentRevision revision_id, feedstore::Content content); + + // Data access. + + const StreamNode* FindNode(ContentTag id) const; + StreamNode* FindNode(ContentTag id); + const feedstore::Content* FindContent(ContentRevision id) const; + ContentTag GetContentTag(const feedwire::ContentId& id) { + return id_map_->GetContentTag(id); + } + + // Returns the list of content that should be visible. + std::vector<ContentRevision> GetVisibleContent(); + + std::string DumpStateForTesting(); + + private: + StreamNode* GetOrMakeNode(ContentTag id); + void ResolveRoot(); + void ResizeNodesIfNeeded(ContentTag id); + void RemoveFromParent(ContentTag node_id); + bool RemoveFromParent(StreamNode* parent, ContentTag node_id); + + const FeatureTree* base_ = nullptr; // Unowned. + ContentIdMap* id_map_; // Unowned. + // Finding the root: + // We pick the root node as the last STREAM node which has no parent. + // In most cases, we can identify the root as the tree is built. + // But in some cases, we need to search all nodes to find the root. + // |computed_root_| is true if |root_tag_| is guaranteed to identify the root. + bool computed_root_ = true; + ContentTag root_tag_; + // All nodes in the forest, included removed nodes. + // This datastructure was selected to make copies efficient. + std::vector<StreamNode> nodes_; + // TODO(harringtond): It may be possible to remove old revisions of content + // to save memory. + std::map<ContentRevision, feedstore::Content> content_; +}; + +} // namespace stream_model +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_STREAM_MODEL_FEATURE_TREE_H_ diff --git a/chromium/components/feed/core/v2/stream_model_unittest.cc b/chromium/components/feed/core/v2/stream_model_unittest.cc new file mode 100644 index 00000000000..d3a0b4c2825 --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model_unittest.cc @@ -0,0 +1,449 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/stream_model.h" + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "base/optional.h" +#include "base/strings/string_number_conversions.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/content_id.pb.h" +#include "components/feed/core/v2/stream_model_update_request.h" +#include "components/feed/core/v2/test/stream_builder.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { +using StoreUpdate = StreamModel::StoreUpdate; +using UiUpdate = StreamModel::UiUpdate; + +std::vector<std::string> GetContentFrames(const StreamModel& model) { + std::vector<std::string> frames; + for (ContentRevision rev : model.GetContentList()) { + const feedstore::Content* content = model.FindContent(rev); + if (content) { + frames.push_back(content->frame()); + } else { + frames.push_back("<null>"); + } + } + return frames; +} + +class TestObserver : public StreamModel::Observer { + public: + explicit TestObserver(StreamModel* model) { model->SetObserver(this); } + + // StreamModel::Observer. + void OnUiUpdate(const UiUpdate& update) override { update_ = update; } + const base::Optional<UiUpdate>& GetUiUpdate() const { return update_; } + bool ContentListChanged() const { + return update_ && update_->content_list_changed; + } + + void Clear() { update_ = base::nullopt; } + + private: + base::Optional<UiUpdate> update_; +}; + +class TestStoreObserver : public StreamModel::StoreObserver { + public: + explicit TestStoreObserver(StreamModel* model) { + model->SetStoreObserver(this); + } + + // StreamModel::StoreObserver. + void OnStoreChange(const StoreUpdate& records) override { update_ = records; } + + const base::Optional<StoreUpdate>& GetUpdate() const { return update_; } + + void Clear() { update_ = base::nullopt; } + + private: + base::Optional<StoreUpdate> update_; +}; + +TEST(StreamModelTest, ConstructEmptyModel) { + StreamModel model; + TestObserver observer(&model); + + EXPECT_EQ(0UL, model.GetContentList().size()); +} + +TEST(StreamModelTest, ExecuteOperationsTypicalStream) { + StreamModel model; + TestObserver observer(&model); + TestStoreObserver store_observer(&model); + + model.ExecuteOperations(MakeTypicalStreamOperations()); + EXPECT_TRUE(observer.ContentListChanged()); + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); + ASSERT_TRUE(store_observer.GetUpdate()); + ASSERT_EQ(MakeTypicalStreamOperations().size(), + store_observer.GetUpdate()->operations.size()); +} + +TEST(StreamModelTest, AddContentWithoutRoot) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations{ + MakeOperation(MakeCluster(0, MakeRootId())), + MakeOperation(MakeContentNode(0, MakeClusterId(0))), + MakeOperation(MakeContent(0)), + }; + model.ExecuteOperations(operations); + + // Without a root, no content is visible. + EXPECT_EQ(std::vector<std::string>({}), GetContentFrames(model)); +} + +// Verify Stream -> Content works. +TEST(StreamModelTest, AddStreamContent) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations{ + MakeOperation(MakeStream()), + MakeOperation(MakeContentNode(0, MakeRootId())), + MakeOperation(MakeContent(0)), + }; + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({"f:0"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, AddRootAsChild) { + // When the root is added as a child, it's no longer considered a root. + StreamModel model; + TestObserver observer(&model); + feedstore::StreamStructure stream_with_parent = MakeStream(); + *stream_with_parent.mutable_parent_id() = MakeContentContentId(0); + std::vector<feedstore::DataOperation> operations{ + MakeOperation(MakeStream()), + MakeOperation(MakeContentNode(0, MakeRootId())), + MakeOperation(MakeContent(0)), + MakeOperation(stream_with_parent), + }; + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({}), GetContentFrames(model)); +} + +// Changing the STREAM root to CLUSTER means it is no longer eligible to be +// the root. +TEST(StreamModelTest, ChangeStreamToCluster) { + StreamModel model; + TestObserver observer(&model); + feedstore::StreamStructure stream_as_cluster = MakeStream(); + stream_as_cluster.set_type(feedstore::StreamStructure::CLUSTER); + + std::vector<feedstore::DataOperation> operations{ + MakeOperation(MakeStream()), + MakeOperation(MakeContentNode(0, MakeRootId())), + MakeOperation(MakeContent(0)), + MakeOperation(stream_as_cluster), + }; + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({}), GetContentFrames(model)); +} + +TEST(StreamModelTest, RemoveCluster) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeRemove(MakeClusterId(0)))); + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({"f:1"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, RemoveContent) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeRemove(MakeContentContentId(0)))); + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({"f:1"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, RemoveRoot) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeRemove(MakeRootId()))); + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>(), GetContentFrames(model)); +} + +TEST(StreamModelTest, RemoveAndAddRoot) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeRemove(MakeRootId()))); + operations.push_back(MakeOperation(MakeStream())); + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, SwitchStreams) { + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeStream(2))); + operations.push_back(MakeOperation(MakeContentNode(9, MakeRootId(2)))); + operations.push_back(MakeOperation(MakeContent(9))); + + model.ExecuteOperations(operations); + + // The last stream added becomes the root, so only children of 'root2' are + // included. + EXPECT_EQ(std::vector<std::string>({"f:9"}), GetContentFrames(model)); + + // Adding the original stream back will re-activate it. + model.ExecuteOperations({MakeOperation(MakeStream())}); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); + + // Removing 'root' will now make 'root2' active again. + model.ExecuteOperations({MakeOperation(MakeRemove(MakeRootId()))}); + EXPECT_EQ(std::vector<std::string>({"f:9"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, RemoveAndUpdateCluster) { + // Remove a cluster and add it back. Adding it back keeps its original + // placement. + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeRemove(MakeClusterId(0)))); + operations.push_back(MakeOperation(MakeCluster(0, MakeRootId()))); + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, RemoveAndAppendToNewParent) { + // Attempt to re-parent a node. This is not allowed, the old parent remains. + StreamModel model; + TestObserver observer(&model); + + std::vector<feedstore::DataOperation> operations = + MakeTypicalStreamOperations(); + operations.push_back(MakeOperation(MakeRemove(MakeClusterId(0)))); + operations.push_back(MakeOperation(MakeCluster(0, MakeClusterId(1)))); + + model.ExecuteOperations(operations); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, EphemeralNewCluster) { + StreamModel model; + TestObserver observer(&model); + + model.ExecuteOperations(MakeTypicalStreamOperations()); + observer.Clear(); + + model.CreateEphemeralChange({ + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }); + + EXPECT_TRUE(observer.ContentListChanged()); + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1", "f:2"}), + GetContentFrames(model)); +} + +TEST(StreamModelTest, CommitEphemeralChange) { + StreamModel model; + TestObserver observer(&model); + + model.ExecuteOperations(MakeTypicalStreamOperations()); + + EphemeralChangeId change_id = model.CreateEphemeralChange({ + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }); + + observer.Clear(); + TestStoreObserver store_observer(&model); + EXPECT_TRUE(model.CommitEphemeralChange(change_id)); + + // Check that the observer's |OnStoreChange()| was called. + ASSERT_TRUE(store_observer.GetUpdate()); + StoreUpdate store_update = *store_observer.GetUpdate(); + ASSERT_EQ(3UL, store_update.operations.size()); + EXPECT_EQ(feedstore::StreamStructure::CLUSTER, + store_update.operations[0].structure().type()); + EXPECT_EQ(feedstore::StreamStructure::CONTENT, + store_update.operations[1].structure().type()); + + // Can't reject after commit. + EXPECT_FALSE(model.RejectEphemeralChange(change_id)); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1", "f:2"}), + GetContentFrames(model)); +} + +TEST(StreamModelTest, RejectEphemeralChange) { + StreamModel model; + TestObserver observer(&model); + + model.ExecuteOperations(MakeTypicalStreamOperations()); + EphemeralChangeId change_id = model.CreateEphemeralChange({ + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }); + observer.Clear(); + + EXPECT_TRUE(model.RejectEphemeralChange(change_id)); + EXPECT_TRUE(observer.ContentListChanged()); + // Can't commit after reject. + EXPECT_FALSE(model.CommitEphemeralChange(change_id)); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); +} + +TEST(StreamModelTest, RejectFirstEphemeralChange) { + StreamModel model; + TestObserver observer(&model); + + model.ExecuteOperations(MakeTypicalStreamOperations()); + EphemeralChangeId change_id1 = model.CreateEphemeralChange({ + MakeOperation(MakeCluster(2, MakeRootId())), + MakeOperation(MakeContentNode(2, MakeClusterId(2))), + MakeOperation(MakeContent(2)), + }); + + model.CreateEphemeralChange({ + MakeOperation(MakeCluster(3, MakeRootId())), + MakeOperation(MakeContentNode(3, MakeClusterId(3))), + MakeOperation(MakeContent(3)), + }); + observer.Clear(); + + EXPECT_TRUE(model.RejectEphemeralChange(change_id1)); + EXPECT_TRUE(observer.ContentListChanged()); + // Can't commit after reject. + EXPECT_FALSE(model.CommitEphemeralChange(change_id1)); + + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1", "f:3"}), + GetContentFrames(model)); +} + +TEST(StreamModelTest, InitialLoad) { + StreamModel model; + TestObserver observer(&model); + TestStoreObserver store_observer(&model); + model.Update(MakeTypicalInitialModelState()); + + // Check that content was added and the store doesn't receive its own update. + EXPECT_TRUE(observer.ContentListChanged()); + EXPECT_EQ(std::vector<std::string>({"f:0", "f:1"}), GetContentFrames(model)); + ASSERT_EQ(1UL, observer.GetUiUpdate()->shared_states.size()); + EXPECT_NE("", observer.GetUiUpdate()->shared_states[0].shared_state_id); + const std::string* shared_state_data = model.FindSharedStateData( + observer.GetUiUpdate()->shared_states[0].shared_state_id); + ASSERT_TRUE(shared_state_data); + EXPECT_EQ("ss:0", *shared_state_data); + EXPECT_FALSE(store_observer.GetUpdate()); +} + +TEST(StreamModelTest, StoreObserverReceivesIncreasingSequenceNumbers) { + StreamModel model; + TestObserver observer(&model); + TestStoreObserver store_observer(&model); + + // Initialize the model starting at sequence number 5. + { + std::unique_ptr<StreamModelUpdateRequest> initial_state = + MakeTypicalInitialModelState(); + initial_state->max_structure_sequence_number = 5; + model.Update(std::move(initial_state)); + } + + model.ExecuteOperations({MakeOperation(MakeRemove(MakeContentContentId(0)))}); + + ASSERT_TRUE(store_observer.GetUpdate()); + EXPECT_EQ(6, store_observer.GetUpdate()->sequence_number); + + store_observer.Clear(); + model.ExecuteOperations({MakeOperation(MakeRemove(MakeContentContentId(0)))}); + + ASSERT_TRUE(store_observer.GetUpdate()); + EXPECT_EQ(7, store_observer.GetUpdate()->sequence_number); +} + +TEST(StreamModelTest, SharedStateCanBeAddedOnlyOnce) { + StreamModel model; + TestObserver observer(&model); + TestStoreObserver store_observer(&model); + + // Update the model twice with this request. The shared state should not + // be added the second time. + StreamModelUpdateRequest update_request; + update_request.source = + StreamModelUpdateRequest::Source::kInitialLoadFromStore; + update_request.content.push_back(MakeContent(0)); + update_request.stream_structures = {MakeStream(), + MakeCluster(0, MakeRootId()), + MakeContentNode(0, MakeClusterId(0))}; + update_request.shared_states.push_back(MakeSharedState(0)); + + model.Update(std::make_unique<StreamModelUpdateRequest>(update_request)); + observer.Clear(); + model.Update(std::make_unique<StreamModelUpdateRequest>(update_request)); + ASSERT_EQ(1UL, observer.GetUiUpdate()->shared_states.size()); + EXPECT_FALSE(observer.GetUiUpdate()->shared_states[0].updated); +} + +TEST(StreamModelTest, ClearAllErasesSharedStates) { + StreamModel model; + TestObserver observer(&model); + TestStoreObserver store_observer(&model); + // CLEAR_ALL is the first operation in the typical initial model state. + // The second Update() will therefore need to remove and add the shared + // state. + model.Update(MakeTypicalInitialModelState()); + observer.Clear(); + model.Update(MakeTypicalInitialModelState()); + + ASSERT_EQ(1UL, observer.GetUiUpdate()->shared_states.size()); + EXPECT_TRUE(observer.GetUiUpdate()->shared_states[0].updated); +} + +} // namespace +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model_update_request.cc b/chromium/components/feed/core/v2/stream_model_update_request.cc new file mode 100644 index 00000000000..1cd8d8d77fc --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model_update_request.cc @@ -0,0 +1,255 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/stream_model_update_request.h" + +#include <utility> + +#include "base/optional.h" +#include "base/time/time.h" +#include "components/feed/core/proto/v2/wire/data_operation.pb.h" +#include "components/feed/core/proto/v2/wire/feature.pb.h" +#include "components/feed/core/proto/v2/wire/feed_response.pb.h" +#include "components/feed/core/proto/v2/wire/payload_metadata.pb.h" +#include "components/feed/core/proto/v2/wire/stream_structure.pb.h" +#include "components/feed/core/proto/v2/wire/token.pb.h" +#include "components/feed/core/v2/proto_util.h" + +namespace feed { + +namespace { + +feedstore::StreamStructure::Operation TranslateOperationType( + feedwire::DataOperation::Operation operation) { + switch (operation) { + case feedwire::DataOperation::UNKNOWN_OPERATION: + return feedstore::StreamStructure::UNKNOWN; + case feedwire::DataOperation::CLEAR_ALL: + return feedstore::StreamStructure::CLEAR_ALL; + case feedwire::DataOperation::UPDATE_OR_APPEND: + return feedstore::StreamStructure::UPDATE_OR_APPEND; + case feedwire::DataOperation::REMOVE: + return feedstore::StreamStructure::REMOVE; + default: + return feedstore::StreamStructure::UNKNOWN; + } +} + +feedstore::StreamStructure::Type TranslateNodeType( + feedwire::Feature::RenderableUnit renderable_unit) { + switch (renderable_unit) { + case feedwire::Feature::UNKNOWN_RENDERABLE_UNIT: + return feedstore::StreamStructure::UNKNOWN_TYPE; + case feedwire::Feature::STREAM: + return feedstore::StreamStructure::STREAM; + case feedwire::Feature::CONTENT: + return feedstore::StreamStructure::CONTENT; + case feedwire::Feature::CLUSTER: + return feedstore::StreamStructure::CLUSTER; + default: + return feedstore::StreamStructure::UNKNOWN_TYPE; + } +} + +struct ConvertedDataOperation { + bool has_stream_structure = false; + feedstore::StreamStructure stream_structure; + bool has_content = false; + feedstore::Content content; + bool has_shared_state = false; + feedstore::StreamSharedState shared_state; +}; + +bool TranslateFeature(feedwire::Feature* feature, + ConvertedDataOperation* result) { + feedstore::StreamStructure::Type type = + TranslateNodeType(feature->renderable_unit()); + result->stream_structure.set_type(type); + + if (type == feedstore::StreamStructure::CONTENT) { + feedwire::Content* wire_content = feature->mutable_content_extension(); + + if (wire_content->type() != feedwire::Content::XSURFACE) + return false; + + // TODO(iwells): We still need score, availability_time_seconds, + // offline_metadata, and representation_data to populate content_info. + + *(result->content.mutable_content_id()) = + result->stream_structure.content_id(); + result->content.set_allocated_frame( + wire_content->mutable_xsurface_content()->release_xsurface_output()); + result->has_content = true; + } + return true; +} + +base::Optional<feedstore::StreamSharedState> TranslateSharedState( + feedwire::ContentId content_id, + feedwire::RenderData* wire_shared_state) { + if (wire_shared_state->render_data_type() != feedwire::RenderData::XSURFACE) { + return base::nullopt; + } + + feedstore::StreamSharedState shared_state; + *shared_state.mutable_content_id() = std::move(content_id); + shared_state.set_allocated_shared_state_data( + wire_shared_state->mutable_xsurface_container()->release_render_data()); + return shared_state; +} + +bool TranslatePayload(feedwire::DataOperation operation, + ConvertedDataOperation* result) { + switch (operation.payload_case()) { + case feedwire::DataOperation::kFeature: { + feedwire::Feature* feature = operation.mutable_feature(); + result->stream_structure.set_allocated_parent_id( + feature->release_parent_id()); + + if (!TranslateFeature(feature, result)) + return false; + } break; + case feedwire::DataOperation::kNextPageToken: { + feedwire::Token* token = operation.mutable_next_page_token(); + result->stream_structure.set_allocated_parent_id( + token->release_parent_id()); + // TODO(iwells): We should be setting token bytes here. + // result->stream_structure.set_allocated_next_page_token( + // token->MutableExtension( + // components::feed::core::proto::ui + // ::stream::NextPageToken::next_page_token_extension + // )->release_next_page_token()); + } break; + case feedwire::DataOperation::kRenderData: { + base::Optional<feedstore::StreamSharedState> shared_state = + TranslateSharedState(result->stream_structure.content_id(), + operation.mutable_render_data()); + if (!shared_state) + return false; + + result->shared_state = std::move(shared_state.value()); + result->has_shared_state = true; + } break; + // Fall through + case feedwire::DataOperation::kInPlaceUpdateHandle: + case feedwire::DataOperation::kTemplates: + case feedwire::DataOperation::PAYLOAD_NOT_SET: + default: + return false; + } + + return true; +} + +base::Optional<ConvertedDataOperation> TranslateDataOperationInternal( + feedwire::DataOperation operation) { + feedstore::StreamStructure::Operation operation_type = + TranslateOperationType(operation.operation()); + + ConvertedDataOperation result; + result.stream_structure.set_operation(operation_type); + result.has_stream_structure = true; + + switch (operation_type) { + case feedstore::StreamStructure::CLEAR_ALL: + return result; + + case feedstore::StreamStructure::UPDATE_OR_APPEND: + if (!operation.has_metadata() || !operation.metadata().has_content_id()) + return base::nullopt; + + result.stream_structure.set_allocated_content_id( + operation.mutable_metadata()->release_content_id()); + + if (!TranslatePayload(std::move(operation), &result)) + return base::nullopt; + break; + + case feedstore::StreamStructure::REMOVE: + if (!operation.has_metadata() || !operation.metadata().has_content_id()) + return base::nullopt; + + result.stream_structure.set_allocated_content_id( + operation.mutable_metadata()->release_content_id()); + break; + + case feedstore::StreamStructure::UNKNOWN: // Fall through + default: + return base::nullopt; + } + + return result; +} + +} // namespace + +StreamModelUpdateRequest::StreamModelUpdateRequest() = default; +StreamModelUpdateRequest::~StreamModelUpdateRequest() = default; +StreamModelUpdateRequest::StreamModelUpdateRequest( + const StreamModelUpdateRequest&) = default; +StreamModelUpdateRequest& StreamModelUpdateRequest::operator=( + const StreamModelUpdateRequest&) = default; + +base::Optional<feedstore::DataOperation> TranslateDataOperation( + feedwire::DataOperation wire_operation) { + feedstore::DataOperation store_operation; + base::Optional<ConvertedDataOperation> converted = + TranslateDataOperationInternal(std::move(wire_operation)); + if (!converted) + return base::nullopt; + + if (!converted->has_stream_structure && !converted->has_content) + return base::nullopt; + + *store_operation.mutable_structure() = std::move(converted->stream_structure); + *store_operation.mutable_content() = std::move(converted->content); + return store_operation; +} + +std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( + feedwire::Response response, + base::TimeDelta response_time, + base::Time current_time) { + if (response.response_version() != feedwire::Response::FEED_RESPONSE) + return nullptr; + + auto result = std::make_unique<StreamModelUpdateRequest>(); + + feedwire::FeedResponse* feed_response = response.mutable_feed_response(); + for (auto& wire_data_operation : *feed_response->mutable_data_operation()) { + if (!wire_data_operation.has_operation()) + continue; + + base::Optional<ConvertedDataOperation> operation = + TranslateDataOperationInternal(std::move(wire_data_operation)); + if (!operation) + continue; + + if (operation->has_stream_structure) { + result->stream_structures.push_back( + std::move(operation->stream_structure)); + } + + if (operation->has_content) + result->content.push_back(std::move(operation.value().content)); + + if (operation->has_shared_state) + result->shared_states.push_back(std::move(operation->shared_state)); + } + + // TODO(harringtond): If there's more than one shared state, record some + // sort of error. + if (!result->shared_states.empty()) { + *result->stream_data.mutable_shared_state_id() = + result->shared_states.front().content_id(); + } + feedstore::SetLastAddedTime(current_time, &result->stream_data); + result->server_response_time = + feed_response->feed_response_metadata().response_time_ms(); + result->response_time = response_time; + + return result; +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/stream_model_update_request.h b/chromium/components/feed/core/v2/stream_model_update_request.h new file mode 100644 index 00000000000..6aea209a782 --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model_update_request.h @@ -0,0 +1,74 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_ +#define COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_ + +#include <memory> +#include <vector> + +#include "base/optional.h" +#include "base/time/time.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/proto/v2/wire/data_operation.pb.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" + +namespace feed { + +// Data for updating StreamModel. This can be sourced from the network or +// persistent storage. +struct StreamModelUpdateRequest { + public: + enum class Source { + kNetworkUpdate, + kInitialLoadFromStore, + }; + + StreamModelUpdateRequest(); + ~StreamModelUpdateRequest(); + StreamModelUpdateRequest(const StreamModelUpdateRequest&); + StreamModelUpdateRequest& operator=(const StreamModelUpdateRequest&); + + // Whether this data originates is from the initial load of content from + // the local data store. + Source source = Source::kNetworkUpdate; + + // The set of Contents marked UPDATE_OR_APPEND in the response, in the order + // in which they were received. + std::vector<feedstore::Content> content; + + // Contains the root ContentId, tokens, a timestamp for when the most recent + // content was added, and a list of ContentIds for clusters in the response. + feedstore::StreamData stream_data; + + // The set of StreamSharedStates marked UPDATE_OR_APPEND in the order in which + // they were received. + std::vector<feedstore::StreamSharedState> shared_states; + + std::vector<feedstore::StreamStructure> stream_structures; + + // If this data originates from the network, this is the server-reported time + // at which the request was fulfilled. + // TODO(harringtond): Use this or remove it. + int64_t server_response_time = 0; + + // If this data originates from the network, this is the time taken by the + // server to produce the response. + // TODO(harringtond): Use this or remove it. + base::TimeDelta response_time; + + int32_t max_structure_sequence_number = 0; +}; + +base::Optional<feedstore::DataOperation> TranslateDataOperation( + feedwire::DataOperation wire_operation); + +std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse( + feedwire::Response response, + base::TimeDelta response_time, + base::Time current_time); + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_ diff --git a/chromium/components/feed/core/v2/stream_model_update_request_unittest.cc b/chromium/components/feed/core/v2/stream_model_update_request_unittest.cc new file mode 100644 index 00000000000..1e70748d99a --- /dev/null +++ b/chromium/components/feed/core/v2/stream_model_update_request_unittest.cc @@ -0,0 +1,146 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/stream_model_update_request.h" + +#include <string> + +#include "base/base_paths.h" +#include "base/files/file_path.h" +#include "base/files/file_util.h" +#include "base/path_service.h" +#include "base/strings/string_number_conversions.h" +#include "base/time/time.h" +#include "components/feed/core/proto/v2/wire/feed_response.pb.h" +#include "components/feed/core/proto/v2/wire/response.pb.h" +#include "components/feed/core/v2/proto_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace feed { +namespace { + +const char kResponsePbPath[] = "components/test/data/feed/response.binarypb"; +constexpr base::TimeDelta kResponseTime = base::TimeDelta::FromSeconds(42); +const base::Time kCurrentTime = + base::Time::UnixEpoch() + base::TimeDelta::FromDays(123); +// TODO(iwells): Replace response.binarypb with a response that uses the new +// wire protocol. +// +// Since we're currently using a Jardin response which includes a +// Piet shared state, and translation skips handling Piet shared states, we +// expect to have only 33 StreamStructures even though there are 34 wire +// operations. +const int kExpectedStreamStructureCount = 33; +const size_t kExpectedContentCount = 10; +const size_t kExpectedSharedStateCount = 0; + +std::string ContentIdToString(const feedwire::ContentId& content_id) { + return "{content_domain: \"" + content_id.content_domain() + + "\", id: " + base::NumberToString(content_id.id()) + ", type: \"" + + feedwire::ContentId::Type_Name(content_id.type()) + "\"}"; +} + +feedwire::Response TestWireResponse() { + // Read and parse response.binarypb. + base::FilePath response_file_path; + CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &response_file_path)); + response_file_path = response_file_path.AppendASCII(kResponsePbPath); + + CHECK(base::PathExists(response_file_path)); + + std::string response_data; + CHECK(base::ReadFileToString(response_file_path, &response_data)); + + feedwire::Response response; + CHECK(response.ParseFromString(response_data)); + return response; +} + +} // namespace + +// TODO(iwells): Test failure cases once the new protos are ready. + +TEST(StreamModelUpdateRequestTest, TranslateRealResponse) { + // Tests how proto translation works on a real response from the server. + // + // The response will periodically need to be updated as changes are made to + // the server. Update testdata/response.textproto and then run + // tools/generate_test_response_binarypb.sh. + + feedwire::Response response = TestWireResponse(); + feedwire::FeedResponse feed_response = response.feed_response(); + + // TODO(iwells): Make these exactly equal once we aren't using an old + // response. + ASSERT_EQ(feed_response.data_operation_size(), + kExpectedStreamStructureCount + 1); + + std::unique_ptr<StreamModelUpdateRequest> translated = + TranslateWireResponse(response, kResponseTime, kCurrentTime); + + ASSERT_TRUE(translated); + EXPECT_EQ(kCurrentTime, feedstore::GetLastAddedTime(translated->stream_data)); + ASSERT_EQ(translated->stream_structures.size(), + static_cast<size_t>(kExpectedStreamStructureCount)); + + const std::vector<feedstore::StreamStructure>& structures = + translated->stream_structures; + + // Check CLEAR_ALL: + EXPECT_EQ(structures[0].operation(), feedstore::StreamStructure::CLEAR_ALL); + + // TODO(iwells): Check the shared state once we have a new + + // Check UPDATE_OR_APPEND for the root: + EXPECT_EQ(structures[1].operation(), + feedstore::StreamStructure::UPDATE_OR_APPEND); + EXPECT_EQ(structures[1].type(), feedstore::StreamStructure::STREAM); + EXPECT_TRUE(structures[1].has_content_id()); + EXPECT_FALSE(structures[1].has_parent_id()); + + feedwire::ContentId root_content_id = structures[1].content_id(); + + // Content: + EXPECT_EQ(structures[2].operation(), + feedstore::StreamStructure::UPDATE_OR_APPEND); + EXPECT_EQ(structures[2].type(), feedstore::StreamStructure::CONTENT); + EXPECT_TRUE(structures[2].has_content_id()); + EXPECT_TRUE(structures[2].has_parent_id()); + + // TODO(iwells): Uncomment when these are available. + // EXPECT_TRUE(structures[3].has_content_info()); + // EXPECT_NE(structures[3].content_info().score(), 0.); + // EXPECT_NE(structures[3].content_info().availability_time_seconds(), 0); + // EXPECT_TRUE(structures[3].content_info().has_representation_data()); + // EXPECT_TRUE(structures[3].content_info().has_offline_metadata()); + + ASSERT_GT(translated->content.size(), 0UL); + EXPECT_EQ(ContentIdToString(translated->content[0].content_id()), + ContentIdToString(structures[2].content_id())); + // TODO(iwells): Check content.frame() once this is available. + + // Non-content structures: + EXPECT_EQ(structures[3].operation(), + feedstore::StreamStructure::UPDATE_OR_APPEND); + // TODO(iwells): This is a CARD. Remove once we have a new response. + EXPECT_EQ(structures[3].type(), feedstore::StreamStructure::UNKNOWN_TYPE); + EXPECT_TRUE(structures[3].has_content_id()); + EXPECT_TRUE(structures[3].has_parent_id()); + + EXPECT_EQ(structures[4].operation(), + feedstore::StreamStructure::UPDATE_OR_APPEND); + EXPECT_EQ(structures[4].type(), feedstore::StreamStructure::CLUSTER); + EXPECT_TRUE(structures[4].has_content_id()); + EXPECT_TRUE(structures[4].has_parent_id()); + EXPECT_EQ(ContentIdToString(structures[4].parent_id()), + ContentIdToString(root_content_id)); + + // The other members: + EXPECT_EQ(translated->content.size(), kExpectedContentCount); + EXPECT_EQ(translated->shared_states.size(), kExpectedSharedStateCount); + + EXPECT_EQ(translated->response_time, kResponseTime); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc new file mode 100644 index 00000000000..c3c2a261a31 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.cc @@ -0,0 +1,127 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/tasks/load_stream_from_store_task.h" + +#include <algorithm> +#include <utility> + +#include "base/time/clock.h" +#include "components/feed/core/proto/v2/store.pb.h" +#include "components/feed/core/v2/feed_store.h" +#include "components/feed/core/v2/proto_util.h" +#include "components/feed/core/v2/public/feed_stream_api.h" +#include "components/feed/core/v2/scheduling.h" +#include "components/feed/core/v2/stream_model_update_request.h" + +namespace feed { + +LoadStreamFromStoreTask::Result::Result() = default; +LoadStreamFromStoreTask::Result::~Result() = default; +LoadStreamFromStoreTask::Result::Result(Result&&) = default; +LoadStreamFromStoreTask::Result& LoadStreamFromStoreTask::Result::operator=( + Result&&) = default; + +LoadStreamFromStoreTask::LoadStreamFromStoreTask( + FeedStore* store, + const base::Clock* clock, + UserClass user_class, + base::OnceCallback<void(Result)> callback) + : store_(store), + clock_(clock), + user_class_(user_class), + result_callback_(std::move(callback)), + update_request_(std::make_unique<StreamModelUpdateRequest>()) {} + +LoadStreamFromStoreTask::~LoadStreamFromStoreTask() = default; + +void LoadStreamFromStoreTask::Run() { + store_->LoadStream( + base::BindOnce(&LoadStreamFromStoreTask::LoadStreamDone, GetWeakPtr())); +} + +void LoadStreamFromStoreTask::LoadStreamDone( + FeedStore::LoadStreamResult result) { + if (result.read_error) { + Complete(LoadStreamStatus::kFailedWithStoreError); + return; + } + if (result.stream_structures.empty()) { + Complete(LoadStreamStatus::kNoStreamDataInStore); + return; + } + if (!ignore_staleness_) { + const base::TimeDelta content_age = + clock_->Now() - feedstore::GetLastAddedTime(result.stream_data); + if (content_age < base::TimeDelta()) { + Complete(LoadStreamStatus::kDataInStoreIsStaleTimestampInFuture); + return; + } else if (ShouldWaitForNewContent(user_class_, true, content_age)) { + Complete(LoadStreamStatus::kDataInStoreIsStale); + return; + } + } + + // TODO(harringtond): Add other failure cases? + + std::vector<ContentId> referenced_content_ids; + for (const feedstore::StreamStructureSet& structure_set : + result.stream_structures) { + for (const feedstore::StreamStructure& structure : + structure_set.structures()) { + if (structure.type() == feedstore::StreamStructure::CONTENT) { + referenced_content_ids.push_back(structure.content_id()); + } + } + } + + store_->ReadContent( + std::move(referenced_content_ids), {result.stream_data.shared_state_id()}, + base::BindOnce(&LoadStreamFromStoreTask::LoadContentDone, GetWeakPtr())); + + update_request_->stream_data = std::move(result.stream_data); + + // Move stream structures into the update request. + // These need sorted by sequence number, and then inserted into + // |update_request_->stream_structures|. + std::sort(result.stream_structures.begin(), result.stream_structures.end(), + [](const feedstore::StreamStructureSet& a, + const feedstore::StreamStructureSet& b) { + return a.sequence_number() < b.sequence_number(); + }); + + for (feedstore::StreamStructureSet& structure_set : + result.stream_structures) { + update_request_->max_structure_sequence_number = + structure_set.sequence_number(); + for (feedstore::StreamStructure& structure : + *structure_set.mutable_structures()) { + update_request_->stream_structures.push_back(std::move(structure)); + } + } +} + +void LoadStreamFromStoreTask::LoadContentDone( + std::vector<feedstore::Content> content, + std::vector<feedstore::StreamSharedState> shared_states) { + update_request_->content = std::move(content); + update_request_->shared_states = std::move(shared_states); + + update_request_->source = + StreamModelUpdateRequest::Source::kInitialLoadFromStore; + + Complete(LoadStreamStatus::kLoadedFromStore); +} + +void LoadStreamFromStoreTask::Complete(LoadStreamStatus status) { + Result task_result; + task_result.status = status; + if (status == LoadStreamStatus::kLoadedFromStore) { + task_result.update_request = std::move(update_request_); + } + std::move(result_callback_).Run(std::move(task_result)); + TaskComplete(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h new file mode 100644 index 00000000000..3718c2c9fd1 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/load_stream_from_store_task.h @@ -0,0 +1,71 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_FROM_STORE_TASK_H_ +#define COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_FROM_STORE_TASK_H_ + +#include <memory> +#include <vector> + +#include "base/callback.h" +#include "base/memory/weak_ptr.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_store.h" +#include "components/offline_pages/task/task.h" + +namespace base { +class Clock; +} + +namespace feed { +struct StreamModelUpdateRequest; + +// Attempts to load stream data from persistent storage. +class LoadStreamFromStoreTask : public offline_pages::Task { + public: + struct Result { + Result(); + ~Result(); + Result(Result&&); + Result& operator=(Result&&); + LoadStreamStatus status = LoadStreamStatus::kNoStatus; + std::unique_ptr<StreamModelUpdateRequest> update_request; + }; + + LoadStreamFromStoreTask(FeedStore* store, + const base::Clock* clock, + UserClass user_class, + base::OnceCallback<void(Result)> callback); + ~LoadStreamFromStoreTask() override; + LoadStreamFromStoreTask(const LoadStreamFromStoreTask&) = delete; + LoadStreamFromStoreTask& operator=(const LoadStreamFromStoreTask&) = delete; + + void IgnoreStalenessForTesting() { ignore_staleness_ = true; } + + private: + void Run() override; + + void LoadStreamDone(FeedStore::LoadStreamResult); + void LoadContentDone(std::vector<feedstore::Content> content, + std::vector<feedstore::StreamSharedState> shared_states); + void Complete(LoadStreamStatus status); + + base::WeakPtr<LoadStreamFromStoreTask> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + + FeedStore* store_; // Unowned. + const base::Clock* clock_; + UserClass user_class_; + bool ignore_staleness_ = false; + base::OnceCallback<void(Result)> result_callback_; + + std::unique_ptr<StreamModelUpdateRequest> update_request_; + + base::WeakPtrFactory<LoadStreamFromStoreTask> weak_ptr_factory_{this}; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_FROM_STORE_TASK_H_ diff --git a/chromium/components/feed/core/v2/tasks/load_stream_task.cc b/chromium/components/feed/core/v2/tasks/load_stream_task.cc new file mode 100644 index 00000000000..82e10860897 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/load_stream_task.cc @@ -0,0 +1,120 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/tasks/load_stream_task.h" + +#include <memory> +#include <utility> + +#include "base/bind_helpers.h" +#include "base/logging.h" +#include "base/time/clock.h" +#include "base/time/time.h" +#include "components/feed/core/proto/v2/wire/client_info.pb.h" +#include "components/feed/core/proto/v2/wire/feed_request.pb.h" +#include "components/feed/core/proto/v2/wire/request.pb.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/feed_stream.h" +#include "components/feed/core/v2/stream_model.h" +#include "components/feed/core/v2/stream_model_update_request.h" + +namespace feed { + +LoadStreamTask::LoadStreamTask(FeedStream* stream, + base::OnceCallback<void(Result)> done_callback) + : stream_(stream), done_callback_(std::move(done_callback)) {} + +LoadStreamTask::~LoadStreamTask() = default; + +void LoadStreamTask::Run() { + // Phase 1. + // - Return early if the model is already loaded. + // - Try to load from persistent storage. + + // Don't load if the model is already loaded. + if (stream_->GetModel()) { + Done(LoadStreamStatus::kModelAlreadyLoaded); + return; + } + + load_from_store_task_ = std::make_unique<LoadStreamFromStoreTask>( + stream_->GetStore(), stream_->GetClock(), stream_->GetUserClass(), + base::BindOnce(&LoadStreamTask::LoadFromStoreComplete, GetWeakPtr())); + load_from_store_task_->Execute(base::DoNothing()); +} + +void LoadStreamTask::LoadFromStoreComplete( + LoadStreamFromStoreTask::Result result) { + load_from_store_status_ = result.status; + // Phase 2. + // - If loading from store works, update the model. + // - Otherwise, try to load from the network. + + if (result.status == LoadStreamStatus::kLoadedFromStore) { + auto model = std::make_unique<StreamModel>(); + model->Update(std::move(result.update_request)); + stream_->LoadModel(std::move(model)); + Done(LoadStreamStatus::kLoadedFromStore); + return; + } + + LoadStreamStatus final_status = stream_->ShouldMakeFeedQueryRequest(); + if (final_status != LoadStreamStatus::kNoStatus) { + Done(final_status); + return; + } + + // TODO(harringtond): Add throttling. + // TODO(harringtond): Request parameters here are all placeholder values. + feedwire::Request request; + feedwire::ClientInfo& client_info = + *request.mutable_feed_request()->mutable_client_info(); + client_info.set_platform_type(feedwire::ClientInfo::ANDROID_ID); + client_info.set_app_type(feedwire::ClientInfo::CHROME); + request.mutable_feed_request()->mutable_feed_query()->set_reason( + feedwire::FeedQuery::MANUAL_REFRESH); + + fetch_start_time_ = base::TimeTicks::Now(); + stream_->GetNetwork()->SendQueryRequest( + request, + base::BindOnce(&LoadStreamTask::QueryRequestComplete, GetWeakPtr())); +} + +void LoadStreamTask::QueryRequestComplete( + FeedNetwork::QueryRequestResult result) { + DCHECK(!stream_->GetModel()); + if (!result.response_body) { + Done(LoadStreamStatus::kNoResponseBody); + return; + } + + std::unique_ptr<StreamModelUpdateRequest> update_request = + stream_->GetWireResponseTranslator()->TranslateWireResponse( + *result.response_body, base::TimeTicks::Now() - fetch_start_time_, + stream_->GetClock()->Now()); + if (!update_request) { + Done(LoadStreamStatus::kProtoTranslationFailed); + return; + } + + stream_->GetStore()->SaveFullStream( + std::make_unique<StreamModelUpdateRequest>(*update_request), + base::DoNothing()); + + auto model = std::make_unique<StreamModel>(); + model->Update(std::move(update_request)); + stream_->LoadModel(std::move(model)); + + Done(LoadStreamStatus::kLoadedFromNetwork); +} + +void LoadStreamTask::Done(LoadStreamStatus status) { + Result result; + result.load_from_store_status = load_from_store_status_; + result.final_status = status; + std::move(done_callback_).Run(result); + TaskComplete(); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/load_stream_task.h b/chromium/components/feed/core/v2/tasks/load_stream_task.h new file mode 100644 index 00000000000..ad19cd90d64 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/load_stream_task.h @@ -0,0 +1,62 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_TASK_H_ +#define COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_TASK_H_ + +#include <memory> + +#include "base/callback.h" +#include "base/memory/weak_ptr.h" +#include "components/feed/core/v2/enums.h" +#include "components/feed/core/v2/feed_network.h" +#include "components/feed/core/v2/tasks/load_stream_from_store_task.h" +#include "components/offline_pages/task/task.h" + +namespace feed { +class FeedStream; + +// Loads the stream model from storage or network. +// If successful, this directly forces a model load in |FeedStream()| +// before completing the task. +// TODO(harringtond): If we read data from the network, it needs to be +// persisted. +class LoadStreamTask : public offline_pages::Task { + public: + struct Result { + Result() = default; + explicit Result(LoadStreamStatus a_final_status) + : final_status(a_final_status) {} + // Final status of loading the stream. + LoadStreamStatus final_status = LoadStreamStatus::kNoStatus; + // Status of just loading the stream from the persistent store, if that + // was attempted. + LoadStreamStatus load_from_store_status = LoadStreamStatus::kNoStatus; + }; + explicit LoadStreamTask(FeedStream* stream, + base::OnceCallback<void(Result)> done_callback); + ~LoadStreamTask() override; + LoadStreamTask(const LoadStreamTask&) = delete; + LoadStreamTask& operator=(const LoadStreamTask&) = delete; + + private: + void Run() override; + base::WeakPtr<LoadStreamTask> GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + + void LoadFromStoreComplete(LoadStreamFromStoreTask::Result result); + void QueryRequestComplete(FeedNetwork::QueryRequestResult result); + void Done(LoadStreamStatus status); + + FeedStream* stream_; // Unowned. + std::unique_ptr<LoadStreamFromStoreTask> load_from_store_task_; + LoadStreamStatus load_from_store_status_ = LoadStreamStatus::kNoStatus; + base::TimeTicks fetch_start_time_; + base::OnceCallback<void(Result)> done_callback_; + base::WeakPtrFactory<LoadStreamTask> weak_ptr_factory_{this}; +}; +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TASKS_LOAD_STREAM_TASK_H_ diff --git a/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc new file mode 100644 index 00000000000..27f73cbb1be --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.cc @@ -0,0 +1,21 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/feed/core/v2/tasks/wait_for_store_initialize_task.h" + +#include "components/feed/core/v2/feed_store.h" + +namespace feed { + +WaitForStoreInitializeTask::WaitForStoreInitializeTask(FeedStore* store) + : store_(store) {} +WaitForStoreInitializeTask::~WaitForStoreInitializeTask() = default; + +void WaitForStoreInitializeTask::Run() { + // |this| stays alive as long as the |store_|, so Unretained is safe. + store_->Initialize(base::BindOnce(&WaitForStoreInitializeTask::TaskComplete, + base::Unretained(this))); +} + +} // namespace feed diff --git a/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h new file mode 100644 index 00000000000..f7640676c68 --- /dev/null +++ b/chromium/components/feed/core/v2/tasks/wait_for_store_initialize_task.h @@ -0,0 +1,31 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_FEED_CORE_V2_TASKS_WAIT_FOR_STORE_INITIALIZE_TASK_H_ +#define COMPONENTS_FEED_CORE_V2_TASKS_WAIT_FOR_STORE_INITIALIZE_TASK_H_ + +#include "components/offline_pages/task/task.h" + +namespace feed { +class FeedStore; + +// Initializes |store|. This task is run first so that other tasks can assume +// storage is initialized. +class WaitForStoreInitializeTask : public offline_pages::Task { + public: + explicit WaitForStoreInitializeTask(FeedStore* store); + ~WaitForStoreInitializeTask() override; + WaitForStoreInitializeTask(const WaitForStoreInitializeTask&) = delete; + WaitForStoreInitializeTask& operator=(const WaitForStoreInitializeTask&) = + delete; + + private: + void Run() override; + + FeedStore* store_; +}; + +} // namespace feed + +#endif // COMPONENTS_FEED_CORE_V2_TASKS_WAIT_FOR_STORE_INITIALIZE_TASK_H_ diff --git a/chromium/components/feed/core/v2/tools/__init__.py b/chromium/components/feed/core/v2/tools/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/chromium/components/feed/core/v2/tools/__init__.py diff --git a/chromium/components/feed/core/v2/tools/feed_response_to_textproto.sh b/chromium/components/feed/core/v2/tools/feed_response_to_textproto.sh new file mode 100755 index 00000000000..48c3da5d410 --- /dev/null +++ b/chromium/components/feed/core/v2/tools/feed_response_to_textproto.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Converts a Feed HTTP response from binary to text using proto definitions from +# Chromium. +# +# Usage: feed_response_to_textproto.sh <in.binarypb> <out.textproto> + +IN_FILE=$1 +OUT_FILE=$2 +TMP_FILE=/tmp/trimmedfeedresponse.binarypb + +CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..) +FEEDPROTO="$CHROMIUM_SRC/components/feed/core/proto" + +# Responses start with a 4-byte length value that must be removed. +tail -c +4 $IN_FILE > $TMP_FILE + +python3 $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \ + --chromium_path=$CHROMIUM_SRC \ + --output_file=$OUT_FILE \ + --source_file=$TMP_FILE \ + --direction=reverse diff --git a/chromium/components/feed/core/v2/tools/generate_test_response_binarypb.sh b/chromium/components/feed/core/v2/tools/generate_test_response_binarypb.sh new file mode 100755 index 00000000000..8265181451c --- /dev/null +++ b/chromium/components/feed/core/v2/tools/generate_test_response_binarypb.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..) +OUT_DIR=$CHROMIUM_SRC/components/test/data/feed + +if [ ! -d $OUT_DIR ]; then + echo "Output directory $OUT_DIR doesn't exist." + exit 1 +fi + +python3 $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \ + --chromium_path=$CHROMIUM_SRC \ + --output_file=$OUT_DIR/response.binarypb \ + --source_file=\ +$CHROMIUM_SRC/components/feed/core/v2/testdata/response.textproto diff --git a/chromium/components/feed/core/v2/tools/protoc_util.py b/chromium/components/feed/core/v2/tools/protoc_util.py new file mode 100755 index 00000000000..41104fe5db5 --- /dev/null +++ b/chromium/components/feed/core/v2/tools/protoc_util.py @@ -0,0 +1,63 @@ +#!/usr/bin/python3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Lint as: python3 +"""The tools provides lot of protoc related helper functions.""" + +import glob +import os +import subprocess + +_protoc_path = None + +def run_command(args, input): + """Uses subprocess to execute the command line args.""" + proc = subprocess.run( + args, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + return proc.stdout + + +def get_protoc_common_args(root_dir, proto_path): + """Returns a list of protoc common args as a list.""" + result = [ + '-I' + os.path.join(root_dir) + ] + for root, _, files in os.walk(os.path.join(root_dir, proto_path)): + result += [os.path.join(root, f) for f in files if f.endswith('.proto')] + + return result + + +def encode_proto(text, message_name, root_dir, proto_path): + """Calls a command line to encode the text string and returns binary bytes.""" + return run_command([protoc_path(root_dir), '--encode=' + message_name] + + get_protoc_common_args(root_dir, proto_path), + text.encode()) + + +def decode_proto(data, message_name, root_dir, proto_path): + """Calls a command line to decode the binary bytes array into text string.""" + return run_command([protoc_path(root_dir), '--decode=' + message_name + ] + get_protoc_common_args(root_dir, proto_path), + data).decode('utf-8') + + +def protoc_path(root_dir): + """Returns the path to the proto compiler, protoc.""" + global _protoc_path + if not _protoc_path: + protoc_list = list( + glob.glob(os.path.join(root_dir, "out") + "/*/protoc")) + list( + glob.glob(os.path.join(root_dir, "out") + "/*/*/protoc")) + if not len(protoc_list): + print("Can't find a suitable build output directory", + "(it should have protoc)") + sys.exit(1) + _protoc_path = protoc_list[0] + return _protoc_path diff --git a/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py b/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py new file mode 100755 index 00000000000..14cb23e915e --- /dev/null +++ b/chromium/components/feed/core/v2/tools/textpb_to_binarypb.py @@ -0,0 +1,81 @@ +#!/usr/bin/python3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Lint as: python3 +"""The tool converts a textpb into a binary proto using chromium protoc binary. + +Make sure you have absl-py installed via 'python3 -m pip install absl-py'. + +Usage example: + python3 ./textpb_to_binarypb.py + --chromium_path ~/chromium/src + --output_file /tmp/binary.pb + --source_file /tmp/original.textpb +""" + +import glob +import os +import protoc_util +import subprocess + +from absl import app +from absl import flags + +DEFAULT_MESSAGE = 'feedwire.Response' + +FLAGS = flags.FLAGS +FLAGS = flags.FLAGS +flags.DEFINE_string('chromium_path', '', 'The path of your chromium depot.') +flags.DEFINE_string('output_file', '', 'The target output binary file path.') +flags.DEFINE_string('source_file', '', + 'The source proto file, in textpb format, path.') +flags.DEFINE_string('message', + DEFAULT_MESSAGE, + 'The message to look for in source_file.') +flags.DEFINE_string('direction', 'forward', + 'Set --direction=reverse to convert binary to text.') + +COMPONENT_FEED_PROTO_PATH = 'components/feed/core/proto' + +def text_to_binary(): + with open(FLAGS.source_file, mode='r') as file: + value_text_proto = file.read() + + encoded = protoc_util.encode_proto(value_text_proto, FLAGS.message, + FLAGS.chromium_path, + COMPONENT_FEED_PROTO_PATH) + with open(FLAGS.output_file, mode='wb') as file: + file.write(encoded) + +def binary_to_text(): + with open(FLAGS.source_file, mode='rb') as file: + value_text_proto = file.read() + + encoded = protoc_util.decode_proto(value_text_proto, FLAGS.message, + FLAGS.chromium_path, + COMPONENT_FEED_PROTO_PATH) + + with open(FLAGS.output_file, mode='w') as file: + file.write(encoded) + +def main(argv): + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + if not FLAGS.chromium_path: + raise app.UsageError('chromium_path flag must be set.') + if not FLAGS.source_file: + raise app.UsageError('source_file flag must be set.') + if not FLAGS.output_file: + raise app.UsageError('output_file flag must be set.') + if FLAGS.direction != 'forward' and FLAGS.direction != 'reverse': + raise app.UsageError('direction must be forward or reverse') + + if FLAGS.direction == 'forward': + text_to_binary() + elif FLAGS.direction == 'reverse': + binary_to_text() + +if __name__ == '__main__': + app.run(main) diff --git a/chromium/components/feed/feed_feature_list.cc b/chromium/components/feed/feed_feature_list.cc index 25e149321be..b8f44049912 100644 --- a/chromium/components/feed/feed_feature_list.cc +++ b/chromium/components/feed/feed_feature_list.cc @@ -24,4 +24,10 @@ const base::FeatureParam<bool> kOnlySetLastRefreshAttemptOnSuccess{ const base::Feature kInterestFeedNotifications{ "InterestFeedNotifications", base::FEATURE_DISABLED_BY_DEFAULT}; +const base::Feature kInterestFeedFeedback{"InterestFeedFeedback", + base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kReportFeedUserActions{"ReportFeedUserActions", + base::FEATURE_DISABLED_BY_DEFAULT}; + } // namespace feed diff --git a/chromium/components/feed/feed_feature_list.h b/chromium/components/feed/feed_feature_list.h index 41342be8e74..fe5e1141bbf 100644 --- a/chromium/components/feed/feed_feature_list.h +++ b/chromium/components/feed/feed_feature_list.h @@ -22,6 +22,12 @@ extern const base::FeatureParam<bool> kOnlySetLastRefreshAttemptOnSuccess; extern const base::Feature kInterestFeedNotifications; +extern const base::Feature kInterestFeedFeedback; + +// Indicates if user card clicks and views in Chrome's feed should be reported +// for personalization. +extern const base::Feature kReportFeedUserActions; + } // namespace feed #endif // COMPONENTS_FEED_FEED_FEATURE_LIST_H_ diff --git a/chromium/components/feed/tools/__init__.py b/chromium/components/feed/tools/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/chromium/components/feed/tools/__init__.py diff --git a/chromium/components/feed/tools/content_dump.py b/chromium/components/feed/tools/content_dump.py index 0f28a9ef481..91d5cdee017 100755 --- a/chromium/components/feed/tools/content_dump.py +++ b/chromium/components/feed/tools/content_dump.py @@ -14,15 +14,16 @@ # Make any desired modifications, and then upload the dump back to the connected # device. # > content_dump.py --device=FA77D0303076 --apk='com.chrome.canary' --reverse +import argparse +import glob import os +import plyvel +import protoc_util import re -import sys -import argparse import subprocess -import glob -from os.path import join, dirname, realpath +import sys -import plyvel +from os.path import join, dirname, realpath # A dynamic import for encoding and decoding of escaped textproto strings. _prototext_mod = None @@ -61,21 +62,8 @@ DUMP_DIR = args.dump_to DB_PATH = args.db CONTENT_DB_PATH = join(DB_PATH, 'content') DEVICE_DB_PATH = "/data/data/{}/app_chrome/Default/feed".format(args.apk) -_protoc_path = None - - -# Returns the path to the proto compiler, protoc. -def protoc_path(): - global _protoc_path - if not _protoc_path: - protoc_list = list(glob.glob(join(ROOT_DIR, "out") + "/*/protoc")) + list( - glob.glob(join(ROOT_DIR, "out") + "/*/*/protoc")) - if not len(protoc_list): - print("Can't find a suitable build output directory", - "(it should have protoc)") - sys.exit(1) - _protoc_path = protoc_list[0] - return _protoc_path +CONTENT_STORAGE_PROTO = ( + 'components/feed_library/core/proto/content_storage.proto') def adb_base_args(): @@ -97,45 +85,6 @@ def adb_push_db(): ["push", CONTENT_DB_PATH, DEVICE_DB_PATH]) -def get_feed_protos(): - result = [ - join(ROOT_DIR, 'components/feed_library/core/proto/content_storage.proto') - ] - for root, _, files in os.walk(join(ROOT_DIR, "third_party/feed_library")): - result += [join(root, f) for f in files if f.endswith('.proto')] - - return result - - -protoc_common_args = [ - '-I' + join(ROOT_DIR, 'third_party/feed_library/src'), '-I' + join(ROOT_DIR) -] + get_feed_protos() - - -def run_command(args, input): - proc = subprocess.run( - args, - input=input, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True) - return proc.stdout - - -# Decode a binary proto into textproto format. -def decode_proto(data, message_name): - return run_command( - [protoc_path(), '--decode=' + message_name] + protoc_common_args, - data).decode('utf-8') - - -# Encode a textproto into binary proto format. -def encode_proto(text, message_name): - return run_command( - [protoc_path(), '--encode=' + message_name] + protoc_common_args, - text.encode()) - - # Ignore DB entries with the 'sp::' prefix, as they are not yet supported. def is_key_supported(key): return not key.startswith('sp::') @@ -155,13 +104,15 @@ def proto_message_from_db_key(key): def extract_db_entry(key, data): # DB entries are feed.ContentStorageProto messages. First extract # the content_data contained within. - text_proto = decode_proto(data, 'feed.ContentStorageProto') + text_proto = protoc_util.decode_proto(data, 'feed.ContentStorageProto', + ROOT_DIR, CONTENT_STORAGE_PROTO) m = re.search(r"content_data: \"((?:\\\"|[^\"])*)\"", text_proto) raw_data = prototext().CUnescape(m.group(1)) # Next, convert raw_data into a textproto. The DB key informs which message # is stored. - result = decode_proto(raw_data, proto_message_from_db_key(key)) + result = protoc_util.decode_proto(raw_data, proto_message_from_db_key(key), + ROOT_DIR, CONTENT_STORAGE_PROTO) return result @@ -196,15 +147,17 @@ def load(): key = file.read().strip() with open(join(DUMP_DIR, f), 'r') as file: value_text_proto = file.read() - value_encoded = encode_proto(value_text_proto, - proto_message_from_db_key(key)) + value_encoded = protoc_util.encode_proto(value_text_proto, + proto_message_from_db_key(key), + ROOT_DIR, CONTENT_STORAGE_PROTO) # Create binary feed.ContentStorageProto by encoding its textproto. content_storage_text = 'key: "{}"\ncontent_data: "{}"'.format( prototext().CEscape(key, False), prototext().CEscape(value_encoded, False)) - store_encoded = encode_proto(content_storage_text, - 'feed.ContentStorageProto') + store_encoded = protoc_util.encode_proto(content_storage_text, + 'feed.ContentStorageProto', + ROOT_DIR, CONTENT_STORAGE_PROTO) db.put(key.encode(), store_encoded) db.close() adb_push_db() diff --git a/chromium/components/feed/tools/mockserver_textpb_to_binary.py b/chromium/components/feed/tools/mockserver_textpb_to_binary.py new file mode 100755 index 00000000000..b6d75745b27 --- /dev/null +++ b/chromium/components/feed/tools/mockserver_textpb_to_binary.py @@ -0,0 +1,64 @@ +#!/usr/bin/python3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Lint as: python3 +"""The tool converts a textpb into a binary proto using chromium protoc binary. + +After converting a feed response textpb file into a mockserver textpb file using +the proto_convertor script, then a engineer runs this script to encode the +mockserver textpb file into a binary proto file that is being used by the feed +card render test (Refers to go/create-a-feed-card-render-test for more). + +Make sure you have absl-py installed via 'python3 -m pip install absl-py'. + +Usage example: + python3 ./mockserver_textpb_to_binary.py + --chromium_path ~/chromium/src + --output_file /tmp/binary.pb + --source_file /tmp/original.textpb + --alsologtostderr +""" + +import glob +import os +import protoc_util +import subprocess + +from absl import app +from absl import flags + +FLAGS = flags.FLAGS +FLAGS = flags.FLAGS +flags.DEFINE_string('chromium_path', '', 'The path of your chromium depot.') +flags.DEFINE_string('output_file', '', 'The target output binary file path.') +flags.DEFINE_string('source_file', '', + 'The source proto file, in textpb format, path.') + +ENCODE_NAMESPACE = 'components.feed.core.proto.wire.mockserver.MockServer' +COMPONENT_FEED_PROTO_PATH = 'components/feed/core/proto' + + +def main(argv): + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + if not FLAGS.chromium_path: + raise app.UsageError('chromium_path flag must be set.') + if not FLAGS.source_file: + raise app.UsageError('source_file flag must be set.') + if not FLAGS.output_file: + raise app.UsageError('output_file flag must be set.') + + with open(FLAGS.source_file) as file: + value_text_proto = file.read() + + encoded = protoc_util.encode_proto(value_text_proto, ENCODE_NAMESPACE, + FLAGS.chromium_path, + COMPONENT_FEED_PROTO_PATH) + with open(FLAGS.output_file, 'wb') as file: + file.write(encoded) + + +if __name__ == '__main__': + app.run(main) diff --git a/chromium/components/feed/tools/protoc_util.py b/chromium/components/feed/tools/protoc_util.py new file mode 100755 index 00000000000..0ff20f2c0b0 --- /dev/null +++ b/chromium/components/feed/tools/protoc_util.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Lint as: python3 +"""The tools provides lot of protoc related helper functions.""" + +import glob +import os +import subprocess + +_protoc_path = None + + +def run_command(args, input): + """Uses subprocess to execute the command line args.""" + proc = subprocess.run( + args, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True) + return proc.stdout + + +def get_protoc_common_args(root_dir, proto_path): + """Returns a list of protoc common args as a list.""" + result = [ + '-I' + os.path.join(root_dir, 'third_party/feed_library/src'), + '-I' + os.path.join(root_dir) + ] + for root, _, files in os.walk(os.path.join(root_dir, proto_path)): + result += [os.path.join(root, f) for f in files if f.endswith('.proto')] + + return result + + +def encode_proto(text, message_name, root_dir, proto_path): + """Calls a command line to encode the text string and returns binary bytes.""" + return run_command([protoc_path(root_dir), '--encode=' + message_name] + + get_protoc_common_args(root_dir, proto_path), + text.encode()) + + +def decode_proto(data, message_name, root_dir, proto_path): + """Calls a command line to decode the binary bytes array into text string.""" + return run_command([protoc_path(root_dir), '--decode=' + message_name + ] + get_protoc_common_args(root_dir, proto_path), + data).decode('utf-8') + + +def protoc_path(root_dir): + """Returns the path to the proto compiler, protoc.""" + global _protoc_path + if not _protoc_path: + protoc_list = list( + glob.glob(os.path.join(root_dir, "out") + "/*/protoc")) + list( + glob.glob(os.path.join(root_dir, "out") + "/*/*/protoc")) + if not len(protoc_list): + print("Can't find a suitable build output directory", + "(it should have protoc)") + sys.exit(1) + _protoc_path = protoc_list[0] + return _protoc_path |