diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2019-08-30 10:22:43 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2019-08-30 12:36:28 +0000 |
commit | 271a6c3487a14599023a9106329505597638d793 (patch) | |
tree | e040d58ffc86c1480b79ca8528020ca9ec919bf8 /chromium/components/ukm | |
parent | 7b2ffa587235a47d4094787d72f38102089f402a (diff) | |
download | qtwebengine-chromium-271a6c3487a14599023a9106329505597638d793.tar.gz |
BASELINE: Update Chromium to 77.0.3865.59
Change-Id: I1e89a5f3b009a9519a6705102ad65c92fe736f21
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/components/ukm')
-rw-r--r-- | chromium/components/ukm/OWNERS | 1 | ||||
-rw-r--r-- | chromium/components/ukm/content/source_url_recorder.cc | 33 | ||||
-rw-r--r-- | chromium/components/ukm/debug/OWNERS | 2 | ||||
-rw-r--r-- | chromium/components/ukm/observers/sync_disable_observer.cc | 4 | ||||
-rw-r--r-- | chromium/components/ukm/test_ukm_recorder.cc | 2 | ||||
-rw-r--r-- | chromium/components/ukm/test_ukm_recorder.h | 2 | ||||
-rw-r--r-- | chromium/components/ukm/ukm_recorder_impl.cc | 139 | ||||
-rw-r--r-- | chromium/components/ukm/ukm_recorder_impl.h | 15 | ||||
-rw-r--r-- | chromium/components/ukm/ukm_service.cc | 3 | ||||
-rw-r--r-- | chromium/components/ukm/ukm_service.h | 2 | ||||
-rw-r--r-- | chromium/components/ukm/ukm_service_unittest.cc | 339 |
11 files changed, 318 insertions, 224 deletions
diff --git a/chromium/components/ukm/OWNERS b/chromium/components/ukm/OWNERS index 684cb163c06..f466cff997c 100644 --- a/chromium/components/ukm/OWNERS +++ b/chromium/components/ukm/OWNERS @@ -1,3 +1,4 @@ file://base/metrics/OWNERS # COMPONENT: Internals>Metrics>UKM +# TEAM: chromium-dev@chromium.org diff --git a/chromium/components/ukm/content/source_url_recorder.cc b/chromium/components/ukm/content/source_url_recorder.cc index 7b24ecff7e4..7c5facc6b9c 100644 --- a/chromium/components/ukm/content/source_url_recorder.cc +++ b/chromium/components/ukm/content/source_url_recorder.cc @@ -73,8 +73,10 @@ class SourceUrlRecorderWebContentsObserver ui::PageTransition transition, bool started_from_context_menu, bool renderer_initiated) override; + void WebContentsDestroyed() override; ukm::SourceId GetLastCommittedSourceId() const; + ukm::SourceId GetLastCommittedFullNavigationOrSameDocumentSourceId() const; // blink::mojom::UkmSourceIdFrameHost void SetDocumentSourceId(int64_t source_id) override; @@ -97,7 +99,7 @@ class SourceUrlRecorderWebContentsObserver void MaybeRecordUrl(content::NavigationHandle* navigation_handle, const GURL& initial_url); - // Recieves document source IDs from the renderer. + // Receives document source IDs from the renderer. content::WebContentsFrameBindingSet<blink::mojom::UkmSourceIdFrameHost> bindings_; @@ -186,6 +188,17 @@ void SourceUrlRecorderWebContentsObserver::DidFinishNavigation( return; } + // Inform the UKM recorder that the previous source is no longer needed to + // be kept alive in memory since we had navigated away. In case of same- + // document navigation, a new source id would have been created similarly to + // full-navigation, thus we are marking the last committed source id + // regardless of which case it came from. + ukm::DelegatingUkmRecorder* ukm_recorder = ukm::DelegatingUkmRecorder::Get(); + if (ukm_recorder) { + ukm_recorder->MarkSourceForDeletion( + GetLastCommittedFullNavigationOrSameDocumentSourceId()); + } + if (navigation_handle->IsSameDocument()) { DCHECK(it == pending_navigations_.end()); HandleSameDocumentNavigation(navigation_handle); @@ -266,11 +279,29 @@ void SourceUrlRecorderWebContentsObserver::DidOpenRequestedURL( new_recorder->opener_source_id_ = GetLastCommittedSourceId(); } +void SourceUrlRecorderWebContentsObserver::WebContentsDestroyed() { + // Inform the UKM recorder that the previous source is no longer needed to + // be kept alive in memory since the tab has been closed or discarded. In case + // of same-document navigation, a new source id would have been created + // similarly to full-navigation, thus we are marking the last committed source + // id regardless of which case it came from. + ukm::DelegatingUkmRecorder* ukm_recorder = ukm::DelegatingUkmRecorder::Get(); + if (ukm_recorder) { + ukm_recorder->MarkSourceForDeletion( + GetLastCommittedFullNavigationOrSameDocumentSourceId()); + } +} + ukm::SourceId SourceUrlRecorderWebContentsObserver::GetLastCommittedSourceId() const { return last_committed_full_navigation_source_id_; } +ukm::SourceId SourceUrlRecorderWebContentsObserver:: + GetLastCommittedFullNavigationOrSameDocumentSourceId() const { + return last_committed_full_navigation_or_same_document_source_id_; +} + void SourceUrlRecorderWebContentsObserver::SetDocumentSourceId( int64_t source_id) { content::RenderFrameHost* main_frame = web_contents()->GetMainFrame(); diff --git a/chromium/components/ukm/debug/OWNERS b/chromium/components/ukm/debug/OWNERS new file mode 100644 index 00000000000..c05db2e16e1 --- /dev/null +++ b/chromium/components/ukm/debug/OWNERS @@ -0,0 +1,2 @@ +# For trivial or mechanical horizontal JS/CSS/HTML changes. +file://ui/webui/PLATFORM_OWNERS diff --git a/chromium/components/ukm/observers/sync_disable_observer.cc b/chromium/components/ukm/observers/sync_disable_observer.cc index 4b3c05490f0..f8e428ac146 100644 --- a/chromium/components/ukm/observers/sync_disable_observer.cc +++ b/chromium/components/ukm/observers/sync_disable_observer.cc @@ -215,7 +215,7 @@ void SyncDisableObserver::OnUrlKeyedDataCollectionConsentStateChanged( void SyncDisableObserver::UpdateSyncState( syncer::SyncService* sync, UrlKeyedDataCollectionConsentHelper* consent_helper) { - DCHECK(base::ContainsKey(previous_states_, sync)); + DCHECK(base::Contains(previous_states_, sync)); const SyncDisableObserver::SyncState& previous_state = previous_states_[sync]; DCHECK(previous_state.anonymized_data_collection_state == DataCollectionState::kIgnored || @@ -250,7 +250,7 @@ void SyncDisableObserver::UpdateSyncState( } void SyncDisableObserver::OnSyncShutdown(syncer::SyncService* sync) { - DCHECK(base::ContainsKey(previous_states_, sync)); + DCHECK(base::Contains(previous_states_, sync)); auto found = consent_helpers_.find(sync); if (found != consent_helpers_.end()) { found->second->RemoveObserver(this); diff --git a/chromium/components/ukm/test_ukm_recorder.cc b/chromium/components/ukm/test_ukm_recorder.cc index b5267ef0a46..9286bdc0248 100644 --- a/chromium/components/ukm/test_ukm_recorder.cc +++ b/chromium/components/ukm/test_ukm_recorder.cc @@ -156,7 +156,7 @@ void TestUkmRecorder::ExpectEntryMetric(const mojom::UkmEntry* entry, EXPECT_EQ(expected_value, *metric) << " for metric:" << metric_name; } -TestAutoSetUkmRecorder::TestAutoSetUkmRecorder() : self_ptr_factory_(this) { +TestAutoSetUkmRecorder::TestAutoSetUkmRecorder() { DelegatingUkmRecorder::Get()->AddDelegate(self_ptr_factory_.GetWeakPtr()); } diff --git a/chromium/components/ukm/test_ukm_recorder.h b/chromium/components/ukm/test_ukm_recorder.h index 58d684b2d68..15f87aa5f5b 100644 --- a/chromium/components/ukm/test_ukm_recorder.h +++ b/chromium/components/ukm/test_ukm_recorder.h @@ -99,7 +99,7 @@ class TestAutoSetUkmRecorder : public TestUkmRecorder { ~TestAutoSetUkmRecorder() override; private: - base::WeakPtrFactory<TestAutoSetUkmRecorder> self_ptr_factory_; + base::WeakPtrFactory<TestAutoSetUkmRecorder> self_ptr_factory_{this}; }; } // namespace ukm diff --git a/chromium/components/ukm/ukm_recorder_impl.cc b/chromium/components/ukm/ukm_recorder_impl.cc index cbcb5351674..4e265f9b44d 100644 --- a/chromium/components/ukm/ukm_recorder_impl.cc +++ b/chromium/components/ukm/ukm_recorder_impl.cc @@ -64,8 +64,8 @@ size_t GetMaxSources() { kUkmFeature, "MaxSources", kDefaultMaxSources)); } -// Gets the maximum number of unreferenced Sources kept after purging sources -// that were added to the log. +// Gets the maximum number of Sources we can kept in memory to defer to the next +// reporting interval at the end of the current reporting cycle. size_t GetMaxKeptSources() { constexpr size_t kDefaultMaxKeptSources = 100; return static_cast<size_t>(base::GetFieldTrialParamByFeatureAsInt( @@ -258,6 +258,12 @@ void UkmRecorderImpl::Purge() { recording_is_continuous_ = false; } +void UkmRecorderImpl::MarkSourceForDeletion(SourceId source_id) { + if (source_id == kInvalidSourceId) + return; + recordings_.obsolete_source_ids.insert(source_id); +} + void UkmRecorderImpl::SetIsWebstoreExtensionCallback( const IsWebstoreExtensionCallback& callback) { is_webstore_extension_callback_ = callback; @@ -266,23 +272,35 @@ void UkmRecorderImpl::SetIsWebstoreExtensionCallback( void UkmRecorderImpl::StoreRecordingsInReport(Report* report) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - std::set<SourceId> ids_seen; + // Set of source ids seen by entries in recordings_. + std::set<SourceId> source_ids_seen; for (const auto& entry : recordings_.entries) { Entry* proto_entry = report->add_entries(); StoreEntryProto(*entry, proto_entry); - ids_seen.insert(entry->source_id); + source_ids_seen.insert(entry->source_id); } + // Number of sources excluded from this report because no entries referred to + // them. + const int num_sources_unsent = + recordings_.sources.size() - source_ids_seen.size(); + // Construct set of whitelisted URLs by merging those carried over from the + // previous report cycle and those from sources recorded in this cycle. std::unordered_set<std::string> url_whitelist; recordings_.carryover_urls_whitelist.swap(url_whitelist); AppendWhitelistedUrls(recordings_.sources, &url_whitelist); - std::vector<std::unique_ptr<UkmSource>> unsent_sources; - int unmatched_sources = 0; + // Number of sources discarded due to not matching a navigation URL. + int num_sources_unmatched = 0; std::unordered_map<ukm::SourceIdType, int> serialized_source_type_counts; - for (auto& kv : recordings_.sources) { + + for (const auto& kv : recordings_.sources) { + // Sources of non-navigation types will not be kept after current report. + if (GetSourceIdType(kv.first) != base::UkmSourceId::Type::NAVIGATION_ID) { + recordings_.obsolete_source_ids.insert(kv.first); + } // If the source id is not whitelisted, don't send it unless it has - // associated entries and the URL matches a URL of a whitelisted source. + // associated entries and the URL matches that of a whitelisted source. // Note: If ShouldRestrictToWhitelistedSourceIds() is true, this logic will // not be hit as the source would have already been filtered in // UpdateSourceURL(). @@ -291,11 +309,12 @@ void UkmRecorderImpl::StoreRecordingsInReport(Report* report) { DCHECK_EQ(1u, kv.second->urls().size()); if (!url_whitelist.count(kv.second->url().spec())) { RecordDroppedSource(DroppedDataReason::NOT_MATCHED); - unmatched_sources++; + MarkSourceForDeletion(kv.first); + num_sources_unmatched++; continue; } - if (!base::ContainsKey(ids_seen, kv.first)) { - unsent_sources.push_back(std::move(kv.second)); + // Omit entryless sources from the report. + if (!base::Contains(source_ids_seen, kv.first)) { continue; } } @@ -353,9 +372,9 @@ void UkmRecorderImpl::StoreRecordingsInReport(Report* report) { UMA_HISTOGRAM_COUNTS_100000("UKM.Entries.SerializedCount2", recordings_.entries.size()); UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.UnsentSourcesCount", - unsent_sources.size()); + num_sources_unsent); UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.UnmatchedSourcesCount", - unmatched_sources); + num_sources_unmatched); UMA_HISTOGRAM_COUNTS_1000( "UKM.Sources.SerializedCount2.Ukm", @@ -367,16 +386,23 @@ void UkmRecorderImpl::StoreRecordingsInReport(Report* report) { "UKM.Sources.SerializedCount2.App", serialized_source_type_counts[ukm::SourceIdType::APP_ID]); + // For each matching id in obsolete_source_ids, remove the Source from + // recordings_.sources. The remaining sources form the deferred sources for + // the next report. + for (const SourceId& source_id : recordings_.obsolete_source_ids) { + recordings_.sources.erase(source_id); + } + recordings_.obsolete_source_ids.clear(); + + // Populate SourceCounts field on the report then clear the recordings. Report::SourceCounts* source_counts_proto = report->mutable_source_counts(); source_counts_proto->set_observed(recordings_.source_counts.observed); source_counts_proto->set_navigation_sources( recordings_.source_counts.navigation_sources); - source_counts_proto->set_unmatched_sources(unmatched_sources); - source_counts_proto->set_deferred_sources(unsent_sources.size()); + source_counts_proto->set_unmatched_sources(num_sources_unmatched); source_counts_proto->set_carryover_sources( recordings_.source_counts.carryover_sources); - recordings_.sources.clear(); recordings_.source_counts.Reset(); recordings_.entries.clear(); recordings_.event_aggregations.clear(); @@ -384,29 +410,36 @@ void UkmRecorderImpl::StoreRecordingsInReport(Report* report) { report->set_is_continuous(recording_is_continuous_); recording_is_continuous_ = true; - // Keep at most |max_kept_sources|, prioritizing most-recent entries (by - // creation time). - const size_t max_kept_sources = GetMaxKeptSources(); - if (unsent_sources.size() > max_kept_sources) { - std::nth_element(unsent_sources.begin(), - unsent_sources.begin() + max_kept_sources, - unsent_sources.end(), - [](const std::unique_ptr<ukm::UkmSource>& lhs, - const std::unique_ptr<ukm::UkmSource>& rhs) { - return lhs->creation_time() > rhs->creation_time(); - }); - unsent_sources.resize(max_kept_sources); - } + // Defer at most GetMaxKeptSources() sources to the next report, + // prioritizing most recently created ones. + int pruned_sources_age = PruneOldSources(GetMaxKeptSources()); + // Record how old the newest truncated source is. + source_counts_proto->set_pruned_sources_age_seconds(pruned_sources_age); - for (auto& source : unsent_sources) { - // We already matched these sources against the URL whitelist. - // Re-whitelist them for the next report. - recordings_.carryover_urls_whitelist.insert(source->url().spec()); - recordings_.sources.emplace(source->id(), std::move(source)); + // Set deferred sources count after pruning. + source_counts_proto->set_deferred_sources(recordings_.sources.size()); + // Same value as the deferred source count, for setting the carryover count in + // the next reporting cycle. + recordings_.source_counts.carryover_sources = recordings_.sources.size(); + + // We already matched these deferred sources against the URL whitelist. + // Re-whitelist them for the next report. + for (const auto& kv : recordings_.sources) { + recordings_.carryover_urls_whitelist.insert(kv.second->url().spec()); } + UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.KeptSourcesCount", recordings_.sources.size()); - recordings_.source_counts.carryover_sources = recordings_.sources.size(); + + // Record number of sources after pruning that were carried over due to not + // having any events in this reporting cycle. + int num_sources_entryless = 0; + for (const auto& kv : recordings_.sources) { + if (!base::Contains(source_ids_seen, kv.first)) { + num_sources_entryless++; + } + } + source_counts_proto->set_entryless_sources(num_sources_entryless); } bool UkmRecorderImpl::ShouldRestrictToWhitelistedSourceIds() const { @@ -418,11 +451,39 @@ bool UkmRecorderImpl::ShouldRestrictToWhitelistedEntries() const { return true; } +int UkmRecorderImpl::PruneOldSources(size_t max_kept_sources) { + if (recordings_.sources.size() <= max_kept_sources) + return 0; + + std::vector<std::pair<base::TimeTicks, ukm::SourceId>> + timestamp_source_id_pairs; + for (const auto& kv : recordings_.sources) { + timestamp_source_id_pairs.push_back( + std::pair<base::TimeTicks, ukm::SourceId>(kv.second->creation_time(), + kv.first)); + } + // Partially sort so that the last |max_kept_sources| elements are the + // newest. + std::nth_element(timestamp_source_id_pairs.begin(), + timestamp_source_id_pairs.end() - max_kept_sources, + timestamp_source_id_pairs.end()); + + for (auto kv = timestamp_source_id_pairs.begin(); + kv != timestamp_source_id_pairs.end() - max_kept_sources; ++kv) { + recordings_.sources.erase(kv->second); + } + + base::TimeDelta pruned_sources_age = + base::TimeTicks::Now() - + (timestamp_source_id_pairs.end() - (max_kept_sources + 1))->first; + return pruned_sources_age.InSeconds(); +} + void UkmRecorderImpl::UpdateSourceURL(SourceId source_id, const GURL& unsanitized_url) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - if (base::ContainsKey(recordings_.sources, source_id)) + if (base::Contains(recordings_.sources, source_id)) return; const GURL sanitized_url = SanitizeURL(unsanitized_url); @@ -444,8 +505,8 @@ void UkmRecorderImpl::RecordNavigation( SourceId source_id, const UkmSource::NavigationData& unsanitized_navigation_data) { DCHECK(GetSourceIdType(source_id) == SourceIdType::NAVIGATION_ID); - DCHECK(!base::ContainsKey(recordings_.sources, source_id)); - // TODO(csharrison): Consider changing this behavior so the Source isn't event + DCHECK(!base::Contains(recordings_.sources, source_id)); + // TODO(csharrison): Consider changing this behavior so the Source isn't even // recorded at all if the final URL in |unsanitized_navigation_data| should // not be recorded. std::vector<GURL> urls; @@ -542,7 +603,7 @@ void UkmRecorderImpl::AddEntry(mojom::UkmEntryPtr entry) { } if (ShouldRestrictToWhitelistedEntries() && - !base::ContainsKey(whitelisted_entry_hashes_, entry->event_hash)) { + !base::Contains(whitelisted_entry_hashes_, entry->event_hash)) { RecordDroppedEntry(DroppedDataReason::NOT_WHITELISTED); event_aggregate.dropped_due_to_whitelist++; for (auto& metric : entry->metrics) diff --git a/chromium/components/ukm/ukm_recorder_impl.h b/chromium/components/ukm/ukm_recorder_impl.h index 65039d8df95..83f9dabd157 100644 --- a/chromium/components/ukm/ukm_recorder_impl.h +++ b/chromium/components/ukm/ukm_recorder_impl.h @@ -19,6 +19,7 @@ #include "base/strings/string_piece.h" #include "services/metrics/public/cpp/ukm_decode.h" #include "services/metrics/public/cpp/ukm_recorder.h" +#include "services/metrics/public/cpp/ukm_source_id.h" #include "services/metrics/public/mojom/ukm_interface.mojom-forward.h" namespace metrics { @@ -67,6 +68,11 @@ class UkmRecorderImpl : public UkmRecorder { // Deletes stored recordings. void Purge(); + // Marks a source as no longer needed to be kept alive in memory. The source + // with given id will be removed from in-memory recordings at the next + // reporting cycle. + void MarkSourceForDeletion(ukm::SourceId source_id) override; + // Sets a callback for determining if an extension URL can be recorded. void SetIsWebstoreExtensionCallback( const IsWebstoreExtensionCallback& callback); @@ -100,6 +106,11 @@ class UkmRecorderImpl : public UkmRecorder { return recordings_.entries; } + // Keep only newest |max_kept_sources| sources when the number of sources + // in recordings_ exceeds this threshold. Returns the age of newest truncated + // source in seconds. + int PruneOldSources(size_t max_kept_sources); + // UkmRecorder: void AddEntry(mojom::UkmEntryPtr entry) override; void UpdateSourceURL(SourceId source_id, const GURL& url) override; @@ -194,6 +205,10 @@ class UkmRecorderImpl : public UkmRecorder { // Data captured by AddEntry(). std::vector<mojom::UkmEntryPtr> entries; + // Source ids that have been marked as no longer needed, to denote the + // subset of |sources| that can be purged after next report. + std::unordered_set<ukm::SourceId> obsolete_source_ids; + // URLs of sources that matched a whitelist url, but were not included in // the report generated by the last log rotation because we haven't seen any // events for that source yet. diff --git a/chromium/components/ukm/ukm_service.cc b/chromium/components/ukm/ukm_service.cc index c16c04c9429..66025adce8a 100644 --- a/chromium/components/ukm/ukm_service.cc +++ b/chromium/components/ukm/ukm_service.cc @@ -84,8 +84,7 @@ UkmService::UkmService(PrefService* pref_service, client_(client), reporting_service_(client, pref_service), initialize_started_(false), - initialize_complete_(false), - self_ptr_factory_(this) { + initialize_complete_(false) { DCHECK(pref_service_); DCHECK(client_); DVLOG(1) << "UkmService::Constructor"; diff --git a/chromium/components/ukm/ukm_service.h b/chromium/components/ukm/ukm_service.h index 87ac70f9f1f..067aac66c2a 100644 --- a/chromium/components/ukm/ukm_service.h +++ b/chromium/components/ukm/ukm_service.h @@ -156,7 +156,7 @@ class UkmService : public UkmRecorderImpl { // Weak pointers factory used to post task on different threads. All weak // pointers managed by this factory have the same lifetime as UkmService. - base::WeakPtrFactory<UkmService> self_ptr_factory_; + base::WeakPtrFactory<UkmService> self_ptr_factory_{this}; DISALLOW_COPY_AND_ASSIGN(UkmService); }; diff --git a/chromium/components/ukm/ukm_service_unittest.cc b/chromium/components/ukm/ukm_service_unittest.cc index b4e1e2d8fe4..5a7dd66d0bf 100644 --- a/chromium/components/ukm/ukm_service_unittest.cc +++ b/chromium/components/ukm/ukm_service_unittest.cc @@ -23,9 +23,10 @@ #include "components/metrics/test_metrics_provider.h" #include "components/metrics/test_metrics_service_client.h" #include "components/prefs/testing_pref_service.h" -#include "components/ukm/unsent_log_store_metrics_impl.h" #include "components/ukm/ukm_pref_names.h" -#include "components/variations/variations_associated_data.h" +#include "components/ukm/ukm_recorder_impl.h" +#include "components/ukm/ukm_service.h" +#include "components/ukm/unsent_log_store_metrics_impl.h" #include "services/metrics/public/cpp/ukm_builders.h" #include "services/metrics/public/cpp/ukm_entry_builder.h" #include "services/metrics/public/cpp/ukm_source.h" @@ -66,6 +67,10 @@ class TestRecordingHelper { recorder_->RecordNavigation(source_id, navigation_data); } + void MarkSourceForDeletion(SourceId source_id) { + recorder_->MarkSourceForDeletion(source_id); + } + private: UkmRecorder* recorder_; @@ -78,44 +83,14 @@ bool TestIsWebstoreExtension(base::StringPiece id) { return (id == "bhcnanendmgjjeghamaccjnochlnhcgj"); } -// TODO(rkaplow): consider making this a generic testing class in -// components/variations. class ScopedUkmFeatureParams { public: - ScopedUkmFeatureParams( - base::FeatureList::OverrideState feature_state, - const std::map<std::string, std::string>& variation_params) { - static const char kTestFieldTrialName[] = "TestTrial"; - static const char kTestExperimentGroupName[] = "TestGroup"; - - variations::testing::ClearAllVariationParams(); - - EXPECT_TRUE(variations::AssociateVariationParams( - kTestFieldTrialName, kTestExperimentGroupName, variation_params)); - - base::FieldTrial* field_trial = base::FieldTrialList::CreateFieldTrial( - kTestFieldTrialName, kTestExperimentGroupName); - - std::unique_ptr<base::FeatureList> feature_list(new base::FeatureList); - feature_list->RegisterFieldTrialOverride(kUkmFeature.name, feature_state, - field_trial); - - // Since we are adding a scoped feature list after browser start, copy over - // the existing feature list to prevent inconsistency. - base::FeatureList* existing_feature_list = base::FeatureList::GetInstance(); - if (existing_feature_list) { - std::string enabled_features; - std::string disabled_features; - base::FeatureList::GetInstance()->GetFeatureOverrides(&enabled_features, - &disabled_features); - feature_list->InitializeFromCommandLine(enabled_features, - disabled_features); - } - - scoped_feature_list_.InitWithFeatureList(std::move(feature_list)); + explicit ScopedUkmFeatureParams(const base::FieldTrialParams& params) { + scoped_feature_list_.InitAndEnableFeatureWithParameters(kUkmFeature, + params); } - ~ScopedUkmFeatureParams() { variations::testing::ClearAllVariationParams(); } + ~ScopedUkmFeatureParams() {} private: base::test::ScopedFeatureList scoped_feature_list_; @@ -212,8 +187,7 @@ TEST_F(UkmServiceTest, EnableDisableSchedule) { TEST_F(UkmServiceTest, PersistAndPurge) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); + ScopedUkmFeatureParams params({{"WhitelistEntries", Entry1And2Whitelist()}}); UkmService service(&prefs_, &client_, true /* restrict_to_whitelisted_entries */); @@ -290,8 +264,7 @@ TEST_F(UkmServiceTest, SourceSerialization) { TEST_F(UkmServiceTest, AddEntryWithEmptyMetrics) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); + ScopedUkmFeatureParams params({{"WhitelistEntries", Entry1And2Whitelist()}}); UkmService service(&prefs_, &client_, true /* restrict_to_whitelisted_entries */); @@ -314,8 +287,7 @@ TEST_F(UkmServiceTest, AddEntryWithEmptyMetrics) { TEST_F(UkmServiceTest, MetricsProviderTest) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); + ScopedUkmFeatureParams params({{"WhitelistEntries", Entry1And2Whitelist()}}); UkmService service(&prefs_, &client_, true /* restrict_to_whitelisted_entries */); @@ -392,8 +364,7 @@ TEST_F(UkmServiceTest, LogsRotation) { TEST_F(UkmServiceTest, LogsUploadedOnlyWhenHavingSourcesOrEntries) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); // Testing two whitelisted Entries. - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); + ScopedUkmFeatureParams params({{"WhitelistEntries", Entry1And2Whitelist()}}); UkmService service(&prefs_, &client_, true /* restrict_to_whitelisted_entries */); @@ -423,11 +394,13 @@ TEST_F(UkmServiceTest, LogsUploadedOnlyWhenHavingSourcesOrEntries) { recorder.UpdateSourceURL(id, GURL("https://google.com/foobar")); TestEvent1(id).Record(&service); + // Do not keep the source in the recorder after the current log. + recorder.MarkSourceForDeletion(id); // Includes a Source and an Entry, so will persist. service.Flush(); EXPECT_EQ(GetPersistedLogCount(), 3); - // Current log has no Sources. + // The recorder contains no Sources or Entries thus will not create a new log. service.Flush(); EXPECT_EQ(GetPersistedLogCount(), 3); } @@ -477,7 +450,6 @@ TEST_F(UkmServiceTest, RestrictToWhitelistedSourceIds) { for (bool restrict_to_whitelisted_source_ids : {true, false}) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); ScopedUkmFeatureParams params( - base::FeatureList::OVERRIDE_ENABLE_FEATURE, {{"RestrictToWhitelistedSourceIds", restrict_to_whitelisted_source_ids ? "true" : "false"}, {"WhitelistEntries", Entry1And2Whitelist()}}); @@ -549,8 +521,7 @@ TEST_F(UkmServiceTest, RecordSessionId) { TEST_F(UkmServiceTest, SourceSize) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); // Set a threshold of number of Sources via Feature Params. - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"MaxSources", "2"}}); + ScopedUkmFeatureParams params({{"MaxSources", "2"}}); ClearPrefs(); UkmService service(&prefs_, &client_, @@ -604,8 +575,7 @@ TEST_F(UkmServiceTest, PurgeMidUpload) { TEST_F(UkmServiceTest, WhitelistEntryTest) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); // Testing two whitelisted Entries. - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); + ScopedUkmFeatureParams params({{"WhitelistEntries", Entry1And2Whitelist()}}); ClearPrefs(); UkmService service(&prefs_, &client_, @@ -676,7 +646,6 @@ TEST_F(UkmServiceTest, UnreferencedNonWhitelistedSources) { base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); // Set a threshold of number of Sources via Feature Params. ScopedUkmFeatureParams params( - base::FeatureList::OVERRIDE_ENABLE_FEATURE, {{"MaxKeptSources", "3"}, {"WhitelistEntries", Entry1And2Whitelist()}, {"RestrictToWhitelistedSourceIds", @@ -722,19 +691,28 @@ TEST_F(UkmServiceTest, UnreferencedNonWhitelistedSources) { EXPECT_EQ(1, GetPersistedLogCount()); auto proto_report = GetPersistedReport(); + // The non-whitelisted source should only be recorded if we aren't + // restricted to whitelisted source ids. if (restrict_to_whitelisted_source_ids) { + // Only the one whitelisted source (whitelisted_id) is recorded. EXPECT_EQ(1, proto_report.source_counts().observed()); + // The one whitelisted source is of navigation type. EXPECT_EQ(1, proto_report.source_counts().navigation_sources()); EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(0, proto_report.source_counts().deferred_sources()); + // The one whitelisted source is also deferred for inclusion in future + // reports. + EXPECT_EQ(1, proto_report.source_counts().deferred_sources()); EXPECT_EQ(0, proto_report.source_counts().carryover_sources()); ASSERT_EQ(1, proto_report.sources_size()); } else { + // 1 whitelisted source and 6 non-whitelisted source. EXPECT_EQ(7, proto_report.source_counts().observed()); + // The one whitelisted source is of navigation type. EXPECT_EQ(1, proto_report.source_counts().navigation_sources()); EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(4, proto_report.source_counts().deferred_sources()); + // Only the navigation type source is deferred. + EXPECT_EQ(1, proto_report.source_counts().deferred_sources()); EXPECT_EQ(0, proto_report.source_counts().carryover_sources()); ASSERT_EQ(3, proto_report.sources_size()); @@ -743,7 +721,6 @@ TEST_F(UkmServiceTest, UnreferencedNonWhitelistedSources) { EXPECT_EQ(ids[2], proto_report.sources(1).id()); EXPECT_EQ(kURL.spec(), proto_report.sources(1).url()); } - // Since MaxKeptSources is 3, only Sources 5, 4, 3 should be retained. // Log entries under 0, 1, 3 and 4. Log them in reverse order - which // shouldn't affect source ordering in the output. @@ -759,35 +736,42 @@ TEST_F(UkmServiceTest, UnreferencedNonWhitelistedSources) { EXPECT_EQ(2, GetPersistedLogCount()); proto_report = GetPersistedReport(); + // The non-whitelisted source should only be recorded if we aren't + // restricted to whitelisted source ids. if (restrict_to_whitelisted_source_ids) { EXPECT_EQ(0, proto_report.source_counts().observed()); EXPECT_EQ(0, proto_report.source_counts().navigation_sources()); EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(0, proto_report.source_counts().deferred_sources()); - EXPECT_EQ(0, proto_report.source_counts().carryover_sources()); - - ASSERT_EQ(0, proto_report.sources_size()); + // The one whitelisted source is deferred again in future reports. + EXPECT_EQ(1, proto_report.source_counts().deferred_sources()); + // Number of sources carried over from the previous report to this report. + EXPECT_EQ(1, proto_report.source_counts().carryover_sources()); + // Only the navigation source is again included in current report and + // there is a new entry associated to it. + ASSERT_EQ(1, proto_report.sources_size()); } else { EXPECT_EQ(0, proto_report.source_counts().observed()); EXPECT_EQ(0, proto_report.source_counts().navigation_sources()); EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); + // Only the navigation type source is deferred. EXPECT_EQ(1, proto_report.source_counts().deferred_sources()); - EXPECT_EQ(3, proto_report.source_counts().carryover_sources()); - - ASSERT_EQ(2, proto_report.sources_size()); - EXPECT_EQ(ids[3], proto_report.sources(0).id()); + // Number of sources carried over from the previous report to this report. + EXPECT_EQ(1, proto_report.source_counts().carryover_sources()); + // Only the navigation source is again included in current report and + // there is a new entry associated to it. + ASSERT_EQ(1, proto_report.sources_size()); + EXPECT_EQ(whitelisted_id, proto_report.sources(0).id()); EXPECT_EQ(kURL.spec(), proto_report.sources(0).url()); - EXPECT_EQ(ids[4], proto_report.sources(1).id()); - EXPECT_EQ(kURL.spec(), proto_report.sources(1).url()); } } } TEST_F(UkmServiceTest, NonWhitelistedUrls) { + // URL to be manually whitelisted using whitelisted source type. const GURL kURL("https://google.com/foobar"); struct { GURL url; - bool expected_kept; + bool expect_in_report; } test_cases[] = { {GURL("https://google.com/foobar"), true}, // For origin-only URLs, only the origin needs to be matched. @@ -797,8 +781,7 @@ TEST_F(UkmServiceTest, NonWhitelistedUrls) { }; base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); + ScopedUkmFeatureParams params({{"WhitelistEntries", Entry1And2Whitelist()}}); for (const auto& test : test_cases) { ClearPrefs(); @@ -816,7 +799,7 @@ TEST_F(UkmServiceTest, NonWhitelistedUrls) { ukm::SourceId whitelist_id = GetWhitelistedSourceId(1); recorder.UpdateSourceURL(whitelist_id, kURL); - // Record non whitelisted ID with a entry. + // Record non whitelisted ID with an entry. ukm::SourceId nonwhitelist_id = GetNonWhitelistedSourceId(100); recorder.UpdateSourceURL(nonwhitelist_id, test.url); TestEvent1(nonwhitelist_id).Record(&service); @@ -827,7 +810,10 @@ TEST_F(UkmServiceTest, NonWhitelistedUrls) { EXPECT_EQ(2, proto_report.source_counts().observed()); EXPECT_EQ(1, proto_report.source_counts().navigation_sources()); - if (test.expected_kept) { + + // If the source id is not whitelisted, don't send it unless it has + // associated entries and the URL matches that of the whitelisted source. + if (test.expect_in_report) { EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); ASSERT_EQ(2, proto_report.sources_size()); EXPECT_EQ(whitelist_id, proto_report.sources(0).id()); @@ -840,122 +826,32 @@ TEST_F(UkmServiceTest, NonWhitelistedUrls) { EXPECT_EQ(whitelist_id, proto_report.sources(0).id()); EXPECT_EQ(kURL, proto_report.sources(0).url()); } - } -} - -TEST_F(UkmServiceTest, NonWhitelistedCarryoverUrls) { - const GURL kURL("https://google.com/foobar"); - struct { - // Source1 is recorded during the first rotation with no entry. - // An entry for it is recorded in the second rotation. - GURL source1_url; - // Should Source1 be seen in second rotation's log. - bool expect_source1; - // Source2 is recorded during the second rotation with an entry. - GURL source2_url; - // Should Source2 be seen in second rotation's log. - bool expect_source2; - } test_cases[] = { - // Recording the URL captures in the whitelist, which will also allow - // exact matches of the same URL. - {GURL("https://google.com/foobar"), true, - GURL("https://google.com/foobar"), true}, - // Capturing a full URL shouldn't allow origin matches. - {GURL("https://google.com/foobar"), true, GURL("https://google.com"), - false}, - // Uncaptured URLs won't get matched. - {GURL("https://google.com/foobar"), true, GURL("https://other.com"), - false}, - // Origin should be capturable, and will remember the same origin. - {GURL("https://google.com"), true, GURL("https://google.com"), true}, - // If the origin is captured, only the origin is remembered. - {GURL("https://google.com"), true, GURL("https://google.com/foobar"), - false}, - // Uncaptured URLs won't get matched. - {GURL("https://google.com"), true, GURL("https://other.com"), false}, - // If the URL isn't captured in the first round, it won't capture later. - {GURL("https://other.com"), false, GURL("https://google.com/foobar"), - false}, - {GURL("https://other.com"), false, GURL("https://google.com"), false}, - // Entries shouldn't whitelist themselves. - {GURL("https://other.com"), false, GURL("https://other.com"), false}, - }; - - base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, - {{"WhitelistEntries", Entry1And2Whitelist()}}); - - for (const auto& test : test_cases) { - ClearPrefs(); - UkmService service(&prefs_, &client_, - true /* restrict_to_whitelisted_entries */); - TestRecordingHelper recorder(&service); - - EXPECT_EQ(GetPersistedLogCount(), 0); - service.Initialize(); - task_runner_->RunUntilIdle(); - service.EnableRecording(/*extensions=*/false); - service.EnableReporting(); - - // Record with whitelisted ID to whitelist the URL. - ukm::SourceId whitelist_id = GetWhitelistedSourceId(1); - recorder.UpdateSourceURL(whitelist_id, kURL); - - // Record test Source1 without an event. - ukm::SourceId nonwhitelist_id1 = GetNonWhitelistedSourceId(100); - recorder.UpdateSourceURL(nonwhitelist_id1, test.source1_url); - - service.Flush(); - ASSERT_EQ(1, GetPersistedLogCount()); - auto proto_report = GetPersistedReport(); - - EXPECT_EQ(2, proto_report.source_counts().observed()); - EXPECT_EQ(1, proto_report.source_counts().navigation_sources()); - EXPECT_EQ(0, proto_report.source_counts().carryover_sources()); - if (test.expect_source1) { - EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(1, proto_report.source_counts().deferred_sources()); - } else { - EXPECT_EQ(1, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(0, proto_report.source_counts().deferred_sources()); - } - ASSERT_EQ(1, proto_report.sources_size()); - EXPECT_EQ(whitelist_id, proto_report.sources(0).id()); - EXPECT_EQ(kURL, proto_report.sources(0).url()); - - // Record the Source2 and events for Source1 and Source2. + // Do a log rotation again, with the same test URL associated to a new + // source id. Since the previous source id of the test case is of + // non-whitelisted type, the carryover URLs list is expected to remain + // be unchanged, thus the the report should still contain the same numbers + // of sources as before, that is, non-whitelisted URLs should not have + // whitelisted themselves during the previous log rotation. ukm::SourceId nonwhitelist_id2 = GetNonWhitelistedSourceId(101); - recorder.UpdateSourceURL(nonwhitelist_id2, test.source2_url); - TestEvent1(nonwhitelist_id1).Record(&service); + recorder.UpdateSourceURL(nonwhitelist_id2, test.url); TestEvent1(nonwhitelist_id2).Record(&service); - service.Flush(); ASSERT_EQ(2, GetPersistedLogCount()); proto_report = GetPersistedReport(); - EXPECT_EQ(1, proto_report.source_counts().observed()); - EXPECT_EQ(0, proto_report.source_counts().navigation_sources()); - EXPECT_EQ(0, proto_report.source_counts().deferred_sources()); - if (!test.expect_source1) { - EXPECT_FALSE(test.expect_source2); - EXPECT_EQ(1, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(0, proto_report.source_counts().carryover_sources()); - ASSERT_EQ(0, proto_report.sources_size()); - } else if (!test.expect_source2) { - EXPECT_EQ(1, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(1, proto_report.source_counts().carryover_sources()); - ASSERT_EQ(1, proto_report.sources_size()); - EXPECT_EQ(nonwhitelist_id1, proto_report.sources(0).id()); - EXPECT_EQ(test.source1_url, proto_report.sources(0).url()); - } else { + if (test.expect_in_report) { EXPECT_EQ(0, proto_report.source_counts().unmatched_sources()); - EXPECT_EQ(1, proto_report.source_counts().carryover_sources()); ASSERT_EQ(2, proto_report.sources_size()); - EXPECT_EQ(nonwhitelist_id1, proto_report.sources(0).id()); - EXPECT_EQ(test.source1_url, proto_report.sources(0).url()); + EXPECT_EQ(whitelist_id, proto_report.sources(0).id()); + EXPECT_EQ(kURL, proto_report.sources(0).url()); EXPECT_EQ(nonwhitelist_id2, proto_report.sources(1).id()); - EXPECT_EQ(test.source2_url, proto_report.sources(1).url()); + EXPECT_EQ(test.url, proto_report.sources(1).url()); + } else { + EXPECT_EQ(1, proto_report.source_counts().unmatched_sources()); + ASSERT_EQ(1, proto_report.sources_size()); + EXPECT_EQ(whitelist_id, proto_report.sources(0).id()); + EXPECT_EQ(kURL, proto_report.sources(0).url()); } } } @@ -980,7 +876,7 @@ TEST_F(UkmServiceTest, SupportedSchemes) { }; base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, {}); + ScopedUkmFeatureParams params({}); UkmService service(&prefs_, &client_, true /* restrict_to_whitelisted_entries */); TestRecordingHelper recorder(&service); @@ -1039,7 +935,7 @@ TEST_F(UkmServiceTest, SupportedSchemesNoExtensions) { }; base::FieldTrialList field_trial_list(nullptr /* entropy_provider */); - ScopedUkmFeatureParams params(base::FeatureList::OVERRIDE_ENABLE_FEATURE, {}); + ScopedUkmFeatureParams params({}); UkmService service(&prefs_, &client_, true /* restrict_to_whitelisted_entries */); TestRecordingHelper recorder(&service); @@ -1142,4 +1038,93 @@ TEST_F(UkmServiceTest, SanitizeChromeUrlParams) { } } +TEST_F(UkmServiceTest, MarkSourceForDeletion) { + UkmService service(&prefs_, &client_, + true /* restrict_to_whitelist_entries */); + TestRecordingHelper recorder(&service); + EXPECT_EQ(0, GetPersistedLogCount()); + service.Initialize(); + task_runner_->RunUntilIdle(); + service.EnableRecording(/*extensions=*/false); + service.EnableReporting(); + + // Seed some dummy sources. + SourceId id0 = GetWhitelistedSourceId(0); + recorder.UpdateSourceURL(id0, GURL("https://www.example0.com/")); + SourceId id1 = GetWhitelistedSourceId(1); + recorder.UpdateSourceURL(id1, GURL("https://www.example1.com/")); + SourceId id2 = GetWhitelistedSourceId(2); + recorder.UpdateSourceURL(id2, GURL("https://www.example2.com/")); + + service.Flush(); + int logs_count = 0; + EXPECT_EQ(++logs_count, GetPersistedLogCount()); + + // All sources are present in the report. + Report proto_report = GetPersistedReport(); + ASSERT_EQ(3, proto_report.sources_size()); + EXPECT_EQ(id0, proto_report.sources(0).id()); + EXPECT_EQ(id1, proto_report.sources(1).id()); + EXPECT_EQ(id2, proto_report.sources(2).id()); + + // Mark source 1 for deletion. Next report will still contain source 1 because + // we might have associated entries. It will no longer be in further report at + // the following cycle. + service.MarkSourceForDeletion(id1); + service.Flush(); + EXPECT_EQ(++logs_count, GetPersistedLogCount()); + + proto_report = GetPersistedReport(); + ASSERT_EQ(3, proto_report.sources_size()); + + service.Flush(); + EXPECT_EQ(++logs_count, GetPersistedLogCount()); + + proto_report = GetPersistedReport(); + ASSERT_EQ(2, proto_report.sources_size()); + EXPECT_EQ(id0, proto_report.sources(0).id()); + EXPECT_EQ(id2, proto_report.sources(1).id()); +} + +TEST_F(UkmServiceTest, PurgeNonNavigationSources) { + UkmService service(&prefs_, &client_, + true /* restrict_to_whitelist_entries */); + TestRecordingHelper recorder(&service); + EXPECT_EQ(0, GetPersistedLogCount()); + service.Initialize(); + task_runner_->RunUntilIdle(); + service.EnableRecording(/*extensions=*/false); + service.EnableReporting(); + + // Seed some dummy sources. + SourceId id0 = ConvertToSourceId(0, SourceIdType::UKM); + recorder.UpdateSourceURL(id0, GURL("https://www.example0.com/")); + SourceId id1 = ConvertToSourceId(1, SourceIdType::NAVIGATION_ID); + recorder.UpdateSourceURL(id1, GURL("https://www.example1.com/")); + SourceId id2 = ConvertToSourceId(2, SourceIdType::APP_ID); + recorder.UpdateSourceURL(id2, GURL("https://www.example2.com/")); + SourceId id3 = ConvertToSourceId(3, SourceIdType::HISTORY_ID); + recorder.UpdateSourceURL(id3, GURL("https://www.example3.com/")); + + service.Flush(); + int logs_count = 0; + EXPECT_EQ(++logs_count, GetPersistedLogCount()); + + // All sources are present except id0 of non-whitelisted UKM type. + Report proto_report = GetPersistedReport(); + ASSERT_EQ(3, proto_report.sources_size()); + EXPECT_EQ(id1, proto_report.sources(0).id()); + EXPECT_EQ(id2, proto_report.sources(1).id()); + EXPECT_EQ(id3, proto_report.sources(2).id()); + + service.Flush(); + EXPECT_EQ(++logs_count, GetPersistedLogCount()); + + // Sources of APP_ID and HISTORY_ID types are not kept between reporting + // cycles, thus only 1 navigation type source remains. + proto_report = GetPersistedReport(); + ASSERT_EQ(1, proto_report.sources_size()); + EXPECT_EQ(id1, proto_report.sources(0).id()); +} + } // namespace ukm |