diff options
Diffstat (limited to 'chromium/components/data_reduction_proxy')
62 files changed, 1941 insertions, 634 deletions
diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc index 40e848405f2..10f161ec10d 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc @@ -105,7 +105,7 @@ bool DataReductionProxyBypassProtocol::MaybeBypassProxyAndPrepareToRetry( DataReductionProxyTypeInfo data_reduction_proxy_type_info; if (!config_->WasDataReductionProxyUsed(request, &data_reduction_proxy_type_info)) { - if (!HasDataReductionProxyViaHeader(response_headers, nullptr)) { + if (!HasDataReductionProxyViaHeader(*response_headers, nullptr)) { ReportResponseProxyServerStatusHistogram( RESPONSE_PROXY_SERVER_STATUS_NON_DRP_NO_VIA); return false; @@ -138,12 +138,12 @@ bool DataReductionProxyBypassProtocol::MaybeBypassProxyAndPrepareToRetry( // At this point, the response is expected to have the data reduction proxy // via header, so detect and report cases where the via header is missing. DataReductionProxyBypassStats::DetectAndRecordMissingViaHeaderResponseCode( - data_reduction_proxy_type_info.proxy_index == 0, response_headers); + data_reduction_proxy_type_info.proxy_index == 0, *response_headers); // GetDataReductionProxyBypassType will only log a net_log event if a bypass // command was sent via the data reduction proxy headers DataReductionProxyBypassType bypass_type = GetDataReductionProxyBypassType( - response_headers, data_reduction_proxy_info); + request->url_chain(), *response_headers, data_reduction_proxy_info); if (proxy_bypass_type) *proxy_bypass_type = bypass_type; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.cc index 3aa6b85d42d..718c1f234b0 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.cc @@ -100,8 +100,8 @@ void DataReductionProxyBypassStats::RecordDataReductionProxyBypassInfo( // static void DataReductionProxyBypassStats::DetectAndRecordMissingViaHeaderResponseCode( bool is_primary, - const net::HttpResponseHeaders* headers) { - if (HasDataReductionProxyViaHeader(headers, NULL)) { + const net::HttpResponseHeaders& headers) { + if (HasDataReductionProxyViaHeader(headers, nullptr)) { // The data reduction proxy via header is present, so don't record anything. return; } @@ -109,11 +109,11 @@ void DataReductionProxyBypassStats::DetectAndRecordMissingViaHeaderResponseCode( if (is_primary) { UMA_HISTOGRAM_SPARSE_SLOWLY( "DataReductionProxy.MissingViaHeader.ResponseCode.Primary", - headers->response_code()); + headers.response_code()); } else { UMA_HISTOGRAM_SPARSE_SLOWLY( "DataReductionProxy.MissingViaHeader.ResponseCode.Fallback", - headers->response_code()); + headers.response_code()); } } @@ -385,7 +385,7 @@ void DataReductionProxyBypassStats::RecordMissingViaHeaderBytes( if (!data_reduction_proxy_config_->WasDataReductionProxyUsed(&request, NULL) || - HasDataReductionProxyViaHeader(request.response_headers(), NULL)) { + HasDataReductionProxyViaHeader(*request.response_headers(), NULL)) { // Only track requests that used the data reduction proxy and had responses // that were missing the data reduction proxy via header. return; @@ -537,6 +537,11 @@ void DataReductionProxyBypassStats::RecordBypassedBytes( "Status503HttpServiceUnavailable", content_length); break; + case BYPASS_EVENT_TYPE_URL_REDIRECT_CYCLE: + UMA_HISTOGRAM_COUNTS( + "DataReductionProxy.BypassedBytes.URLRedirectCycle", + content_length); + break; default: break; } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.h index 22674e9f1aa..222e2246172 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.h @@ -45,7 +45,7 @@ class DataReductionProxyBypassStats // proxy via header is not present. static void DetectAndRecordMissingViaHeaderResponseCode( bool is_primary, - const net::HttpResponseHeaders* headers); + const net::HttpResponseHeaders& headers); // |config| outlives this class instance. |unreachable_callback| provides a // hook to inform the user that the Data Reduction Proxy is unreachable. diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc index 064d0aa45e2..cd7ddb4139b 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc @@ -335,6 +335,63 @@ class DataReductionProxyBypassStatsEndToEndTest : public testing::Test { return request; } + // Create and execute a fake request that goes through a redirect loop using + // the data reduction proxy stack. + std::unique_ptr<net::URLRequest> CreateAndExecuteURLRedirectCycleRequest() { + MockRead redirect_mock_reads_1[] = { + MockRead("HTTP/1.1 302 Found\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Location: http://bar.com/\r\n\r\n"), + MockRead(""), MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_1( + redirect_mock_reads_1, arraysize(redirect_mock_reads_1), nullptr, 0); + mock_socket_factory_.AddSocketDataProvider( + &redirect_socket_data_provider_1); + + // The response after the redirect comes through proxy. + MockRead redirect_mock_reads_2[] = { + MockRead("HTTP/1.1 302 Found\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Location: http://foo.com/\r\n\r\n"), + MockRead(""), MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_2( + redirect_mock_reads_2, arraysize(redirect_mock_reads_2), nullptr, 0); + mock_socket_factory_.AddSocketDataProvider( + &redirect_socket_data_provider_2); + + // The response after the redirect comes through proxy and there is a + // redirect cycle. + MockRead redirect_mock_reads_3[] = { + MockRead("HTTP/1.1 302 Found\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Location: http://bar.com/\r\n\r\n"), + MockRead(""), MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_3( + redirect_mock_reads_3, arraysize(redirect_mock_reads_3), nullptr, 0); + mock_socket_factory_.AddSocketDataProvider( + &redirect_socket_data_provider_3); + + // Data reduction proxy should be bypassed, and the response should come + // directly. + MockRead response_mock_reads[] = { + MockRead("HTTP/1.1 200 OK\r\n\r\n"), MockRead(kBody.c_str()), + MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider response_socket_data_provider( + response_mock_reads, arraysize(response_mock_reads), nullptr, 0); + mock_socket_factory_.AddSocketDataProvider(&response_socket_data_provider); + + std::unique_ptr<net::URLRequest> request( + context_.CreateRequest(GURL("http://foo.com"), net::IDLE, &delegate_)); + request->set_method("GET"); + request->Start(); + drp_test_context_->RunUntilIdle(); + return request; + } + void set_proxy_service(net::ProxyService* proxy_service) { context_.set_proxy_service(proxy_service); } @@ -397,6 +454,7 @@ class DataReductionProxyBypassStatsEndToEndTest : public testing::Test { "DataReductionProxy.BypassedBytes.Status502HttpBadGateway", "DataReductionProxy.BypassedBytes.Status503HttpServiceUnavailable", "DataReductionProxy.BypassedBytes.NetworkErrorOther", + "DataReductionProxy.BypassedBytes.RedirectCycle", }; for (const std::string& histogram : kHistograms) { @@ -508,6 +566,31 @@ TEST_F(DataReductionProxyBypassStatsEndToEndTest, BypassedBytesNoRetry) { } } +// Verify that when there is a URL redirect cycle, data reduction proxy is +// bypassed for a single request. +TEST_F(DataReductionProxyBypassStatsEndToEndTest, URLRedirectCycle) { + InitializeContext(); + ClearBadProxies(); + base::HistogramTester histogram_tester_1; + CreateAndExecuteURLRedirectCycleRequest(); + + histogram_tester_1.ExpectUniqueSample( + "DataReductionProxy.BypassedBytes.URLRedirectCycle", kBody.size(), 1); + ExpectOtherBypassedBytesHistogramsEmpty( + histogram_tester_1, "DataReductionProxy.BypassedBytes.URLRedirectCycle"); + + // The second request should be sent via the proxy. + base::HistogramTester histogram_tester_2; + CreateAndExecuteRequest(GURL("http://bar.com"), net::LOAD_NORMAL, net::OK, + "HTTP/1.1 200 OK\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n\r\n", + kNextBody.c_str(), nullptr, nullptr); + histogram_tester_2.ExpectUniqueSample( + "DataReductionProxy.BypassedBytes.NotBypassed", kNextBody.size(), 1); + ExpectOtherBypassedBytesHistogramsEmpty( + histogram_tester_2, "DataReductionProxy.BypassedBytes.NotBypassed"); +} + TEST_F(DataReductionProxyBypassStatsEndToEndTest, BypassedBytesProxyOverridden) { std::unique_ptr<net::ProxyService> proxy_service( @@ -684,8 +767,10 @@ TEST_F(DataReductionProxyBypassStatsEndToEndTest, kErrorBody.c_str(), "HTTP/1.1 200 OK\r\n\r\n", kBody.c_str()); - histogram_tester.ExpectUniqueSample( - "DataReductionProxy.ConfigService.HTTPRequests", 1, 1); + EXPECT_LT( + 0u, histogram_tester + .GetAllSamples("DataReductionProxy.ConfigService.HTTPRequests") + .size()); // The first request caused the proxy to be marked as bad, so this second // request should not come through the proxy. @@ -701,10 +786,8 @@ TEST_F(DataReductionProxyBypassStatsEndToEndTest, ExpectOtherBypassedBytesHistogramsEmpty(histogram_tester, test_case.histogram_name); - // "DataReductionProxy.ConfigService.HTTPRequests" should not be recorded - // for bypassed requests. - histogram_tester.ExpectUniqueSample( - "DataReductionProxy.ConfigService.HTTPRequests", 1, 1); + histogram_tester.ExpectBucketCount( + "DataReductionProxy.ConfigService.HTTPRequests", 0, 0); } } @@ -874,7 +957,7 @@ TEST_F(DataReductionProxyBypassStatsEndToEndTest, new net::HttpResponseHeaders(raw_headers)); DataReductionProxyBypassStats::DetectAndRecordMissingViaHeaderResponseCode( - test_cases[i].is_primary, headers.get()); + test_cases[i].is_primary, *headers); if (test_cases[i].expected_primary_sample == -1) { histogram_tester.ExpectTotalCount(kPrimaryHistogramName, 0); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc index 5d4b87c9187..04280b08bc7 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc @@ -9,6 +9,7 @@ #include "base/bind.h" #include "base/command_line.h" +#include "base/feature_list.h" #include "base/location.h" #include "base/logging.h" #include "base/memory/ptr_util.h" @@ -20,6 +21,7 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h" #include "components/data_reduction_proxy/core/browser/data_usage_store.h" #include "components/data_reduction_proxy/core/browser/data_use_group.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_pref_names.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h" #include "components/data_reduction_proxy/proto/data_store.pb.h" @@ -76,7 +78,7 @@ void AddInt64ToListPref(size_t index, int64_t length, base::ListValue* list_update) { int64_t value = GetInt64PrefValue(*list_update, index) + length; - list_update->Set(index, new base::StringValue(base::Int64ToString(value))); + list_update->Set(index, new base::Value(base::Int64ToString(value))); } // DailyContentLengthUpdate maintains a data saving pref. The pref is a list @@ -367,6 +369,16 @@ void DataReductionProxyCompressionStats::Init() { base::Bind( &DataReductionProxyCompressionStats::OnDataUsageReportingPrefChanged, weak_factory_.GetWeakPtr())); + +#if defined(OS_ANDROID) + if (!base::FeatureList::IsEnabled(features::kDataReductionSiteBreakdown)) { + // If the user is moved out of the experiment make sure that data usage + // reporting is not enabled and the map is cleared. + SetDataUsageReportingEnabled(false); + DeleteHistoricalDataUsage(); + } +#endif + if (data_usage_reporting_enabled_.GetValue()) { current_data_usage_load_status_ = LOADING; service_->LoadCurrentDataUsageBucket(base::Bind( @@ -424,20 +436,6 @@ void DataReductionProxyCompressionStats::Init() { InitListPref(prefs::kDailyOriginalContentLengthWithDataReductionProxyEnabled); } -void DataReductionProxyCompressionStats::UpdateDataSavings( - const std::string& data_usage_host, - int64_t data_used, - int64_t original_size) { - DCHECK(thread_checker_.CalledOnValidThread()); - // Data is recorded at the URLRequest level, so an update should only change - // the original size amount by the savings amount. - int64_t update_to_original_size = original_size - data_used; - int64_t update_to_data_used = 0; - RecordData(update_to_data_used, update_to_original_size, - true /* data_saver_enabled */, UPDATE, data_usage_host, - std::string()); -} - void DataReductionProxyCompressionStats::UpdateContentLengths( int64_t data_used, int64_t original_size, @@ -695,9 +693,56 @@ void DataReductionProxyCompressionStats::OnCurrentDataUsageLoaded( current_data_usage_load_status_ = LOADED; } +void DataReductionProxyCompressionStats::SetDataUsageReportingEnabled( + bool enabled) { + DCHECK(thread_checker_.CalledOnValidThread()); + if (data_usage_reporting_enabled_.GetValue() != enabled) { + data_usage_reporting_enabled_.SetValue(enabled); + OnDataUsageReportingPrefChanged(); + } +} + void DataReductionProxyCompressionStats::ClearDataSavingStatistics() { DeleteHistoricalDataUsage(); + pref_service_->ClearPref(prefs::kDailyHttpContentLengthLastUpdateDate); + pref_service_->ClearPref(prefs::kHttpReceivedContentLength); + pref_service_->ClearPref(prefs::kHttpOriginalContentLength); + + pref_service_->ClearPref(prefs::kDailyHttpOriginalContentLengthApplication); + pref_service_->ClearPref(prefs::kDailyHttpOriginalContentLengthVideo); + pref_service_->ClearPref(prefs::kDailyHttpOriginalContentLengthUnknown); + pref_service_->ClearPref(prefs::kDailyHttpReceivedContentLengthApplication); + pref_service_->ClearPref(prefs::kDailyHttpReceivedContentLengthVideo); + pref_service_->ClearPref(prefs::kDailyHttpReceivedContentLengthUnknown); + + pref_service_->ClearPref( + prefs::kDailyOriginalContentLengthViaDataReductionProxyApplication); + pref_service_->ClearPref( + prefs::kDailyOriginalContentLengthViaDataReductionProxyVideo); + pref_service_->ClearPref( + prefs::kDailyOriginalContentLengthViaDataReductionProxyUnknown); + pref_service_->ClearPref( + prefs::kDailyContentLengthViaDataReductionProxyApplication); + pref_service_->ClearPref( + prefs::kDailyContentLengthViaDataReductionProxyVideo); + pref_service_->ClearPref( + prefs::kDailyContentLengthViaDataReductionProxyUnknown); + + pref_service_->ClearPref( + prefs:: + kDailyOriginalContentLengthWithDataReductionProxyEnabledApplication); + pref_service_->ClearPref( + prefs::kDailyOriginalContentLengthWithDataReductionProxyEnabledVideo); + pref_service_->ClearPref( + prefs::kDailyOriginalContentLengthWithDataReductionProxyEnabledUnknown); + pref_service_->ClearPref( + prefs::kDailyContentLengthWithDataReductionProxyEnabledApplication); + pref_service_->ClearPref( + prefs::kDailyContentLengthWithDataReductionProxyEnabledVideo); + pref_service_->ClearPref( + prefs::kDailyContentLengthWithDataReductionProxyEnabledUnknown); + pref_service_->ClearPref( prefs::kDailyContentLengthHttpsWithDataReductionProxyEnabled); pref_service_->ClearPref( @@ -1209,6 +1254,7 @@ void DataReductionProxyCompressionStats::DeleteHistoricalDataUsage() { void DataReductionProxyCompressionStats::GetHistoricalDataUsageImpl( const HistoricalDataUsageCallback& get_data_usage_callback, const base::Time& now) { +#if !defined(OS_ANDROID) if (current_data_usage_load_status_ != LOADED) { // If current data usage has not yet loaded, we return an empty array. The // extension can retry after a slight delay. @@ -1218,10 +1264,13 @@ void DataReductionProxyCompressionStats::GetHistoricalDataUsageImpl( base::MakeUnique<std::vector<DataUsageBucket>>()); return; } +#endif - PersistDataUsage(); + if (current_data_usage_load_status_ == LOADED) + PersistDataUsage(); - if (!DataUsageStore::AreInSameInterval(data_usage_map_last_updated_, now)) { + if (!data_usage_map_last_updated_.is_null() && + !DataUsageStore::AreInSameInterval(data_usage_map_last_updated_, now)) { data_usage_map_.clear(); data_usage_map_last_updated_ = base::Time(); @@ -1243,7 +1292,19 @@ void DataReductionProxyCompressionStats::OnDataUsageReportingPrefChanged() { weak_factory_.GetWeakPtr())); } } else { +// Don't delete the historical data on Android, but clear the map. +#if defined(OS_ANDROID) + DCHECK(current_data_usage_load_status_ != LOADING); + + if (current_data_usage_load_status_ == LOADED) + PersistDataUsage(); + + data_usage_map_.clear(); + data_usage_map_last_updated_ = base::Time(); + data_usage_map_is_dirty_ = false; +#else DeleteHistoricalDataUsage(); +#endif current_data_usage_load_status_ = NOT_LOADED; } } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h index 60c00200f0d..b3cf152cc31 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h @@ -22,7 +22,6 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.h" #include "components/data_reduction_proxy/core/browser/db_data_owner.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_pref_names.h" -#include "components/data_reduction_proxy/core/common/data_savings_recorder.h" #include "components/prefs/pref_member.h" class PrefService; @@ -62,13 +61,6 @@ class DataReductionProxyCompressionStats { const base::TimeDelta& delay); ~DataReductionProxyCompressionStats(); - // Records detailed data usage broken down by connection type and domain. - // Assumes that the |data_used| has been recoreded by previous calls to - // UpdateContentLengths. - void UpdateDataSavings(const std::string& data_usage_host, - int64_t data_used, - int64_t original_size); - // Records detailed data usage broken down by connection type and domain. Also // records daily data savings statistics to prefs and reports data savings // UMA. |compressed_size| and |original_size| are measured in bytes. @@ -133,6 +125,11 @@ class DataReductionProxyCompressionStats { // for the last stored interval. void OnCurrentDataUsageLoaded(std::unique_ptr<DataUsageBucket> data_usage); + // Sets the value of |prefs::kDataUsageReportingEnabled| to |enabled|. + // Initializes data usage statistics in memory when pref is enabled and + // persists data usage to memory when pref is disabled. + void SetDataUsageReportingEnabled(bool enabled); + private: // Enum to track the state of loading data usage from storage. enum CurrentDataUsageLoadStatus { NOT_LOADED = 0, LOADING = 1, LOADED = 2 }; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc index cadbcd3cda7..e9a5338c84c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc @@ -163,13 +163,13 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { for (size_t i = 0; i < kNumDaysInHistory; ++i) { original_daily_content_length_list->Set( - i, new base::StringValue(base::SizeTToString(i))); + i, new base::Value(base::SizeTToString(i))); } received_daily_content_length_list->Clear(); for (size_t i = 0; i < kNumDaysInHistory / 2; ++i) { received_daily_content_length_list->Set( - i, new base::StringValue(base::SizeTToString(i))); + i, new base::Value(base::SizeTToString(i))); } } @@ -178,8 +178,7 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { base::ListValue* update = compression_stats_->GetList(pref); update->Clear(); for (size_t i = 0; i < kNumDaysInHistory; ++i) { - update->Insert( - 0, base::MakeUnique<base::StringValue>(base::Int64ToString(0))); + update->Insert(0, base::MakeUnique<base::Value>(base::Int64ToString(0))); } } @@ -227,17 +226,20 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { // Verify the pref list values are equal to the given values. // If the count of values is less than kNumDaysInHistory, zeros are assumed // at the beginning. - void VerifyPrefList(const char* pref, const int64_t* values, size_t count) { - ASSERT_GE(kNumDaysInHistory, count); + void VerifyPrefList(const char* pref, + const int64_t* values, + size_t count, + size_t num_days_in_history) { + ASSERT_GE(num_days_in_history, count); base::ListValue* update = compression_stats_->GetList(pref); - ASSERT_EQ(kNumDaysInHistory, update->GetSize()) << "Pref: " << pref; + ASSERT_EQ(num_days_in_history, update->GetSize()) << "Pref: " << pref; for (size_t i = 0; i < count; ++i) { EXPECT_EQ(values[i], - GetListPrefInt64Value(*update, kNumDaysInHistory - count + i)) - << pref << "; index=" << (kNumDaysInHistory - count + i); + GetListPrefInt64Value(*update, num_days_in_history - count + i)) + << pref << "; index=" << (num_days_in_history - count + i); } - for (size_t i = 0; i < kNumDaysInHistory - count; ++i) { + for (size_t i = 0; i < num_days_in_history - count; ++i) { EXPECT_EQ(0, GetListPrefInt64Value(*update, i)) << "index=" << i; } } @@ -260,31 +262,31 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { const int64_t* original_via_data_reduction_proxy_values, size_t original_via_data_reduction_proxy_count, const int64_t* received_via_data_reduction_proxy_values, - size_t received_via_data_reduction_proxy_count) { + size_t received_via_data_reduction_proxy_count, + size_t num_days_in_history) { VerifyPrefList(data_reduction_proxy::prefs::kDailyHttpOriginalContentLength, - original_values, original_count); + original_values, original_count, num_days_in_history); VerifyPrefList(data_reduction_proxy::prefs::kDailyHttpReceivedContentLength, - received_values, received_count); + received_values, received_count, num_days_in_history); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyOriginalContentLengthWithDataReductionProxyEnabled, + original_with_data_reduction_proxy_enabled_values, + original_with_data_reduction_proxy_enabled_count, + num_days_in_history); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyContentLengthWithDataReductionProxyEnabled, + received_with_data_reduction_proxy_enabled_values, + received_with_data_reduction_proxy_count, + num_days_in_history); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyOriginalContentLengthViaDataReductionProxy, + original_via_data_reduction_proxy_values, + original_via_data_reduction_proxy_count, + num_days_in_history); VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyOriginalContentLengthWithDataReductionProxyEnabled, - original_with_data_reduction_proxy_enabled_values, - original_with_data_reduction_proxy_enabled_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyContentLengthWithDataReductionProxyEnabled, - received_with_data_reduction_proxy_enabled_values, - received_with_data_reduction_proxy_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyOriginalContentLengthViaDataReductionProxy, - original_via_data_reduction_proxy_values, - original_via_data_reduction_proxy_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyContentLengthViaDataReductionProxy, + data_reduction_proxy::prefs::kDailyContentLengthViaDataReductionProxy, received_via_data_reduction_proxy_values, - received_via_data_reduction_proxy_count); + received_via_data_reduction_proxy_count, num_days_in_history); VerifyPrefInt64( data_reduction_proxy::prefs::kDailyHttpOriginalContentLengthApplication, @@ -342,39 +344,39 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { const int64_t* unknown_with_data_reduction_proxy_enabled_values, size_t unknown_with_data_reduction_proxy_enabled_count) { VerifyPrefList(data_reduction_proxy::prefs::kDailyHttpOriginalContentLength, - original_values, original_count); + original_values, original_count, kNumDaysInHistory); VerifyPrefList(data_reduction_proxy::prefs::kDailyHttpReceivedContentLength, - received_values, received_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyOriginalContentLengthWithDataReductionProxyEnabled, - original_with_data_reduction_proxy_enabled_values, - original_with_data_reduction_proxy_enabled_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyContentLengthWithDataReductionProxyEnabled, - received_with_data_reduction_proxy_enabled_values, - received_with_data_reduction_proxy_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyContentLengthHttpsWithDataReductionProxyEnabled, - https_with_data_reduction_proxy_enabled_values, - https_with_data_reduction_proxy_enabled_count); + received_values, received_count, kNumDaysInHistory); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyOriginalContentLengthWithDataReductionProxyEnabled, + original_with_data_reduction_proxy_enabled_values, + original_with_data_reduction_proxy_enabled_count, + kNumDaysInHistory); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyContentLengthWithDataReductionProxyEnabled, + received_with_data_reduction_proxy_enabled_values, + received_with_data_reduction_proxy_count, kNumDaysInHistory); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyContentLengthHttpsWithDataReductionProxyEnabled, + https_with_data_reduction_proxy_enabled_values, + https_with_data_reduction_proxy_enabled_count, + kNumDaysInHistory); VerifyPrefList( data_reduction_proxy::prefs:: kDailyContentLengthShortBypassWithDataReductionProxyEnabled, short_bypass_with_data_reduction_proxy_enabled_values, - short_bypass_with_data_reduction_proxy_enabled_count); + short_bypass_with_data_reduction_proxy_enabled_count, + kNumDaysInHistory); VerifyPrefList( data_reduction_proxy::prefs:: kDailyContentLengthLongBypassWithDataReductionProxyEnabled, long_bypass_with_data_reduction_proxy_enabled_values, - long_bypass_with_data_reduction_proxy_enabled_count); - VerifyPrefList( - data_reduction_proxy::prefs:: - kDailyContentLengthUnknownWithDataReductionProxyEnabled, - unknown_with_data_reduction_proxy_enabled_values, - unknown_with_data_reduction_proxy_enabled_count); + long_bypass_with_data_reduction_proxy_enabled_count, kNumDaysInHistory); + VerifyPrefList(data_reduction_proxy::prefs:: + kDailyContentLengthUnknownWithDataReductionProxyEnabled, + unknown_with_data_reduction_proxy_enabled_values, + unknown_with_data_reduction_proxy_enabled_count, + kNumDaysInHistory); } int64_t GetInt64(const char* pref_path) { @@ -433,6 +435,10 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { compression_stats_->DeleteHistoricalDataUsage(); } + void ClearDataSavingStatistics() { + compression_stats_->ClearDataSavingStatistics(); + } + void DeleteBrowsingHistory(const base::Time& start, const base::Time& end) { compression_stats_->DeleteBrowsingHistory(start, end); } @@ -544,8 +550,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, TEST_F(DataReductionProxyCompressionStatsTest, StatsRestoredOnOnRestart) { base::ListValue list_value; - list_value.Insert( - 0, base::MakeUnique<base::StringValue>(base::Int64ToString(1234))); + list_value.Insert(0, + base::MakeUnique<base::Value>(base::Int64ToString(1234))); pref_service()->Set(prefs::kDailyHttpOriginalContentLength, list_value); ResetCompressionStatsWithDelay( @@ -584,28 +590,6 @@ TEST_F(DataReductionProxyCompressionStatsTest, TotalLengths) { GetInt64(data_reduction_proxy::prefs::kHttpOriginalContentLength)); } -TEST_F(DataReductionProxyCompressionStatsTest, TotalLengthsUpdate) { - const int64_t kOriginalLength = 200; - const int64_t kReceivedLength = 100; - - compression_stats()->UpdateDataSavings(std::string(), kReceivedLength, - kOriginalLength); - - EXPECT_EQ(0, - GetInt64(data_reduction_proxy::prefs::kHttpReceivedContentLength)); - EXPECT_EQ(kOriginalLength - kReceivedLength, - GetInt64(data_reduction_proxy::prefs::kHttpOriginalContentLength)); - - // Record the same numbers again, and total lengths should be doubled. - compression_stats()->UpdateDataSavings(std::string(), kReceivedLength, - kOriginalLength); - - EXPECT_EQ(0, - GetInt64(data_reduction_proxy::prefs::kHttpReceivedContentLength)); - EXPECT_EQ(kOriginalLength * 2 - kReceivedLength * 2, - GetInt64(data_reduction_proxy::prefs::kHttpOriginalContentLength)); -} - TEST_F(DataReductionProxyCompressionStatsTest, OneResponse) { const int64_t kOriginalLength = 200; const int64_t kReceivedLength = 100; @@ -617,9 +601,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, OneResponse) { FakeNow()); VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original, 1, received, 1, - original, 1, received, 1); + original, 1, received, 1, original, 1, received, 1, original, 1, received, + 1, kNumDaysInHistory); } TEST_F(DataReductionProxyCompressionStatsTest, MultipleResponses) { @@ -629,9 +612,9 @@ TEST_F(DataReductionProxyCompressionStatsTest, MultipleResponses) { int64_t received[] = {kReceivedLength}; RecordContentLengthPrefs( kReceivedLength, kOriginalLength, false, UNKNOWN_TYPE, FakeNow()); - VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - NULL, 0, NULL, 0, NULL, 0, NULL, 0); + VerifyDailyDataSavingContentLengthPrefLists(original, 1, received, 1, NULL, 0, + NULL, 0, NULL, 0, NULL, 0, + kNumDaysInHistory); RecordContentLengthPrefs( kReceivedLength, kOriginalLength, true, UNKNOWN_TYPE, FakeNow()); @@ -640,9 +623,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, MultipleResponses) { int64_t original_proxy_enabled[] = {kOriginalLength}; int64_t received_proxy_enabled[] = {kReceivedLength}; VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original_proxy_enabled, 1, received_proxy_enabled, 1, - NULL, 0, NULL, 0); + original, 1, received, 1, original_proxy_enabled, 1, + received_proxy_enabled, 1, NULL, 0, NULL, 0, kNumDaysInHistory); RecordContentLengthPrefs( kReceivedLength, kOriginalLength, true, VIA_DATA_REDUCTION_PROXY, @@ -654,9 +636,9 @@ TEST_F(DataReductionProxyCompressionStatsTest, MultipleResponses) { int64_t original_via_proxy[] = {kOriginalLength}; int64_t received_via_proxy[] = {kReceivedLength}; VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original_proxy_enabled, 1, received_proxy_enabled, 1, - original_via_proxy, 1, received_via_proxy, 1); + original, 1, received, 1, original_proxy_enabled, 1, + received_proxy_enabled, 1, original_via_proxy, 1, received_via_proxy, 1, + kNumDaysInHistory); RecordContentLengthPrefs( kReceivedLength, kOriginalLength, true, UNKNOWN_TYPE, FakeNow()); @@ -665,18 +647,18 @@ TEST_F(DataReductionProxyCompressionStatsTest, MultipleResponses) { original_proxy_enabled[0] += kOriginalLength; received_proxy_enabled[0] += kReceivedLength; VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original_proxy_enabled, 1, received_proxy_enabled, 1, - original_via_proxy, 1, received_via_proxy, 1); + original, 1, received, 1, original_proxy_enabled, 1, + received_proxy_enabled, 1, original_via_proxy, 1, received_via_proxy, 1, + kNumDaysInHistory); RecordContentLengthPrefs( kReceivedLength, kOriginalLength, false, UNKNOWN_TYPE, FakeNow()); original[0] += kOriginalLength; received[0] += kReceivedLength; VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original_proxy_enabled, 1, received_proxy_enabled, 1, - original_via_proxy, 1, received_via_proxy, 1); + original, 1, received, 1, original_proxy_enabled, 1, + received_proxy_enabled, 1, original_via_proxy, 1, received_via_proxy, 1, + kNumDaysInHistory); } TEST_F(DataReductionProxyCompressionStatsTest, RequestType) { @@ -784,12 +766,10 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardOneDay) { int64_t original_via_data_reduction_proxy[] = {kOriginalLength, 0}; int64_t received_via_data_reduction_proxy[] = {kReceivedLength, 0}; VerifyDailyDataSavingContentLengthPrefLists( - original, 2, - received, 2, - original_with_data_reduction_proxy_enabled, 2, + original, 2, received, 2, original_with_data_reduction_proxy_enabled, 2, received_with_data_reduction_proxy_enabled, 2, - original_via_data_reduction_proxy, 2, - received_via_data_reduction_proxy, 2); + original_via_data_reduction_proxy, 2, received_via_data_reduction_proxy, + 2, kNumDaysInHistory); // Proxy enabled. Not via proxy. RecordContentLengthPrefs( @@ -799,12 +779,10 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardOneDay) { original_with_data_reduction_proxy_enabled[1] += kOriginalLength; received_with_data_reduction_proxy_enabled[1] += kReceivedLength; VerifyDailyDataSavingContentLengthPrefLists( - original, 2, - received, 2, - original_with_data_reduction_proxy_enabled, 2, + original, 2, received, 2, original_with_data_reduction_proxy_enabled, 2, received_with_data_reduction_proxy_enabled, 2, - original_via_data_reduction_proxy, 2, - received_via_data_reduction_proxy, 2); + original_via_data_reduction_proxy, 2, received_via_data_reduction_proxy, + 2, kNumDaysInHistory); // Proxy enabled and via proxy. RecordContentLengthPrefs( @@ -817,12 +795,10 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardOneDay) { original_via_data_reduction_proxy[1] += kOriginalLength; received_via_data_reduction_proxy[1] += kReceivedLength; VerifyDailyDataSavingContentLengthPrefLists( - original, 2, - received, 2, - original_with_data_reduction_proxy_enabled, 2, + original, 2, received, 2, original_with_data_reduction_proxy_enabled, 2, received_with_data_reduction_proxy_enabled, 2, - original_via_data_reduction_proxy, 2, - received_via_data_reduction_proxy, 2); + original_via_data_reduction_proxy, 2, received_via_data_reduction_proxy, + 2, kNumDaysInHistory); // Proxy enabled and via proxy, with content length greater than max int32_t. const int64_t kBigOriginalLength = 0x300000000LL; // 12G. @@ -839,7 +815,7 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardOneDay) { original, 2, received, 2, original_with_data_reduction_proxy_enabled, 2, received_with_data_reduction_proxy_enabled, 2, original_via_data_reduction_proxy, 2, received_via_data_reduction_proxy, - 2); + 2, kNumDaysInHistory); } TEST_F(DataReductionProxyCompressionStatsTest, PartialDayTimeChange) { @@ -852,9 +828,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, PartialDayTimeChange) { kReceivedLength, kOriginalLength, true, VIA_DATA_REDUCTION_PROXY, FakeNow()); VerifyDailyDataSavingContentLengthPrefLists( - original, 2, received, 2, - original, 2, received, 2, - original, 2, received, 2); + original, 2, received, 2, original, 2, received, 2, original, 2, received, + 2, kNumDaysInHistory); // Forward 10 hours, stay in the same day. // See kLastUpdateTime: "Now" in test is 03:45am. @@ -865,9 +840,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, PartialDayTimeChange) { original[1] += kOriginalLength; received[1] += kReceivedLength; VerifyDailyDataSavingContentLengthPrefLists( - original, 2, received, 2, - original, 2, received, 2, - original, 2, received, 2); + original, 2, received, 2, original, 2, received, 2, original, 2, received, + 2, kNumDaysInHistory); // Forward 11 more hours, comes to tomorrow. AddFakeTimeDeltaInHours(11); @@ -877,9 +851,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, PartialDayTimeChange) { int64_t original2[] = {kOriginalLength * 2, kOriginalLength}; int64_t received2[] = {kReceivedLength * 2, kReceivedLength}; VerifyDailyDataSavingContentLengthPrefLists( - original2, 2, received2, 2, - original2, 2, received2, 2, - original2, 2, received2, 2); + original2, 2, received2, 2, original2, 2, received2, 2, original2, 2, + received2, 2, kNumDaysInHistory); } TEST_F(DataReductionProxyCompressionStatsTest, ForwardMultipleDays) { @@ -904,9 +877,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardMultipleDays) { int64_t original[] = {kOriginalLength, 0, 0, kOriginalLength}; int64_t received[] = {kReceivedLength, 0, 0, kReceivedLength}; VerifyDailyDataSavingContentLengthPrefLists( - original, 4, received, 4, - original, 4, received, 4, - original, 4, received, 4); + original, 4, received, 4, original, 4, received, 4, original, 4, received, + 4, kNumDaysInHistory); // Forward four more days. AddFakeTimeDeltaInHours(4 * 24); @@ -920,9 +892,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardMultipleDays) { kReceivedLength, 0, 0, kReceivedLength, 0, 0, 0, kReceivedLength, }; VerifyDailyDataSavingContentLengthPrefLists( - original2, 8, received2, 8, - original2, 8, received2, 8, - original2, 8, received2, 8); + original2, 8, received2, 8, original2, 8, received2, 8, original2, 8, + received2, 8, kNumDaysInHistory); histogram_tester.ExpectUniqueSample( "DataReductionProxy.SavingsCleared.NegativeSystemClock", false, 3); @@ -934,9 +905,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardMultipleDays) { int64_t original3[] = {kOriginalLength}; int64_t received3[] = {kReceivedLength}; VerifyDailyDataSavingContentLengthPrefLists( - original3, 1, received3, 1, - original3, 1, received3, 1, - original3, 1, received3, 1); + original3, 1, received3, 1, original3, 1, received3, 1, original3, 1, + received3, 1, kNumDaysInHistory); histogram_tester.ExpectUniqueSample( "DataReductionProxy.SavingsCleared.NegativeSystemClock", false, 4); @@ -946,9 +916,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, ForwardMultipleDays) { kReceivedLength, kOriginalLength, true, VIA_DATA_REDUCTION_PROXY, FakeNow()); VerifyDailyDataSavingContentLengthPrefLists( - original3, 1, received3, 1, - original3, 1, received3, 1, - original3, 1, received3, 1); + original3, 1, received3, 1, original3, 1, received3, 1, original3, 1, + received3, 1, kNumDaysInHistory); histogram_tester.ExpectUniqueSample( "DataReductionProxy.SavingsCleared.NegativeSystemClock", false, 5); } @@ -974,9 +943,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, BackwardAndForwardOneDay) { original[0] += kOriginalLength; received[0] += kReceivedLength; VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original, 1, received, 1, - original, 1, received, 1); + original, 1, received, 1, original, 1, received, 1, original, 1, received, + 1, kNumDaysInHistory); histogram_tester.ExpectUniqueSample( "DataReductionProxy.SavingsCleared.NegativeSystemClock", false, 2); @@ -988,9 +956,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, BackwardAndForwardOneDay) { int64_t original2[] = {kOriginalLength * 2, kOriginalLength}; int64_t received2[] = {kReceivedLength * 2, kReceivedLength}; VerifyDailyDataSavingContentLengthPrefLists( - original2, 2, received2, 2, - original2, 2, received2, 2, - original2, 2, received2, 2); + original2, 2, received2, 2, original2, 2, received2, 2, original2, 2, + received2, 2, kNumDaysInHistory); histogram_tester.ExpectUniqueSample( "DataReductionProxy.SavingsCleared.NegativeSystemClock", false, 3); } @@ -1014,9 +981,8 @@ TEST_F(DataReductionProxyCompressionStatsTest, BackwardTwoDays) { kReceivedLength, kOriginalLength, true, VIA_DATA_REDUCTION_PROXY, FakeNow()); VerifyDailyDataSavingContentLengthPrefLists( - original, 1, received, 1, - original, 1, received, 1, - original, 1, received, 1); + original, 1, received, 1, original, 1, received, 1, original, 1, received, + 1, kNumDaysInHistory); histogram_tester.ExpectTotalCount( "DataReductionProxy.SavingsCleared.NegativeSystemClock", 2); histogram_tester.ExpectBucketCount( @@ -1143,6 +1109,7 @@ TEST_F(DataReductionProxyCompressionStatsTest, DisableDataUsageRecording) { DisableDataUsageReporting(); base::RunLoop().RunUntilIdle(); +#if !defined(OS_ANDROID) // Data usage on disk must be deleted. auto expected_data_usage1 = base::MakeUnique<std::vector<data_reduction_proxy::DataUsageBucket>>( @@ -1158,6 +1125,26 @@ TEST_F(DataReductionProxyCompressionStatsTest, DisableDataUsageRecording) { GetHistoricalDataUsage(base::Bind(&DataUsageLoadVerifier::OnLoadDataUsage, base::Unretained(&verifier2)), now); +#else + // For Android don't delete data usage. + auto expected_data_usage = + base::MakeUnique<std::vector<data_reduction_proxy::DataUsageBucket>>( + kNumExpectedBuckets); + data_reduction_proxy::PerConnectionDataUsage* connection_usage = + expected_data_usage->at(kNumExpectedBuckets - 1).add_connection_usage(); + data_reduction_proxy::PerSiteDataUsage* site_usage = + connection_usage->add_site_usage(); + site_usage->set_hostname("www.foo.com"); + site_usage->set_data_used(1000); + site_usage->set_original_size(1250); + + DataUsageLoadVerifier verifier(std::move(expected_data_usage)); + + GetHistoricalDataUsage(base::Bind(&DataUsageLoadVerifier::OnLoadDataUsage, + base::Unretained(&verifier)), + now); +#endif + base::RunLoop().RunUntilIdle(); } @@ -1341,4 +1328,45 @@ TEST_F(DataReductionProxyCompressionStatsTest, DeleteBrowsingHistory) { base::RunLoop().RunUntilIdle(); } +TEST_F(DataReductionProxyCompressionStatsTest, ClearDataSavingStatistics) { + EnableDataUsageReporting(); + base::RunLoop().RunUntilIdle(); + + base::Time now = base::Time::Now(); + base::Time fifteen_mins_ago = now - TimeDelta::FromMinutes(15); + // Fake record to be from 15 minutes ago so that it is flushed to storage. + RecordDataUsage("https://www.bar.com", 900, 1100, fifteen_mins_ago); + + RecordDataUsage("https://www.foo.com", 1000, 1250, now); + + const int64_t kOriginalLength = 200; + const int64_t kReceivedLength = 100; + int64_t original[] = {kOriginalLength}; + int64_t received[] = {kReceivedLength}; + + RecordContentLengthPrefs(kReceivedLength, kOriginalLength, true, + VIA_DATA_REDUCTION_PROXY, FakeNow()); + + VerifyDailyDataSavingContentLengthPrefLists( + original, 1, received, 1, original, 1, received, 1, original, 1, received, + 1, kNumDaysInHistory); + + ClearDataSavingStatistics(); + base::RunLoop().RunUntilIdle(); + + auto expected_data_usage = + base::MakeUnique<std::vector<data_reduction_proxy::DataUsageBucket>>( + kNumExpectedBuckets); + DataUsageLoadVerifier verifier(std::move(expected_data_usage)); + + GetHistoricalDataUsage(base::Bind(&DataUsageLoadVerifier::OnLoadDataUsage, + base::Unretained(&verifier)), + now); + base::RunLoop().RunUntilIdle(); + + VerifyDailyDataSavingContentLengthPrefLists(nullptr, 0, nullptr, 0, nullptr, + 0, nullptr, 0, nullptr, 0, + nullptr, 0, 0); +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc index afcbb44bdb4..9ffb7f87fd1 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc @@ -312,20 +312,12 @@ class WarmupURLFetcher : public net::URLFetcherDelegate { fetcher_->Start(); } - void SetWarmupURLFetcherCallbackForTesting( - base::Callback<void()> warmup_url_fetched_callback) { - fetch_completion_callback_ = warmup_url_fetched_callback; - } - private: void OnURLFetchComplete(const net::URLFetcher* source) override { DCHECK_EQ(source, fetcher_.get()); UMA_HISTOGRAM_BOOLEAN( "DataReductionProxy.WarmupURL.FetchSuccessful", source->GetStatus().status() == net::URLRequestStatus::SUCCESS); - - if (fetch_completion_callback_) - fetch_completion_callback_.Run(); } scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_; @@ -333,9 +325,6 @@ class WarmupURLFetcher : public net::URLFetcherDelegate { // The URLFetcher being used for fetching the warmup URL. std::unique_ptr<net::URLFetcher> fetcher_; - // Called upon the completion of fetching of the warmup URL. May be null. - base::Callback<void()> fetch_completion_callback_; - DISALLOW_COPY_AND_ASSIGN(WarmupURLFetcher); }; @@ -871,14 +860,6 @@ void DataReductionProxyConfig::FetchWarmupURL() { warmup_url_fetcher_->FetchWarmupURL(); } -void DataReductionProxyConfig::SetWarmupURLFetcherCallbackForTesting( - base::Callback<void()> warmup_url_fetched_callback) { - DCHECK(thread_checker_.CalledOnValidThread()); - - warmup_url_fetcher_->SetWarmupURLFetcherCallbackForTesting( - warmup_url_fetched_callback); -} - void DataReductionProxyConfig::SetLoFiModeOff() { DCHECK(thread_checker_.CalledOnValidThread()); lofi_off_ = true; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h index 34348166047..5bc7b57a8b9 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h @@ -228,10 +228,6 @@ class DataReductionProxyConfig // Updates the Data Reduction Proxy configurator with the current config. void UpdateConfigForTesting(bool enabled, bool restricted); - // Updates the callback that is called when the warmup URL has been fetched. - void SetWarmupURLFetcherCallbackForTesting( - base::Callback<void()> warmup_url_fetched_callback); - private: friend class MockDataReductionProxyConfig; friend class TestDataReductionProxyConfig; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.h index c14d723f453..0a54c8671bd 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.h @@ -120,7 +120,7 @@ class DataReductionProxyConfigServiceClient // Examines |response_headers| to determine if an authentication failure // occurred on a Data Reduction Proxy. Returns true if authentication failure - // occured, and the session key specified in |request_headers| matches the + // occurred, and the session key specified in |request_headers| matches the // current session in use by the client. If an authentication failure is // detected, it fetches a new config. bool ShouldRetryDueToAuthFailure( diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h index 4198e68ad23..05b59b391e3 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h @@ -118,7 +118,6 @@ class TestDataReductionProxyConfig : public DataReductionProxyConfig { void SetIsCaptivePortal(bool is_captive_portal); using DataReductionProxyConfig::UpdateConfigForTesting; - using DataReductionProxyConfig::SetWarmupURLFetcherCallbackForTesting; private: bool GetIsCaptivePortal() const override; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc index 0a917ea4bf6..f70b5e700b8 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc @@ -165,17 +165,6 @@ class DataReductionProxyConfigTest : public testing::Test { 1); } - void WarmupURLFetchedCallBack() const { - warmup_url_fetched_run_loop_->Quit(); - } - - void WarmUpURLFetchedRunLoop() { - warmup_url_fetched_run_loop_.reset(new base::RunLoop()); - // |warmup_url_fetched_run_loop_| will run until WarmupURLFetchedCallBack() - // is called. - warmup_url_fetched_run_loop_->Run(); - } - void RunUntilIdle() { test_context_->RunUntilIdle(); } @@ -213,7 +202,6 @@ class DataReductionProxyConfigTest : public testing::Test { std::unique_ptr<net::NetworkChangeNotifier> network_change_notifier_; base::MessageLoopForIO message_loop_; - std::unique_ptr<base::RunLoop> warmup_url_fetched_run_loop_; std::unique_ptr<DataReductionProxyTestContext> test_context_; std::unique_ptr<TestDataReductionProxyParams> expected_params_; }; @@ -375,9 +363,6 @@ TEST_F(DataReductionProxyConfigTest, WarmupURL) { new net::TestURLRequestContextGetter(task_runner()); config.InitializeOnIOThread(request_context_getter_.get(), request_context_getter_.get()); - config.SetWarmupURLFetcherCallbackForTesting( - base::Bind(&DataReductionProxyConfigTest::WarmupURLFetchedCallBack, - base::Unretained(this))); // Set the connection type to WiFi so that warm up URL is fetched even if // the test device does not have connectivity. @@ -387,12 +372,8 @@ TEST_F(DataReductionProxyConfigTest, WarmupURL) { test.data_reduction_proxy_enabled && test.enabled_via_field_trial; if (warmup_url_enabled) { - // Block until warm up URL is fetched successfully. - WarmUpURLFetchedRunLoop(); histogram_tester.ExpectUniqueSample( "DataReductionProxy.WarmupURL.FetchInitiated", 1, 1); - histogram_tester.ExpectUniqueSample( - "DataReductionProxy.WarmupURL.FetchSuccessful", 1, 1); } // Set the connection type to 4G so that warm up URL is fetched even if @@ -402,12 +383,8 @@ TEST_F(DataReductionProxyConfigTest, WarmupURL) { RunUntilIdle(); if (warmup_url_enabled) { - // Block until warm up URL is fetched successfully. - WarmUpURLFetchedRunLoop(); histogram_tester.ExpectUniqueSample( "DataReductionProxy.WarmupURL.FetchInitiated", 1, 2); - histogram_tester.ExpectUniqueSample( - "DataReductionProxy.WarmupURL.FetchSuccessful", 1, 2); } else { histogram_tester.ExpectTotalCount( "DataReductionProxy.WarmupURL.FetchInitiated", 0); @@ -424,8 +401,6 @@ TEST_F(DataReductionProxyConfigTest, WarmupURL) { if (warmup_url_enabled) { histogram_tester.ExpectUniqueSample( "DataReductionProxy.WarmupURL.FetchInitiated", 1, 2); - histogram_tester.ExpectUniqueSample( - "DataReductionProxy.WarmupURL.FetchSuccessful", 1, 2); } else { histogram_tester.ExpectTotalCount( "DataReductionProxy.WarmupURL.FetchInitiated", 0); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator_unittest.cc index a98ba92d026..fbdf6768048 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator_unittest.cc @@ -8,6 +8,7 @@ #include <string> #include <vector> +#include "base/test/scoped_task_environment.h" #include "base/values.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_server.h" @@ -60,7 +61,7 @@ class DataReductionProxyConfiguratorTest : public testing::Test { } } - base::MessageLoop message_loop_; + base::test::ScopedTaskEnvironment scoped_task_environment_; std::unique_ptr<DataReductionProxyTestContext> test_context_; std::unique_ptr<DataReductionProxyConfigurator> config_; }; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc index 5452658353b..425d1be797f 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc @@ -14,16 +14,23 @@ const void* const kDataReductionProxyUserDataKey = DataReductionProxyData::DataReductionProxyData() : used_data_reduction_proxy_(false), lofi_requested_(false), + lite_page_received_(false), + lofi_received_(false), effective_connection_type_(net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN) {} +DataReductionProxyData::~DataReductionProxyData() {} + std::unique_ptr<DataReductionProxyData> DataReductionProxyData::DeepCopy() const { std::unique_ptr<DataReductionProxyData> copy(new DataReductionProxyData()); copy->used_data_reduction_proxy_ = used_data_reduction_proxy_; copy->lofi_requested_ = lofi_requested_; + copy->lite_page_received_ = lite_page_received_; + copy->lofi_received_ = lofi_received_; copy->session_key_ = session_key_; copy->request_url_ = request_url_; copy->effective_connection_type_ = effective_connection_type_; + copy->page_id_ = page_id_; return copy; } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h index 3b4b53f8546..1f12d47e502 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h @@ -5,10 +5,13 @@ #ifndef COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_DATA_H_ #define COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_DATA_H_ +#include <stdint.h> + #include <memory> #include <string> #include "base/macros.h" +#include "base/optional.h" #include "base/supports_user_data.h" #include "net/nqe/effective_connection_type.h" #include "url/gurl.h" @@ -24,6 +27,7 @@ namespace data_reduction_proxy { class DataReductionProxyData : public base::SupportsUserData::Data { public: DataReductionProxyData(); + ~DataReductionProxyData() override; // Whether the DataReductionProxy was used for this request or navigation. bool used_data_reduction_proxy() const { return used_data_reduction_proxy_; } @@ -39,14 +43,24 @@ class DataReductionProxyData : public base::SupportsUserData::Data { lofi_requested_ = lofi_requested; } - // The session key used for this request. + // Whether a lite page response was seen for the request or navigation. + bool lite_page_received() const { return lite_page_received_; } + void set_lite_page_received(bool lite_page_received) { + lite_page_received_ = lite_page_received; + } + + // Whether a lite page response was seen for the request or navigation. + bool lofi_received() const { return lofi_received_; } + void set_lofi_received(bool lofi_received) { lofi_received_ = lofi_received; } + + // The session key used for this request. Only set for main frame requests. std::string session_key() const { return session_key_; } void set_session_key(const std::string& session_key) { session_key_ = session_key; } // The URL the frame is navigating to. This may change during the navigation - // when encountering a server redirect. + // when encountering a server redirect. Only set for main frame requests. GURL request_url() const { return request_url_; } void set_request_url(const GURL& request_url) { request_url_ = request_url; } @@ -60,6 +74,11 @@ class DataReductionProxyData : public base::SupportsUserData::Data { effective_connection_type_ = effective_connection_type; } + // An identifier that is guaranteed to be unique to each page load during a + // data saver session. Only present on main frame requests. + const base::Optional<uint64_t>& page_id() const { return page_id_; } + void set_page_id(uint64_t page_id) { page_id_ = page_id; } + // Removes |this| from |request|. static void ClearData(net::URLRequest* request); @@ -85,6 +104,12 @@ class DataReductionProxyData : public base::SupportsUserData::Data { // slow. bool lofi_requested_; + // Whether a lite page response was seen for the request or navigation. + bool lite_page_received_; + + // Whether a lite page response was seen for the request or navigation. + bool lofi_received_; + // The session key used for this request or navigation. std::string session_key_; @@ -96,6 +121,10 @@ class DataReductionProxyData : public base::SupportsUserData::Data { // set for main frame requests only. net::EffectiveConnectionType effective_connection_type_; + // An identifier that is guaranteed to be unique to each page load during a + // data saver session. Only present on main frame requests. + base::Optional<uint64_t> page_id_; + DISALLOW_COPY_AND_ASSIGN(DataReductionProxyData); }; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc index 09aaaa199ba..8ba21079bf2 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc @@ -4,7 +4,10 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h" +#include <stdint.h> + #include <memory> +#include <string> #include "base/memory/ptr_util.h" #include "base/message_loop/message_loop.h" @@ -41,19 +44,38 @@ TEST_F(DataReductionProxyDataTest, BasicSettersAndGetters) { data->set_lofi_requested(false); EXPECT_FALSE(data->lofi_requested()); + EXPECT_FALSE(data->lite_page_received()); + data->set_lite_page_received(true); + EXPECT_TRUE(data->lite_page_received()); + data->set_lite_page_received(false); + EXPECT_FALSE(data->lite_page_received()); + + EXPECT_FALSE(data->lofi_received()); + data->set_lofi_received(true); + EXPECT_TRUE(data->lofi_received()); + data->set_lofi_received(false); + EXPECT_FALSE(data->lofi_received()); + EXPECT_EQ(std::string(), data->session_key()); - EXPECT_EQ(GURL(std::string()), data->request_url()); std::string session_key = "test-key"; data->set_session_key(session_key); EXPECT_EQ(session_key, data->session_key()); + + EXPECT_EQ(GURL(std::string()), data->request_url()); GURL test_url("test-url"); data->set_request_url(test_url); EXPECT_EQ(test_url, data->request_url()); + EXPECT_EQ(net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN, data->effective_connection_type()); data->set_effective_connection_type(net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); EXPECT_EQ(net::EFFECTIVE_CONNECTION_TYPE_OFFLINE, data->effective_connection_type()); + + EXPECT_FALSE(data->page_id()); + uint64_t page_id = 1; + data->set_page_id(page_id); + EXPECT_EQ(page_id, data->page_id().value()); } TEST_F(DataReductionProxyDataTest, AddToURLRequest) { @@ -76,7 +98,7 @@ TEST_F(DataReductionProxyDataTest, AddToURLRequest) { TEST_F(DataReductionProxyDataTest, DeepCopy) { const struct { bool data_reduction_used; - bool lofi_on; + bool lofi_test_value; } tests[] = { { false, true, @@ -97,17 +119,23 @@ TEST_F(DataReductionProxyDataTest, DeepCopy) { static const GURL kTestURL("test-url"); std::unique_ptr<DataReductionProxyData> data(new DataReductionProxyData()); data->set_used_data_reduction_proxy(tests[i].data_reduction_used); - data->set_lofi_requested(tests[i].lofi_on); + data->set_lofi_requested(tests[i].lofi_test_value); + data->set_lite_page_received(tests[i].lofi_test_value); + data->set_lofi_received(tests[i].lofi_test_value); data->set_session_key(kSessionKey); data->set_request_url(kTestURL); data->set_effective_connection_type(net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); + data->set_page_id(2u); std::unique_ptr<DataReductionProxyData> copy = data->DeepCopy(); - EXPECT_EQ(tests[i].lofi_on, copy->lofi_requested()); + EXPECT_EQ(tests[i].lofi_test_value, copy->lofi_requested()); + EXPECT_EQ(tests[i].lofi_test_value, copy->lite_page_received()); + EXPECT_EQ(tests[i].lofi_test_value, copy->lofi_received()); EXPECT_EQ(tests[i].data_reduction_used, copy->used_data_reduction_proxy()); EXPECT_EQ(kSessionKey, copy->session_key()); EXPECT_EQ(kTestURL, copy->request_url()); EXPECT_EQ(net::EFFECTIVE_CONNECTION_TYPE_OFFLINE, copy->effective_connection_type()); + EXPECT_EQ(2u, data->page_id().value()); } } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc index a9c344e187b..12c809f54e4 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc @@ -99,7 +99,7 @@ void DataReductionProxyDelegate::OnResolveProxy( !config_->secure_proxy_allowed(), proxies_for_http); OnResolveProxyHandler(url, method, proxy_config, - proxy_service.proxy_retry_info(), config_, io_data_, + proxy_service.proxy_retry_info(), *config_, io_data_, result); if (!first_data_saver_request_recorded_ && !result->is_empty() && @@ -244,9 +244,7 @@ bool DataReductionProxyDelegate::SupportsQUIC( const net::ProxyServer& proxy_server) const { DCHECK(thread_checker_.CalledOnValidThread()); // Enable QUIC for whitelisted proxies. - // TODO(tbansal): Use client config service to control this whitelist. - return base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kDataReductionProxyEnableQuicOnNonCoreProxies) || + return params::IsQuicEnabledForNonCoreProxies() || proxy_server == net::ProxyServer(net::ProxyServer::SCHEME_HTTPS, net::HostPortPair(kDataReductionCoreProxy, 443)); @@ -277,12 +275,11 @@ void OnResolveProxyHandler( const std::string& method, const net::ProxyConfig& proxy_config, const net::ProxyRetryInfoMap& proxy_retry_info, - const DataReductionProxyConfig* data_reduction_proxy_config, + const DataReductionProxyConfig& data_reduction_proxy_config, DataReductionProxyIOData* io_data, net::ProxyInfo* result) { - DCHECK(data_reduction_proxy_config); DCHECK(result->is_empty() || result->is_direct() || - !data_reduction_proxy_config->IsDataReductionProxy( + !data_reduction_proxy_config.IsDataReductionProxy( result->proxy_server(), NULL)); if (!util::EligibleForDataReductionProxy(*result, url, method)) @@ -306,12 +303,12 @@ void OnResolveProxyHandler( // The |proxy_config| must be valid otherwise the proxy cannot be used. DCHECK(proxy_config.is_valid() || !data_saver_proxy_used); - if (data_reduction_proxy_config->enabled_by_user_and_reachable() && - url.SchemeIsHTTPOrHTTPS() && !url.SchemeIsCryptographic() && - !net::IsLocalhost(url.host()) && - (!proxy_config.is_valid() || data_saver_proxy_used)) { - UMA_HISTOGRAM_BOOLEAN("DataReductionProxy.ConfigService.HTTPRequests", - data_saver_proxy_used); + if (data_reduction_proxy_config.enabled_by_user_and_reachable() && + url.SchemeIs(url::kHttpScheme) && !net::IsLocalhost(url.host_piece()) && + !params::IsIncludedInHoldbackFieldTrial()) { + UMA_HISTOGRAM_BOOLEAN( + "DataReductionProxy.ConfigService.HTTPRequests", + !data_reduction_proxy_config.GetProxiesForHttp().empty()); } } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h index 2b54b4672ec..d412dc3b1cc 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h @@ -157,7 +157,7 @@ void OnResolveProxyHandler( const std::string& method, const net::ProxyConfig& proxy_config, const net::ProxyRetryInfoMap& proxy_retry_info, - const DataReductionProxyConfig* data_reduction_proxy_config, + const DataReductionProxyConfig& data_reduction_proxy_config, DataReductionProxyIOData* io_data, net::ProxyInfo* result); } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc index fb48f0e39bd..f7a2306fd34 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc @@ -772,7 +772,7 @@ TEST_F(DataReductionProxyDelegateTest, OnResolveProxyHandler) { // Another proxy is used. It should be used afterwards. result.Use(other_proxy_info); OnResolveProxyHandler(url, "GET", data_reduction_proxy_config, - empty_proxy_retry_info, config(), nullptr, &result); + empty_proxy_retry_info, *config(), nullptr, &result); EXPECT_EQ(other_proxy_info.proxy_server(), result.proxy_server()); // A direct connection is used. The data reduction proxy should be used @@ -781,7 +781,7 @@ TEST_F(DataReductionProxyDelegateTest, OnResolveProxyHandler) { result.Use(direct_proxy_info); net::ProxyConfig::ID prev_id = result.config_id(); OnResolveProxyHandler(url, "GET", data_reduction_proxy_config, - empty_proxy_retry_info, config(), nullptr, &result); + empty_proxy_retry_info, *config(), nullptr, &result); EXPECT_EQ(data_reduction_proxy_info.proxy_server(), result.proxy_server()); // Only the proxy list should be updated, not the proxy info. EXPECT_EQ(result.config_id(), prev_id); @@ -792,7 +792,7 @@ TEST_F(DataReductionProxyDelegateTest, OnResolveProxyHandler) { prev_id = result.config_id(); OnResolveProxyHandler( GURL("ws://echo.websocket.org/"), "GET", data_reduction_proxy_config, - data_reduction_proxy_retry_info, config(), nullptr, &result); + data_reduction_proxy_retry_info, *config(), nullptr, &result); EXPECT_TRUE(result.proxy_server().is_direct()); EXPECT_EQ(result.config_id(), prev_id); @@ -800,30 +800,30 @@ TEST_F(DataReductionProxyDelegateTest, OnResolveProxyHandler) { result.UseDirect(); OnResolveProxyHandler(GURL("wss://echo.websocket.org/"), "GET", data_reduction_proxy_config, empty_proxy_retry_info, - config(), nullptr, &result); + *config(), nullptr, &result); EXPECT_TRUE(result.is_direct()); result.UseDirect(); OnResolveProxyHandler(GURL("wss://echo.websocket.org/"), "GET", data_reduction_proxy_config, empty_proxy_retry_info, - config(), nullptr, &result); + *config(), nullptr, &result); EXPECT_TRUE(result.is_direct()); // POST methods go direct. result.UseDirect(); OnResolveProxyHandler(url, "POST", data_reduction_proxy_config, - empty_proxy_retry_info, config(), nullptr, &result); + empty_proxy_retry_info, *config(), nullptr, &result); EXPECT_TRUE(result.is_direct()); // Without DataCompressionProxyCriticalBypass Finch trial set, the // BYPASS_DATA_REDUCTION_PROXY load flag should be ignored. result.UseDirect(); OnResolveProxyHandler(url, "GET", data_reduction_proxy_config, - empty_proxy_retry_info, config(), nullptr, &result); + empty_proxy_retry_info, *config(), nullptr, &result); EXPECT_FALSE(result.is_direct()); OnResolveProxyHandler(url, "GET", data_reduction_proxy_config, - empty_proxy_retry_info, config(), nullptr, + empty_proxy_retry_info, *config(), nullptr, &other_proxy_info); EXPECT_FALSE(other_proxy_info.is_direct()); } @@ -907,15 +907,14 @@ TEST_F(DataReductionProxyDelegateTest, HTTPRequests) { net::ProxyInfo result; result.Use(direct_proxy_info); OnResolveProxyHandler(url, "GET", data_reduction_proxy_config, - empty_proxy_retry_info, config(), nullptr, &result); + empty_proxy_retry_info, *config(), nullptr, &result); histogram_tester.ExpectTotalCount( "DataReductionProxy.ConfigService.HTTPRequests", test.expect_histogram ? 1 : 0); if (test.expect_histogram) { histogram_tester.ExpectUniqueSample( - "DataReductionProxy.ConfigService.HTTPRequests", - test.use_direct_proxy ? 0 : 1, 1); + "DataReductionProxy.ConfigService.HTTPRequests", 1, 1); } } } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc index 70390e396ac..4d8f08897a2 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc @@ -14,6 +14,7 @@ #include "base/memory/ref_counted.h" #include "base/message_loop/message_loop.h" #include "base/run_loop.h" +#include "base/test/histogram_tester.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h" @@ -399,6 +400,64 @@ TEST_F(DataReductionProxyInterceptorEndToEndTest, RedirectWithoutRetry) { EXPECT_EQ(1, delegate().received_redirect_count()); } +// Test that data reduction proxy is byppassed if there is a URL redirect cycle. +TEST_F(DataReductionProxyInterceptorEndToEndTest, URLRedirectCycle) { + base::HistogramTester histogram_tester; + MockRead redirect_mock_reads_1[] = { + MockRead("HTTP/1.1 302 Found\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Location: http://bar.com/\r\n\r\n"), + MockRead(""), MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_1( + redirect_mock_reads_1, arraysize(redirect_mock_reads_1), nullptr, 0); + mock_socket_factory()->AddSocketDataProvider( + &redirect_socket_data_provider_1); + + MockRead redirect_mock_reads_2[] = { + MockRead("HTTP/1.1 302 Found\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Location: http://foo.com/\r\n\r\n"), + MockRead(""), MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_2( + redirect_mock_reads_2, arraysize(redirect_mock_reads_2), nullptr, 0); + mock_socket_factory()->AddSocketDataProvider( + &redirect_socket_data_provider_2); + + // Redirect cycle. + MockRead redirect_mock_reads_3[] = { + MockRead("HTTP/1.1 302 Found\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Location: http://bar.com/\r\n\r\n"), + MockRead(""), MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_3( + redirect_mock_reads_3, arraysize(redirect_mock_reads_3), nullptr, 0); + mock_socket_factory()->AddSocketDataProvider( + &redirect_socket_data_provider_3); + + // Data reduction proxy should be bypassed. + MockRead redirect_mock_reads_4[] = { + MockRead("HTTP/1.1 200 OK\r\n\r\n"), MockRead(kBody.c_str()), + MockRead(net::SYNCHRONOUS, net::OK), + }; + net::StaticSocketDataProvider redirect_socket_data_provider_4( + redirect_mock_reads_4, arraysize(redirect_mock_reads_4), nullptr, 0); + mock_socket_factory()->AddSocketDataProvider( + &redirect_socket_data_provider_4); + + std::unique_ptr<net::URLRequest> request = + CreateAndExecuteRequest(GURL("http://foo.com")); + + EXPECT_EQ(net::OK, delegate().request_status()); + EXPECT_EQ(200, request->GetResponseCode()); + EXPECT_EQ(kBody, delegate().data_received()); + EXPECT_FALSE(request->was_fetched_via_proxy()); + histogram_tester.ExpectTotalCount( + "DataReductionProxy.BypassedBytes.URLRedirectCycle", 1); +} + TEST_F(DataReductionProxyInterceptorEndToEndTest, ResponseWithBypassAndRetry) { // The first try gives a bypass. MockRead initial_mock_reads[] = { diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.cc index b9a4168edbc..cfe96f4bfc4 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.cc @@ -36,7 +36,7 @@ DataReductionProxyRequestType GetDataReductionProxyRequestType( // Data Reduction Proxy, since 304s aren't required to have a Via header even // if they came through the Data Reduction Proxy. if (request.response_headers() && - (HasDataReductionProxyViaHeader(request.response_headers(), nullptr) || + (HasDataReductionProxyViaHeader(*request.response_headers(), nullptr) || (request.response_headers()->response_code() == net::HTTP_NOT_MODIFIED && config.WasDataReductionProxyUsed(&request, nullptr)))) { return VIA_DATA_REDUCTION_PROXY; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc index 8bae820f90e..12d8b56770d 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc @@ -23,6 +23,7 @@ #include "components/data_reduction_proxy/core/common/data_reduction_proxy_util.h" #include "components/data_reduction_proxy/core/common/lofi_decider.h" #include "net/base/load_flags.h" +#include "net/base/mime_util.h" #include "net/http/http_request_headers.h" #include "net/http/http_response_headers.h" #include "net/nqe/network_quality_estimator.h" @@ -46,27 +47,31 @@ namespace { // |freshness_lifetime| contains information on how long the resource will be // fresh for and how long is the usability. void RecordContentLengthHistograms(bool lofi_low_header_added, + bool is_https, + bool is_video, int64_t received_content_length, int64_t original_content_length, const base::TimeDelta& freshness_lifetime) { // Add the current resource to these histograms only when a valid // X-Original-Content-Length header is present. if (original_content_length >= 0) { - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthWithValidOCL", - received_content_length); - UMA_HISTOGRAM_COUNTS("Net.HttpOriginalContentLengthWithValidOCL", - original_content_length); - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthDifferenceWithValidOCL", - original_content_length - received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthWithValidOCL", + received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpOriginalContentLengthWithValidOCL", + original_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthDifferenceWithValidOCL", + original_content_length - received_content_length); // Populate Lo-Fi content length histograms. if (lofi_low_header_added) { - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthWithValidOCL.LoFiOn", - received_content_length); - UMA_HISTOGRAM_COUNTS("Net.HttpOriginalContentLengthWithValidOCL.LoFiOn", - original_content_length); - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthDifferenceWithValidOCL.LoFiOn", - original_content_length - received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthWithValidOCL.LoFiOn", + received_content_length); + UMA_HISTOGRAM_COUNTS_1M( + "Net.HttpOriginalContentLengthWithValidOCL.LoFiOn", + original_content_length); + UMA_HISTOGRAM_COUNTS_1M( + "Net.HttpContentLengthDifferenceWithValidOCL.LoFiOn", + original_content_length - received_content_length); } } else { @@ -74,11 +79,22 @@ void RecordContentLengthHistograms(bool lofi_low_header_added, // length if the X-Original-Content-Header is not present. original_content_length = received_content_length; } - UMA_HISTOGRAM_COUNTS("Net.HttpContentLength", received_content_length); - UMA_HISTOGRAM_COUNTS("Net.HttpOriginalContentLength", - original_content_length); - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthDifference", - original_content_length - received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength", received_content_length); + if (is_https) { + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength.Https", + received_content_length); + } else { + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength.Http", + received_content_length); + } + if (is_video) { + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength.Video", + received_content_length); + } + UMA_HISTOGRAM_COUNTS_1M("Net.HttpOriginalContentLength", + original_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthDifference", + original_content_length - received_content_length); UMA_HISTOGRAM_CUSTOM_COUNTS("Net.HttpContentFreshnessLifetime", freshness_lifetime.InSeconds(), base::TimeDelta::FromHours(1).InSeconds(), @@ -86,17 +102,17 @@ void RecordContentLengthHistograms(bool lofi_low_header_added, 100); if (freshness_lifetime.InSeconds() <= 0) return; - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthCacheable", - received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthCacheable", + received_content_length); if (freshness_lifetime.InHours() < 4) return; - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthCacheable4Hours", - received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthCacheable4Hours", + received_content_length); if (freshness_lifetime.InHours() < 24) return; - UMA_HISTOGRAM_COUNTS("Net.HttpContentLengthCacheable24Hours", - received_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthCacheable24Hours", + received_content_length); } // Given a |request| that went through the Data Reduction Proxy, this function @@ -113,6 +129,19 @@ int64_t EstimateOriginalReceivedBytes(const net::URLRequest& request) { util::CalculateEffectiveOCL(request); } +// Verifies that the chrome proxy related request headers are set correctly. +// |via_chrome_proxy| is true if the request is being fetched via Chrome Data +// Saver proxy. +void VerifyHttpRequestHeaders(bool via_chrome_proxy, + const net::HttpRequestHeaders& headers) { + if (via_chrome_proxy) { + DCHECK(headers.HasHeader(chrome_proxy_header())); + } else { + DCHECK(!headers.HasHeader(chrome_proxy_header())); + DCHECK(!headers.HasHeader(chrome_proxy_accept_transform_header())); + } +} + } // namespace DataReductionProxyNetworkDelegate::DataReductionProxyNetworkDelegate( @@ -205,6 +234,17 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( DCHECK(data_reduction_proxy_config_); DCHECK(request); + // If there was a redirect or request bypass, use the same page ID for both + // requests. As long as the session ID has not changed. Re-issued requests + // and client redirects will be assigned a new page ID as they are different + // URLRequests. + DataReductionProxyData* data = DataReductionProxyData::GetData(*request); + base::Optional<uint64_t> page_id; + if (data && data->session_key() == + data_reduction_proxy_request_options_->GetSecureSession()) { + page_id = data->page_id(); + } + // Reset |request|'s DataReductionProxyData. DataReductionProxyData::ClearData(request); @@ -212,10 +252,10 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( if (!WasEligibleWithoutHoldback(*request, proxy_info, proxy_retry_info)) return; // For the holdback field trial, still log UMA as if the proxy was used. - DataReductionProxyData* data = - DataReductionProxyData::GetDataAndCreateIfNecessary(request); + data = DataReductionProxyData::GetDataAndCreateIfNecessary(request); if (data) data->set_used_data_reduction_proxy(true); + VerifyHttpRequestHeaders(false, *headers); return; } @@ -241,23 +281,26 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( // Chrome-Proxy-Accept-Transform header. lofi_decider->RemoveAcceptTransformHeader(headers); } + VerifyHttpRequestHeaders(false, *headers); return; } // Retrieves DataReductionProxyData from a request, creating a new instance // if needed. - DataReductionProxyData* data = - DataReductionProxyData::GetDataAndCreateIfNecessary(request); + data = DataReductionProxyData::GetDataAndCreateIfNecessary(request); if (data) { data->set_used_data_reduction_proxy(true); - data->set_session_key( - data_reduction_proxy_request_options_->GetSecureSession()); - data->set_request_url(request->url()); - if ((request->load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) && - request->context()->network_quality_estimator()) { - data->set_effective_connection_type(request->context() - ->network_quality_estimator() - ->GetEffectiveConnectionType()); + // Only set GURL, NQE and session key string for main frame requests since + // they are not needed for sub-resources. + if (request->load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) { + data->set_session_key( + data_reduction_proxy_request_options_->GetSecureSession()); + data->set_request_url(request->url()); + if (request->context()->network_quality_estimator()) { + data->set_effective_connection_type(request->context() + ->network_quality_estimator() + ->GetEffectiveConnectionType()); + } } } @@ -274,17 +317,46 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( } MaybeAddBrotliToAcceptEncodingHeader(proxy_info, headers, *request); - data_reduction_proxy_request_options_->AddRequestHeader(headers); + // Generate a page ID for main frame requests that don't already have one. + // TODO(ryansturm): remove LOAD_MAIN_FRAME_DEPRECATED from d_r_p. + // crbug.com/709621 + if (request->load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) { + if (!page_id) { + page_id = data_reduction_proxy_request_options_->GeneratePageId(); + } + data->set_page_id(page_id.value()); + } + + data_reduction_proxy_request_options_->AddRequestHeader(headers, page_id); + if (lofi_decider) lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(headers); + VerifyHttpRequestHeaders(true, *headers); } void DataReductionProxyNetworkDelegate::OnBeforeRedirectInternal( net::URLRequest* request, const GURL& new_location) { // Since this is after a redirect response, reset |request|'s - // DataReductionProxyData. + // DataReductionProxyData, but keep page ID and session. + // TODO(ryansturm): Change ClearData logic to have persistent and + // non-persistent (WRT redirects) data. + // crbug.com/709564 + DataReductionProxyData* data = DataReductionProxyData::GetData(*request); + base::Optional<uint64_t> page_id; + if (data && data->session_key() == + data_reduction_proxy_request_options_->GetSecureSession()) { + page_id = data->page_id(); + } + DataReductionProxyData::ClearData(request); + + if (page_id) { + data = DataReductionProxyData::GetDataAndCreateIfNecessary(request); + data->set_page_id(page_id.value()); + data->set_session_key( + data_reduction_proxy_request_options_->GetSecureSession()); + } } void DataReductionProxyNetworkDelegate::OnCompletedInternal( @@ -338,6 +410,25 @@ void DataReductionProxyNetworkDelegate::OnCompletedInternal( RecordContentLength(*request, request_type, original_content_length); } +void DataReductionProxyNetworkDelegate::OnHeadersReceivedInternal( + net::URLRequest* request, + const net::CompletionCallback& callback, + const net::HttpResponseHeaders* original_response_headers, + scoped_refptr<net::HttpResponseHeaders>* override_response_headers, + GURL* allowed_unsafe_redirect_url) { + if (!original_response_headers) + return; + if (IsEmptyImagePreview(*original_response_headers)) { + DataReductionProxyData* data = + DataReductionProxyData::GetDataAndCreateIfNecessary(request); + data->set_lofi_received(true); + } else if (IsLitePagePreview(*original_response_headers)) { + DataReductionProxyData* data = + DataReductionProxyData::GetDataAndCreateIfNecessary(request); + data->set_lite_page_received(true); + } +} + void DataReductionProxyNetworkDelegate::CalculateAndRecordDataUsage( const net::URLRequest& request, DataReductionProxyRequestType request_type) { @@ -395,14 +486,21 @@ void DataReductionProxyNetworkDelegate::RecordContentLength( ->GetFreshnessLifetimes(request.response_info().response_time) .freshness; + bool is_https = request.url().SchemeIs("https"); + bool is_video = false; + std::string mime_type; + if (request.response_headers()->GetMimeType(&mime_type)) { + is_video = net::MatchesMimeType("video/*", mime_type); + } + RecordContentLengthHistograms( // |data_reduction_proxy_io_data_| can be NULL for Webview. data_reduction_proxy_io_data_ && data_reduction_proxy_io_data_->IsEnabled() && data_reduction_proxy_io_data_->lofi_decider() && data_reduction_proxy_io_data_->lofi_decider()->IsUsingLoFi(request), - request.received_response_content_length(), original_content_length, - freshness_lifetime); + is_https, is_video, request.received_response_content_length(), + original_content_length, freshness_lifetime); if (data_reduction_proxy_io_data_ && data_reduction_proxy_bypass_stats_) { // Record BypassedBytes histograms for the request. diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h index f36cea458de..33c717f128e 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h @@ -115,6 +115,15 @@ class DataReductionProxyNetworkDelegate : public net::LayeredNetworkDelegate { void OnCompletedInternal(net::URLRequest* request, bool started) override; + // Checks if a LoFi or Lite Pages response was received and sets the state on + // DataReductionProxyData for |request|. + void OnHeadersReceivedInternal( + net::URLRequest* request, + const net::CompletionCallback& callback, + const net::HttpResponseHeaders* original_response_headers, + scoped_refptr<net::HttpResponseHeaders>* override_response_headers, + GURL* allowed_unsafe_redirect_url) override; + // Calculates actual data usage that went over the network at the HTTP layer // (e.g. not including network layer overhead) and estimates original data // usage for |request|. diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc index 028c43e6088..5c3886d257a 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc @@ -12,11 +12,13 @@ #include <utility> #include "base/command_line.h" +#include "base/files/file_util.h" #include "base/macros.h" #include "base/memory/ptr_util.h" #include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/numerics/safe_conversions.h" +#include "base/path_service.h" #include "base/run_loop.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" @@ -57,6 +59,7 @@ #include "net/test/test_data_directory.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_job_factory_impl.h" +#include "net/url_request/url_request_status.h" #include "net/url_request/url_request_test_util.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" @@ -69,6 +72,38 @@ using TestNetworkDelegate = net::NetworkDelegateImpl; const char kOtherProxy[] = "testproxy:17"; const char kTestURL[] = "http://www.google.com/"; +const char kSecureTestURL[] = "https://www.google.com/"; + +const std::string kReceivedValidOCLHistogramName = + "Net.HttpContentLengthWithValidOCL"; +const std::string kOriginalValidOCLHistogramName = + "Net.HttpOriginalContentLengthWithValidOCL"; +const std::string kDifferenceValidOCLHistogramName = + "Net.HttpContentLengthDifferenceWithValidOCL"; + +// Lo-Fi histograms. +const std::string kReceivedValidOCLLoFiOnHistogramName = + "Net.HttpContentLengthWithValidOCL.LoFiOn"; +const std::string kOriginalValidOCLLoFiOnHistogramName = + "Net.HttpOriginalContentLengthWithValidOCL.LoFiOn"; +const std::string kDifferenceValidOCLLoFiOnHistogramName = + "Net.HttpContentLengthDifferenceWithValidOCL.LoFiOn"; + +const std::string kReceivedHistogramName = "Net.HttpContentLength"; +const std::string kReceivedInsecureHistogramName = "Net.HttpContentLength.Http"; +const std::string kReceivedSecureHistogramName = "Net.HttpContentLength.Https"; +const std::string kReceivedVideoHistogramName = "Net.HttpContentLength.Video"; +const std::string kOriginalHistogramName = "Net.HttpOriginalContentLength"; +const std::string kDifferenceHistogramName = "Net.HttpContentLengthDifference"; +const std::string kFreshnessLifetimeHistogramName = + "Net.HttpContentFreshnessLifetime"; +const std::string kCacheableHistogramName = "Net.HttpContentLengthCacheable"; +const std::string kCacheable4HoursHistogramName = + "Net.HttpContentLengthCacheable4Hours"; +const std::string kCacheable24HoursHistogramName = + "Net.HttpContentLengthCacheable24Hours"; +const int64_t kResponseContentLength = 100; +const int64_t kOriginalContentLength = 200; #if defined(OS_ANDROID) const Client kClient = Client::CHROME_ANDROID; @@ -177,29 +212,48 @@ class TestLoFiUIService : public LoFiUIService { bool on_lofi_response_; }; +enum ProxyTestConfig { USE_SECURE_PROXY, USE_INSECURE_PROXY, BYPASS_PROXY }; + class DataReductionProxyNetworkDelegateTest : public testing::Test { public: DataReductionProxyNetworkDelegateTest() - : context_(true), context_storage_(&context_) {} - - void Init(bool use_secure_proxy, bool enable_brotli_globally) { - net::ProxyServer proxy_server = - use_secure_proxy - ? net::ProxyServer::FromURI("https://origin.net:443", - net::ProxyServer::SCHEME_HTTPS) - : net::ProxyServer::FromURI("http://origin.net:80", - net::ProxyServer::SCHEME_HTTP); + : context_(true), + context_storage_(&context_), + ssl_socket_data_provider_(net::ASYNC, net::OK) { + ssl_socket_data_provider_.next_proto = net::kProtoHTTP11; + ssl_socket_data_provider_.cert = net::ImportCertFromFile( + net::GetTestCertsDirectory(), "unittest.selfsigned.der"); + } + void Init(ProxyTestConfig proxy_config, bool enable_brotli_globally) { + net::ProxyServer proxy_server; + switch (proxy_config) { + case BYPASS_PROXY: + proxy_server = net::ProxyServer::Direct(); + break; + case USE_SECURE_PROXY: + proxy_server = net::ProxyServer::FromURI( + "https://origin.net:443", net::ProxyServer::SCHEME_HTTPS); + break; + case USE_INSECURE_PROXY: + proxy_server = net::ProxyServer::FromURI("http://origin.net:80", + net::ProxyServer::SCHEME_HTTP); + break; + } proxy_service_ = net::ProxyService::CreateFixedFromPacResult(proxy_server.ToPacString()); context_.set_proxy_service(proxy_service_.get()); - test_context_ = (DataReductionProxyTestContext::Builder() - .WithClient(kClient) - .WithMockClientSocketFactory(&mock_socket_factory_) - .WithURLRequestContext(&context_) - .WithProxiesForHttp({DataReductionProxyServer( - proxy_server, ProxyServer::UNSPECIFIED_TYPE)}) - .Build()); + DataReductionProxyTestContext::Builder builder; + builder = builder.WithClient(kClient) + .WithMockClientSocketFactory(&mock_socket_factory_) + .WithURLRequestContext(&context_); + + if (proxy_config != BYPASS_PROXY) { + builder = builder.WithProxiesForHttp({DataReductionProxyServer( + proxy_server, ProxyServer::UNSPECIFIED_TYPE)}); + } + + test_context_ = builder.Build(); context_.set_client_socket_factory(&mock_socket_factory_); test_context_->AttachToURLRequestContext(&context_storage_); @@ -213,6 +267,7 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { test_context_->io_data()->set_lofi_ui_service(std::move(lofi_ui_service)); context_.set_enable_brotli(enable_brotli_globally); + context_.set_network_quality_estimator(&test_network_quality_estimator_); context_.Init(); test_context_->EnableDataReductionProxyWithSecureProxyCheckSuccess(); @@ -253,7 +308,8 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { const GURL& url, net::HttpRequestHeaders* request_headers, const std::string& response_headers, - int64_t response_content_length) { + int64_t response_content_length, + int load_flags) { const std::string response_body( base::checked_cast<size_t>(response_content_length), ' '); @@ -268,12 +324,31 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { context_.CreateRequest(url, net::IDLE, &delegate); if (request_headers) request->SetExtraRequestHeaders(*request_headers); + request->SetLoadFlags(request->load_flags() | load_flags); request->Start(); base::RunLoop().RunUntilIdle(); return request; } + // Reads brotli encoded content to |encoded_brotli_buffer_|. + void ReadBrotliFile() { + // Get the path of data directory. + const size_t kDefaultBufferSize = 4096; + base::FilePath data_dir; + PathService::Get(base::DIR_SOURCE_ROOT, &data_dir); + data_dir = data_dir.AppendASCII("net"); + data_dir = data_dir.AppendASCII("data"); + data_dir = data_dir.AppendASCII("filter_unittests"); + + // Read data from the encoded file into buffer. + base::FilePath encoded_file_path; + encoded_file_path = data_dir.AppendASCII("google.br"); + ASSERT_TRUE( + base::ReadFileToString(encoded_file_path, &encoded_brotli_buffer_)); + ASSERT_GE(kDefaultBufferSize, encoded_brotli_buffer_.size()); + } + // Fetches a single URL request, verifies the correctness of Accept-Encoding // header, and verifies that the response is cached only if |expect_cached| // is set to true. Each line in |response_headers| should end with "\r\n" and @@ -284,17 +359,21 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { const std::string& response_headers, bool expect_cached, bool expect_brotli) { + test_network_quality_estimator()->set_effective_connection_type( + net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN); GURL url(kTestURL); - net::SSLSocketDataProvider ssl_socket_data_provider(net::ASYNC, net::OK); int response_body_size = 140; - const std::string response_body( - base::checked_cast<size_t>(response_body_size), ' '); + std::string response_body; + if (expect_brotli && !expect_cached) { + response_body = encoded_brotli_buffer_; + response_body_size = response_body.size(); + } else { + response_body = + std::string(base::checked_cast<size_t>(response_body_size), ' '); + } - ssl_socket_data_provider.next_proto = net::kProtoHTTP11; - ssl_socket_data_provider.cert = net::ImportCertFromFile( - net::GetTestCertsDirectory(), "unittest.selfsigned.der"); - mock_socket_factory_.AddSSLSocketDataProvider(&ssl_socket_data_provider); + mock_socket_factory_.AddSSLSocketDataProvider(&ssl_socket_data_provider_); net::MockRead reads[] = {net::MockRead(response_headers.c_str()), net::MockRead(response_body.c_str()), @@ -358,6 +437,8 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { request->Start(); base::RunLoop().RunUntilIdle(); + EXPECT_EQ(0, request->status().ToNetError()); + if (!expect_cached) { EXPECT_EQ(response_body_size, request->received_response_content_length()); @@ -367,6 +448,10 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { VerifyBrotliPresent(request.get(), expect_brotli); } else { EXPECT_TRUE(request->was_cached()); + std::string content_encoding_value; + request->GetResponseHeaderByName("Content-Encoding", + &content_encoding_value); + EXPECT_EQ(expect_brotli, content_encoding_value == "br"); } } @@ -377,15 +462,93 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { EXPECT_TRUE(request_headers_sent.GetHeader("Accept-Encoding", &accept_encoding_value)); EXPECT_NE(std::string::npos, accept_encoding_value.find("gzip")); + + std::string content_encoding_value; + request->GetResponseHeaderByName("Content-Encoding", + &content_encoding_value); + if (expect_brotli) { // Brotli should be the last entry in the Accept-Encoding header. EXPECT_EQ(accept_encoding_value.length() - 2, accept_encoding_value.find("br")); + EXPECT_EQ("br", content_encoding_value); } else { EXPECT_EQ(std::string::npos, accept_encoding_value.find("br")); } } + void FetchURLRequestAndVerifyPageIdDirective(const std::string& page_id_value, + bool redirect_once) { + std::string response_headers = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 140\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "x-original-content-length: 200\r\n" + "Cache-Control: max-age=1200\r\n" + "Vary: accept-encoding\r\n\r\n"; + + GURL url(kTestURL); + + int response_body_size = 140; + std::string response_body = + std::string(base::checked_cast<size_t>(response_body_size), ' '); + + mock_socket_factory_.AddSSLSocketDataProvider(&ssl_socket_data_provider_); + + net::MockRead redirect_reads[] = { + net::MockRead("HTTP/1.1 302 Redirect\r\n"), + net::MockRead("Location: http://www.google.com/\r\n"), + net::MockRead("Content-Length: 0\r\n\r\n"), + net::MockRead(net::SYNCHRONOUS, net::OK), + net::MockRead(response_headers.c_str()), + net::MockRead(response_body.c_str()), + net::MockRead(net::SYNCHRONOUS, net::OK)}; + + net::MockRead reads[] = {net::MockRead(response_headers.c_str()), + net::MockRead(response_body.c_str()), + net::MockRead(net::SYNCHRONOUS, net::OK)}; + + EXPECT_FALSE( + io_data()->test_request_options()->GetHeaderValueForTesting().empty()); + + std::string mock_write = + "GET http://www.google.com/ HTTP/1.1\r\nHost: " + "www.google.com\r\nProxy-Connection: " + "keep-alive\r\nUser-Agent:\r\nAccept-Encoding: gzip, " + "deflate\r\nAccept-Language: en-us,fr\r\n" + "Chrome-Proxy: " + + io_data()->test_request_options()->GetHeaderValueForTesting() + + (page_id_value.empty() ? "" : (", " + page_id_value)) + "\r\n\r\n"; + + net::MockWrite redirect_writes[] = {net::MockWrite(mock_write.c_str()), + net::MockWrite(mock_write.c_str())}; + + net::MockWrite writes[] = {net::MockWrite(mock_write.c_str())}; + + std::unique_ptr<net::StaticSocketDataProvider> socket; + if (!redirect_once) { + socket = base::MakeUnique<net::StaticSocketDataProvider>( + reads, arraysize(reads), writes, arraysize(writes)); + } else { + socket = base::MakeUnique<net::StaticSocketDataProvider>( + redirect_reads, arraysize(redirect_reads), redirect_writes, + arraysize(redirect_writes)); + } + + mock_socket_factory_.AddSocketDataProvider(socket.get()); + + net::TestDelegate delegate; + std::unique_ptr<net::URLRequest> request = + context_.CreateRequest(url, net::IDLE, &delegate); + if (!page_id_value.empty()) { + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + } + + request->Start(); + base::RunLoop().RunUntilIdle(); + } + void DelegateStageDone(int result) {} void NotifyNetworkDelegate(net::URLRequest* request, @@ -430,6 +593,14 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { TestLoFiDecider* lofi_decider() const { return lofi_decider_; } + net::TestNetworkQualityEstimator* test_network_quality_estimator() { + return &test_network_quality_estimator_; + } + + net::SSLSocketDataProvider* ssl_socket_data_provider() { + return &ssl_socket_data_provider_; + } + private: base::MessageLoopForIO message_loop_; net::MockClientSocketFactory mock_socket_factory_; @@ -440,12 +611,20 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { TestLoFiDecider* lofi_decider_; TestLoFiUIService* lofi_ui_service_; std::unique_ptr<DataReductionProxyTestContext> test_context_; + net::TestNetworkQualityEstimator test_network_quality_estimator_; + + net::SSLSocketDataProvider ssl_socket_data_provider_; + + std::unique_ptr<net::StaticSocketDataProvider> socket_; + + // Encoded Brotli content read from a file. May be empty. + std::string encoded_brotli_buffer_; }; TEST_F(DataReductionProxyNetworkDelegateTest, AuthenticationTest) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); std::unique_ptr<net::URLRequest> fake_request( - FetchURLRequest(GURL(kTestURL), nullptr, std::string(), 0)); + FetchURLRequest(GURL(kTestURL), nullptr, std::string(), 0, 0)); net::ProxyInfo data_reduction_proxy_info; net::ProxyRetryInfoMap proxy_retry_info; @@ -454,6 +633,13 @@ TEST_F(DataReductionProxyNetworkDelegateTest, AuthenticationTest) { data_reduction_proxy_info.UseNamedProxy(data_reduction_proxy); net::HttpRequestHeaders headers; + // Call network delegate methods to ensure that appropriate chrome proxy + // headers get added/removed. + network_delegate()->NotifyBeforeStartTransaction( + fake_request.get(), + base::Bind(&DataReductionProxyNetworkDelegateTest::DelegateStageDone, + base::Unretained(this)), + &headers); network_delegate()->NotifyBeforeSendHeaders(fake_request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); @@ -466,7 +652,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, AuthenticationTest) { } TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); // Enable Lo-Fi. const struct { bool lofi_switch_enabled; @@ -617,7 +803,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { } TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataConfigurations) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); const struct { bool lofi_on; bool used_data_reduction_proxy; @@ -655,10 +841,8 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataConfigurations) { net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; - net::TestNetworkQualityEstimator test_network_quality_estimator; - test_network_quality_estimator.set_effective_connection_type( + test_network_quality_estimator()->set_effective_connection_type( net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); - context()->set_network_quality_estimator(&test_network_quality_estimator); std::unique_ptr<net::URLRequest> request = context()->CreateRequest( GURL(kTestURL), net::RequestPriority::IDLE, nullptr); @@ -666,6 +850,14 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataConfigurations) { : 0); lofi_decider()->SetIsUsingLoFi(test.lofi_on); io_data()->request_options()->SetSecureSession("fake-session"); + + // Call network delegate methods to ensure that appropriate chrome proxy + // headers get added/removed. + network_delegate()->NotifyBeforeStartTransaction( + request.get(), + base::Bind(&DataReductionProxyNetworkDelegateTest::DelegateStageDone, + base::Unretained(this)), + &headers); network_delegate()->NotifyBeforeSendHeaders( request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); DataReductionProxyData* data = @@ -678,8 +870,8 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataConfigurations) { : net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN, data->effective_connection_type()); EXPECT_TRUE(data->used_data_reduction_proxy()); - EXPECT_EQ(GURL(kTestURL), data->request_url()); - EXPECT_EQ("fake-session", data->session_key()); + EXPECT_EQ(test.main_frame ? GURL(kTestURL) : GURL(), data->request_url()); + EXPECT_EQ(test.main_frame ? "fake-session" : "", data->session_key()); EXPECT_EQ(test.lofi_on, data->lofi_requested()); } } @@ -687,7 +879,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataConfigurations) { TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataHoldbackConfigurations) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); const struct { bool data_reduction_proxy_enabled; bool used_direct; @@ -734,28 +926,35 @@ TEST_F(DataReductionProxyNetworkDelegateTest, } TEST_F(DataReductionProxyNetworkDelegateTest, RedirectRequestDataCleared) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); net::ProxyInfo data_reduction_proxy_info; std::string data_reduction_proxy; base::TrimString(params()->DefaultOrigin(), "/", &data_reduction_proxy); data_reduction_proxy_info.UseNamedProxy(data_reduction_proxy); // Main frame loaded. Lo-Fi should be used. - net::HttpRequestHeaders headers; + net::HttpRequestHeaders headers_original; net::ProxyRetryInfoMap proxy_retry_info; - net::TestNetworkQualityEstimator test_network_quality_estimator; - test_network_quality_estimator.set_effective_connection_type( + test_network_quality_estimator()->set_effective_connection_type( net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); - context()->set_network_quality_estimator(&test_network_quality_estimator); std::unique_ptr<net::URLRequest> request = context()->CreateRequest( GURL(kTestURL), net::RequestPriority::IDLE, nullptr); request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); lofi_decider()->SetIsUsingLoFi(true); io_data()->request_options()->SetSecureSession("fake-session"); + + // Call network delegate methods to ensure that appropriate chrome proxy + // headers get added/removed. + network_delegate()->NotifyBeforeStartTransaction( + request.get(), + base::Bind(&DataReductionProxyNetworkDelegateTest::DelegateStageDone, + base::Unretained(this)), + &headers_original); network_delegate()->NotifyBeforeSendHeaders( - request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); + request.get(), data_reduction_proxy_info, proxy_retry_info, + &headers_original); DataReductionProxyData* data = DataReductionProxyData::GetData(*request.get()); @@ -773,46 +972,28 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RedirectRequestDataCleared) { // DataReductionProxyData. network_delegate()->NotifyBeforeRedirect(request.get(), GURL(kTestURL)); data = DataReductionProxyData::GetData(*request.get()); - EXPECT_FALSE(data); + EXPECT_FALSE(data && data->used_data_reduction_proxy()); // Call NotifyBeforeSendHeaders again with different proxy info to check that - // new data isn't added. + // new data isn't added. Use a new set of headers since the redirected HTTP + // jobs do not reuse headers from the previous jobs. Also, call network + // delegate methods to ensure that appropriate chrome proxy headers get + // added/removed. + net::HttpRequestHeaders headers_redirect; + network_delegate()->NotifyBeforeStartTransaction( + request.get(), + base::Bind(&DataReductionProxyNetworkDelegateTest::DelegateStageDone, + base::Unretained(this)), + &headers_redirect); network_delegate()->NotifyBeforeSendHeaders( - request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); + request.get(), data_reduction_proxy_info, proxy_retry_info, + &headers_redirect); data = DataReductionProxyData::GetData(*request.get()); EXPECT_FALSE(data); } TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { - Init(false, false); - const std::string kReceivedValidOCLHistogramName = - "Net.HttpContentLengthWithValidOCL"; - const std::string kOriginalValidOCLHistogramName = - "Net.HttpOriginalContentLengthWithValidOCL"; - const std::string kDifferenceValidOCLHistogramName = - "Net.HttpContentLengthDifferenceWithValidOCL"; - - // Lo-Fi histograms. - const std::string kReceivedValidOCLLoFiOnHistogramName = - "Net.HttpContentLengthWithValidOCL.LoFiOn"; - const std::string kOriginalValidOCLLoFiOnHistogramName = - "Net.HttpOriginalContentLengthWithValidOCL.LoFiOn"; - const std::string kDifferenceValidOCLLoFiOnHistogramName = - "Net.HttpContentLengthDifferenceWithValidOCL.LoFiOn"; - - const std::string kReceivedHistogramName = "Net.HttpContentLength"; - const std::string kOriginalHistogramName = "Net.HttpOriginalContentLength"; - const std::string kDifferenceHistogramName = - "Net.HttpContentLengthDifference"; - const std::string kFreshnessLifetimeHistogramName = - "Net.HttpContentFreshnessLifetime"; - const std::string kCacheableHistogramName = "Net.HttpContentLengthCacheable"; - const std::string kCacheable4HoursHistogramName = - "Net.HttpContentLengthCacheable4Hours"; - const std::string kCacheable24HoursHistogramName = - "Net.HttpContentLengthCacheable24Hours"; - const int64_t kResponseContentLength = 100; - const int64_t kOriginalContentLength = 200; + Init(USE_INSECURE_PROXY, false); base::HistogramTester histogram_tester; @@ -825,7 +1006,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { base::Int64ToString(kOriginalContentLength) + "\r\n\r\n"; std::unique_ptr<net::URLRequest> fake_request(FetchURLRequest( - GURL(kTestURL), nullptr, response_headers, kResponseContentLength)); + GURL(kTestURL), nullptr, response_headers, kResponseContentLength, 0)); fake_request->SetLoadFlags(fake_request->load_flags() | net::LOAD_MAIN_FRAME_DEPRECATED); @@ -842,6 +1023,12 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { kOriginalContentLength - kResponseContentLength, 1); histogram_tester.ExpectUniqueSample(kReceivedHistogramName, kResponseContentLength, 1); + histogram_tester.ExpectUniqueSample(kReceivedInsecureHistogramName, + kResponseContentLength, 1); + histogram_tester.ExpectTotalCount(kReceivedSecureHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedVideoHistogramName, 0); + histogram_tester.ExpectUniqueSample(kReceivedInsecureHistogramName, + kResponseContentLength, 1); histogram_tester.ExpectUniqueSample(kOriginalHistogramName, kOriginalContentLength, 1); histogram_tester.ExpectUniqueSample( @@ -902,7 +1089,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { config()->ShouldEnableLoFi(*fake_request.get())); fake_request = (FetchURLRequest(GURL(kTestURL), nullptr, response_headers, - kResponseContentLength)); + kResponseContentLength, 0)); fake_request->SetLoadFlags(fake_request->load_flags() | net::LOAD_MAIN_FRAME_DEPRECATED); @@ -932,8 +1119,57 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { } } +TEST_F(DataReductionProxyNetworkDelegateTest, NetVideoHistograms) { + Init(USE_INSECURE_PROXY, false); + + base::HistogramTester histogram_tester; + + // Check video + std::string video_response_headers = + "HTTP/1.1 200 OK\r\n" + "Date: Wed, 28 Nov 2007 09:40:09 GMT\r\n" + "Expires: Mon, 24 Nov 2014 12:45:26 GMT\r\n" + "Content-Type: video/mp4\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "x-original-content-length: " + + base::Int64ToString(kOriginalContentLength) + "\r\n\r\n"; + + FetchURLRequest(GURL(kTestURL), nullptr, video_response_headers, + kResponseContentLength, 0); + + histogram_tester.ExpectUniqueSample(kReceivedInsecureHistogramName, + kResponseContentLength, 1); + histogram_tester.ExpectTotalCount(kReceivedSecureHistogramName, 0); + histogram_tester.ExpectUniqueSample(kReceivedVideoHistogramName, + kResponseContentLength, 1); +} + +TEST_F(DataReductionProxyNetworkDelegateTest, NetSSLHistograms) { + Init(BYPASS_PROXY, false); + + base::HistogramTester histogram_tester; + + // Check https + std::string secure_response_headers = + "HTTP/1.1 200 OK\r\n" + "Date: Wed, 28 Nov 2007 09:40:09 GMT\r\n" + "Expires: Mon, 24 Nov 2014 12:45:26 GMT\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "x-original-content-length: " + + base::Int64ToString(kOriginalContentLength) + "\r\n\r\n"; + + mock_socket_factory()->AddSSLSocketDataProvider(ssl_socket_data_provider()); + FetchURLRequest(GURL(kSecureTestURL), nullptr, secure_response_headers, + kResponseContentLength, 0); + + histogram_tester.ExpectTotalCount(kReceivedInsecureHistogramName, 0); + histogram_tester.ExpectUniqueSample(kReceivedSecureHistogramName, + kResponseContentLength, 1); + histogram_tester.ExpectTotalCount(kReceivedVideoHistogramName, 0); +} + TEST_F(DataReductionProxyNetworkDelegateTest, OnCompletedInternalLoFi) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); // Enable Lo-Fi. const struct { bool lofi_response; @@ -953,15 +1189,17 @@ TEST_F(DataReductionProxyNetworkDelegateTest, OnCompletedInternalLoFi) { response_headers += "Chrome-Proxy-Content-Transform: empty-image\r\n"; response_headers += "\r\n"; - FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140); - + auto request = + FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140, 0); + EXPECT_EQ(tests[i].lofi_response, + DataReductionProxyData::GetData(*request)->lofi_received()); VerifyDidNotifyLoFiResponse(tests[i].lofi_response); } } TEST_F(DataReductionProxyNetworkDelegateTest, TestLoFiTransformationTypeHistogram) { - Init(false, false); + Init(USE_INSECURE_PROXY, false); const char kLoFiTransformationTypeHistogram[] = "DataReductionProxy.LoFi.TransformationType"; base::HistogramTester histogram_tester; @@ -969,7 +1207,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, net::HttpRequestHeaders request_headers; request_headers.SetHeader("chrome-proxy-accept-transform", "lite-page"); lofi_decider()->ignore_is_using_data_reduction_proxy_check(); - FetchURLRequest(GURL(kTestURL), &request_headers, std::string(), 140); + FetchURLRequest(GURL(kTestURL), &request_headers, std::string(), 140, 0); histogram_tester.ExpectBucketCount(kLoFiTransformationTypeHistogram, NO_TRANSFORMATION_LITE_PAGE_REQUESTED, 1); @@ -982,7 +1220,9 @@ TEST_F(DataReductionProxyNetworkDelegateTest, "x-original-content-length: 200\r\n"; response_headers += "\r\n"; - FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140); + auto request = + FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140, 0); + EXPECT_TRUE(DataReductionProxyData::GetData(*request)->lite_page_received()); histogram_tester.ExpectBucketCount(kLoFiTransformationTypeHistogram, LITE_PAGE, 1); @@ -992,7 +1232,9 @@ TEST_F(DataReductionProxyNetworkDelegateTest, // disabled globally. TEST_F(DataReductionProxyNetworkDelegateTest, BrotliAdvertisement_BrotliDisabled) { - Init(true /* use_secure_proxy */, false /* enable_brotli_globally */); + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + + ReadBrotliFile(); std::string response_headers = "HTTP/1.1 200 OK\r\n" @@ -1012,7 +1254,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, // is fetched from an insecure proxy. TEST_F(DataReductionProxyNetworkDelegateTest, BrotliAdvertisementInsecureProxy) { - Init(false /* use_secure_proxy */, true /* enable_brotli_globally */); + Init(USE_INSECURE_PROXY, true /* enable_brotli_globally */); std::string response_headers = "HTTP/1.1 200 OK\r\n" "Content-Length: 140\r\n" @@ -1025,7 +1267,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, // Use secure sockets when fetching the request since Brotli is only enabled // for secure connections. std::unique_ptr<net::URLRequest> request( - FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140)); + FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140, 0)); EXPECT_EQ(140, request->received_response_content_length()); EXPECT_NE(0, request->GetTotalSentBytes()); EXPECT_NE(0, request->GetTotalReceivedBytes()); @@ -1038,7 +1280,7 @@ TEST_F(DataReductionProxyNetworkDelegateTest, // disabled via data reduction proxy field trial. TEST_F(DataReductionProxyNetworkDelegateTest, BrotliAdvertisementDisabledViaFieldTrial) { - Init(true /* use_secure_proxy */, true /* enable_brotli_globally */); + Init(USE_SECURE_PROXY, true /* enable_brotli_globally */); base::FieldTrialList field_trial_list(nullptr); ASSERT_TRUE(base::FieldTrialList::CreateFieldTrial( @@ -1060,14 +1302,14 @@ TEST_F(DataReductionProxyNetworkDelegateTest, // Test that Brotli is correctly added to the accept-encoding header when it is // enabled globally. TEST_F(DataReductionProxyNetworkDelegateTest, BrotliAdvertisement) { - Init(true /* use_secure_proxy */, true /* enable_brotli_globally */); + Init(USE_SECURE_PROXY, true /* enable_brotli_globally */); std::string response_headers = "HTTP/1.1 200 OK\r\n" - "Content-Length: 140\r\n" "Via: 1.1 Chrome-Compression-Proxy\r\n" "x-original-content-length: 200\r\n" "Cache-Control: max-age=1200\r\n" + "Content-Encoding: br\r\n" "Vary: accept-encoding\r\n"; response_headers += "\r\n"; @@ -1075,6 +1317,102 @@ TEST_F(DataReductionProxyNetworkDelegateTest, BrotliAdvertisement) { FetchURLRequestAndVerifyBrotli(nullptr, response_headers, true, true); } +TEST_F(DataReductionProxyNetworkDelegateTest, IncrementingMainFramePageId) { + // This is unaffacted by brotil and insecure proxy. + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + + io_data()->request_options()->SetSecureSession("new-session"); + + FetchURLRequestAndVerifyPageIdDirective("pid=1", false); + + FetchURLRequestAndVerifyPageIdDirective("pid=2", false); + + FetchURLRequestAndVerifyPageIdDirective("pid=3", false); +} + +TEST_F(DataReductionProxyNetworkDelegateTest, ResetSessionResetsId) { + // This is unaffacted by brotil and insecure proxy. + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + + io_data()->request_options()->SetSecureSession("new-session"); + + FetchURLRequestAndVerifyPageIdDirective("pid=1", false); + + io_data()->request_options()->SetSecureSession("new-session-2"); + + FetchURLRequestAndVerifyPageIdDirective("pid=1", false); +} + +TEST_F(DataReductionProxyNetworkDelegateTest, SubResourceNoPageId) { + // This is unaffacted by brotil and insecure proxy. + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + io_data()->request_options()->SetSecureSession("new-session"); + FetchURLRequestAndVerifyPageIdDirective(std::string(), false); +} + +TEST_F(DataReductionProxyNetworkDelegateTest, RedirectSharePid) { + // This is unaffacted by brotil and insecure proxy. + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + + io_data()->request_options()->SetSecureSession("new-session"); + + FetchURLRequestAndVerifyPageIdDirective("pid=1", true); +} + +TEST_F(DataReductionProxyNetworkDelegateTest, + SessionChangeResetsPageIDOnRedirect) { + // This test calls directly into network delegate as it is difficult to mock + // state changing in between redirects within an URLRequest's lifetime. + + // This is unaffacted by brotil and insecure proxy. + Init(USE_INSECURE_PROXY, false /* enable_brotli_globally */); + net::ProxyInfo data_reduction_proxy_info; + std::string data_reduction_proxy; + base::TrimString(params()->DefaultOrigin(), "/", &data_reduction_proxy); + data_reduction_proxy_info.UseNamedProxy(data_reduction_proxy); + + std::unique_ptr<net::URLRequest> request = context()->CreateRequest( + GURL(kTestURL), net::RequestPriority::IDLE, nullptr); + request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); + io_data()->request_options()->SetSecureSession("fake-session"); + + net::HttpRequestHeaders headers; + net::ProxyRetryInfoMap proxy_retry_info; + + // Send a request and verify the page ID is 1. + network_delegate()->NotifyBeforeSendHeaders( + request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); + DataReductionProxyData* data = + DataReductionProxyData::GetData(*request.get()); + EXPECT_TRUE(data_reduction_proxy_info.is_http()); + EXPECT_EQ(1u, data->page_id().value()); + + // Send a second request and verify the page ID incremements. + request = context()->CreateRequest(GURL(kTestURL), net::RequestPriority::IDLE, + nullptr); + request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); + + network_delegate()->NotifyBeforeSendHeaders( + request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); + data = DataReductionProxyData::GetData(*request.get()); + EXPECT_EQ(2u, data->page_id().value()); + + // Verify that redirects are the same page ID. + network_delegate()->NotifyBeforeRedirect(request.get(), GURL(kTestURL)); + network_delegate()->NotifyBeforeSendHeaders( + request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); + data = DataReductionProxyData::GetData(*request.get()); + EXPECT_EQ(2u, data->page_id().value()); + + // Verify that redirects into a new session get a new page ID. + network_delegate()->NotifyBeforeRedirect(request.get(), GURL(kTestURL)); + io_data()->request_options()->SetSecureSession("new-session"); + network_delegate()->NotifyBeforeSendHeaders( + request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); + data = DataReductionProxyData::GetData(*request.get()); + EXPECT_EQ(1u, data->page_id().value()); +} + } // namespace } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc index 7b0d644f0b5..4d21e6666ed 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc @@ -6,9 +6,11 @@ #include <stdint.h> +#include "base/bind_helpers.h" #include "base/metrics/histogram_macros.h" #include "base/optional.h" #include "base/rand_util.h" +#include "base/threading/thread_task_runner_handle.h" #include "base/time/time.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.h" @@ -36,7 +38,8 @@ static const char kHistogramAttempted[] = // timing and data reduction proxy state. void AddDataToPageloadMetrics(const DataReductionProxyData& request_data, const DataReductionProxyPageLoadTiming& timing, - PageloadMetrics* request) { + PageloadMetrics* request, + bool opted_out) { request->set_session_key(request_data.session_key()); // For the timing events, any of them could be zero. Fill the message as a // best effort. @@ -92,6 +95,32 @@ void AddDataToPageloadMetrics(const DataReductionProxyData& request_data, request_data.effective_connection_type())); request->set_compressed_page_size_bytes(timing.network_bytes); request->set_original_page_size_bytes(timing.original_network_bytes); + + if (request_data.page_id()) { + request->set_page_id(request_data.page_id().value()); + } + + bool was_preview_shown = false; + if (request_data.lofi_received()) { + request->set_previews_type(PageloadMetrics_PreviewsType_LOFI); + was_preview_shown = true; + } else if (request_data.lite_page_received()) { + request->set_previews_type(PageloadMetrics_PreviewsType_LITE_PAGE); + was_preview_shown = true; + } else { + request->set_previews_type(PageloadMetrics_PreviewsType_NONE); + } + + if (!was_preview_shown || timing.app_background_occurred) { + request->set_previews_opt_out(PageloadMetrics_PreviewsOptOut_UNKNOWN); + return; + } + + if (opted_out) { + request->set_previews_opt_out(PageloadMetrics_PreviewsOptOut_OPT_OUT); + return; + } + request->set_previews_opt_out(PageloadMetrics_PreviewsOptOut_NON_OPT_OUT); } // Adds |current_time| as the metrics sent time to |request_data|, and returns @@ -115,6 +144,7 @@ DataReductionProxyPingbackClient::DataReductionProxyPingbackClient( pingback_reporting_fraction_(0.0) {} DataReductionProxyPingbackClient::~DataReductionProxyPingbackClient() { + DCHECK(opt_outs_.empty()); DCHECK(thread_checker_.CalledOnValidThread()); } @@ -137,8 +167,18 @@ void DataReductionProxyPingbackClient::SendPingback( UMA_HISTOGRAM_BOOLEAN(kHistogramAttempted, send_pingback); if (!send_pingback) return; + + bool opted_out = false; + if (request_data.page_id()) { + auto opt_out = opt_outs_.find(NavigationID(request_data.page_id().value(), + request_data.session_key())); + opted_out = opt_out != opt_outs_.end(); + if (opted_out) + opt_outs_.erase(opt_out); + } + PageloadMetrics* pageload_metrics = metrics_request_.add_pageloads(); - AddDataToPageloadMetrics(request_data, timing, pageload_metrics); + AddDataToPageloadMetrics(request_data, timing, pageload_metrics, opted_out); if (current_fetcher_.get()) return; DCHECK_EQ(1, metrics_request_.pageloads_size()); @@ -187,4 +227,26 @@ void DataReductionProxyPingbackClient::SetPingbackReportingFraction( pingback_reporting_fraction_ = pingback_reporting_fraction; } +void DataReductionProxyPingbackClient::AddOptOut( + const NavigationID& navigation_id) { + opt_outs_.emplace(navigation_id); +} + +void DataReductionProxyPingbackClient::ClearNavigationKeySync( + const NavigationID& navigation_id) { + opt_outs_.erase(navigation_id); +} + +void DataReductionProxyPingbackClient::ClearNavigationKeyAsync( + const NavigationID& navigation_id) { + base::ThreadTaskRunnerHandle::Get()->PostTask( + FROM_HERE, + base::Bind(&DataReductionProxyPingbackClient::ClearNavigationKeySync, + base::Unretained(this), navigation_id)); +} + +size_t DataReductionProxyPingbackClient::OptOutsSizeForTesting() const { + return opt_outs_.size(); +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.h index a91a44cf0d9..1f437b6a85c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.h @@ -5,8 +5,12 @@ #ifndef COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_PINGBACK_CLIENT_H_ #define COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_PINGBACK_CLIENT_H_ +#include <stdint.h> + #include <memory> +#include <set> #include <string> +#include <utility> #include "base/macros.h" #include "base/threading/thread_checker.h" @@ -27,6 +31,8 @@ namespace data_reduction_proxy { class DataReductionProxyData; struct DataReductionProxyPageLoadTiming; +using NavigationID = std::pair<uint64_t, std::string>; + // Manages pingbacks about page load timing information to the data saver proxy // server. This class is not thread safe. class DataReductionProxyPingbackClient : public net::URLFetcherDelegate { @@ -46,6 +52,17 @@ class DataReductionProxyPingbackClient : public net::URLFetcherDelegate { // call to SendPingback. void SetPingbackReportingFraction(float pingback_reporting_fraction); + // Adds an opt out for |tab_identifier_key| for a data saver |page_id|. An opt + // out occurs when users dismiss the preview in favor of the full page. + void AddOptOut(const NavigationID& navigation_id); + + // Removes any stored data associated with |tab_identifier_key| in a task that + // runs later. + void ClearNavigationKeyAsync(const NavigationID& navigation_id); + + // The total number of pending loads being tracked due to opt outs. + size_t OptOutsSizeForTesting() const; + protected: // Generates a float in the range [0, 1). Virtualized in testing. virtual float GenerateRandomFloat() const; @@ -66,6 +83,9 @@ class DataReductionProxyPingbackClient : public net::URLFetcherDelegate { // reset to an empty RecordPageloadMetricsRequest. void CreateFetcherForDataAndStart(); + // Removes any stored data associated with |tab_identifier_key|. + void ClearNavigationKeySync(const NavigationID& navigation_id); + net::URLRequestContextGetter* url_request_context_; // The URL for the data saver proxy's ping back service. @@ -80,6 +100,9 @@ class DataReductionProxyPingbackClient : public net::URLFetcherDelegate { // The probability of sending a pingback to the server. float pingback_reporting_fraction_; + // The map of tab identifier keys to page IDs. + std::set<NavigationID> opt_outs_; + base::ThreadChecker thread_checker_; DISALLOW_COPY_AND_ASSIGN(DataReductionProxyPingbackClient); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client_unittest.cc index 70d0992d008..65c03f3309a 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client_unittest.cc @@ -6,6 +6,7 @@ #include <stdint.h> +#include <list> #include <memory> #include <string> @@ -13,6 +14,7 @@ #include "base/memory/ptr_util.h" #include "base/message_loop/message_loop.h" #include "base/optional.h" +#include "base/run_loop.h" #include "base/test/histogram_tester.h" #include "base/time/time.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h" @@ -86,25 +88,7 @@ class TestDataReductionProxyPingbackClient class DataReductionProxyPingbackClientTest : public testing::Test { public: - DataReductionProxyPingbackClientTest() - : timing_( - base::Time::FromJsTime(1500) /* navigation_start */, - base::Optional<base::TimeDelta>( - base::TimeDelta::FromMilliseconds(1600)) /* response_start */, - base::Optional<base::TimeDelta>( - base::TimeDelta::FromMilliseconds(1700)) /* load_event_start */, - base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( - 1800)) /* first_image_paint */, - base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( - 1900)) /* first_contentful_paint */, - base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( - 2000)) /* experimental_first_meaningful_paint */, - base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( - 100)) /* parse_blocked_on_script_load_duration */, - base::Optional<base::TimeDelta>( - base::TimeDelta::FromMilliseconds(2000)) /* parse_stop */, - kBytes /* network_bytes */, - kBytesOriginal /* original_network_bytes */) {} + DataReductionProxyPingbackClientTest() {} TestDataReductionProxyPingbackClient* pingback_client() const { return pingback_client_.get(); @@ -116,31 +100,60 @@ class DataReductionProxyPingbackClientTest : public testing::Test { pingback_client_ = base::WrapUnique<TestDataReductionProxyPingbackClient>( new TestDataReductionProxyPingbackClient( request_context_getter_.get())); + page_id_ = 0u; } - void CreateAndSendPingback() { + void CreateAndSendPingback(bool lofi_received, + bool lite_page_received, + bool app_background_occurred) { + timing_ = base::MakeUnique<DataReductionProxyPageLoadTiming>( + base::Time::FromJsTime(1500) /* navigation_start */, + base::Optional<base::TimeDelta>( + base::TimeDelta::FromMilliseconds(1600)) /* response_start */, + base::Optional<base::TimeDelta>( + base::TimeDelta::FromMilliseconds(1700)) /* load_event_start */, + base::Optional<base::TimeDelta>( + base::TimeDelta::FromMilliseconds(1800)) /* first_image_paint */, + base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( + 1900)) /* first_contentful_paint */, + base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( + 2000)) /* experimental_first_meaningful_paint */, + base::Optional<base::TimeDelta>(base::TimeDelta::FromMilliseconds( + 100)) /* parse_blocked_on_script_load_duration */, + base::Optional<base::TimeDelta>( + base::TimeDelta::FromMilliseconds(2000)) /* parse_stop */, + kBytes /* network_bytes */, kBytesOriginal /* original_network_bytes */, + app_background_occurred /* app_background_occurred */); + DataReductionProxyData request_data; request_data.set_session_key(kSessionKey); request_data.set_request_url(GURL(kFakeURL)); request_data.set_effective_connection_type( net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); + request_data.set_lofi_received(lofi_received); + request_data.set_lite_page_received(lite_page_received); + request_data.set_page_id(page_id_); factory()->set_remove_fetcher_on_delete(true); - pingback_client()->SendPingback(request_data, timing_); + pingback_client()->SendPingback(request_data, *timing_); + page_id_++; } net::TestURLFetcherFactory* factory() { return &factory_; } - const DataReductionProxyPageLoadTiming& timing() { return timing_; } + const DataReductionProxyPageLoadTiming& timing() { return *timing_; } const base::HistogramTester& histogram_tester() { return histogram_tester_; } + uint64_t page_id() const { return page_id_; } + private: base::MessageLoopForIO message_loop_; scoped_refptr<net::URLRequestContextGetter> request_context_getter_; std::unique_ptr<TestDataReductionProxyPingbackClient> pingback_client_; net::TestURLFetcherFactory factory_; - DataReductionProxyPageLoadTiming timing_; + std::unique_ptr<DataReductionProxyPageLoadTiming> timing_; base::HistogramTester histogram_tester_; + uint64_t page_id_; }; TEST_F(DataReductionProxyPingbackClientTest, VerifyPingbackContent) { @@ -150,7 +163,10 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyPingbackContent) { pingback_client()->SetPingbackReportingFraction(1.0f); base::Time current_time = base::Time::UnixEpoch(); pingback_client()->set_current_time(current_time); - CreateAndSendPingback(); + uint64_t data_page_id = page_id(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); @@ -189,6 +205,12 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyPingbackContent) { EXPECT_EQ(kFakeURL, pageload_metrics.first_request_url()); EXPECT_EQ(kBytes, pageload_metrics.compressed_page_size_bytes()); EXPECT_EQ(kBytesOriginal, pageload_metrics.original_page_size_bytes()); + EXPECT_EQ(data_page_id, pageload_metrics.page_id()); + + EXPECT_EQ(PageloadMetrics_PreviewsType_NONE, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_UNKNOWN, + pageload_metrics.previews_opt_out()); EXPECT_EQ( PageloadMetrics_EffectiveConnectionType_EFFECTIVE_CONNECTION_TYPE_OFFLINE, @@ -206,13 +228,22 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyTwoPingbacksBatchedContent) { base::Time current_time = base::Time::UnixEpoch(); pingback_client()->set_current_time(current_time); // First pingback - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); // Two more pingbacks batched together. - CreateAndSendPingback(); + std::list<uint64_t> page_ids; + page_ids.push_back(page_id()); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 2); - CreateAndSendPingback(); + page_ids.push_back(page_id()); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 3); // Ignore the first pingback. @@ -261,6 +292,8 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyTwoPingbacksBatchedContent) { EXPECT_EQ(kFakeURL, pageload_metrics.first_request_url()); EXPECT_EQ(kBytes, pageload_metrics.compressed_page_size_bytes()); EXPECT_EQ(kBytesOriginal, pageload_metrics.original_page_size_bytes()); + EXPECT_EQ(page_ids.front(), pageload_metrics.page_id()); + page_ids.pop_front(); EXPECT_EQ( PageloadMetrics_EffectiveConnectionType_EFFECTIVE_CONNECTION_TYPE_OFFLINE, pageload_metrics.effective_connection_type()); @@ -276,9 +309,13 @@ TEST_F(DataReductionProxyPingbackClientTest, SendTwoPingbacks) { EXPECT_FALSE(factory()->GetFetcherByID(0)); pingback_client()->OverrideRandom(true, 0.5f); pingback_client()->SetPingbackReportingFraction(1.0f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 2); net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); @@ -297,7 +334,9 @@ TEST_F(DataReductionProxyPingbackClientTest, NoPingbackSent) { EXPECT_FALSE(factory()->GetFetcherByID(0)); pingback_client()->OverrideRandom(true, 0.5f); pingback_client()->SetPingbackReportingFraction(0.0f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, false, 1); histogram_tester().ExpectTotalCount(kHistogramSucceeded, 0); EXPECT_FALSE(factory()->GetFetcherByID(0)); @@ -311,7 +350,9 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyReportingBehvaior) { // pingback is created. pingback_client()->SetPingbackReportingFraction(0.5f); pingback_client()->OverrideRandom(true, 0.4f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); EXPECT_TRUE(test_fetcher); @@ -321,7 +362,9 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyReportingBehvaior) { // Verify that if the random number is greater than the reporting fraction, // the pingback is not created. pingback_client()->OverrideRandom(true, 0.6f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectBucketCount(kHistogramAttempted, false, 1); test_fetcher = factory()->GetFetcherByID(0); EXPECT_FALSE(test_fetcher); @@ -331,7 +374,9 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyReportingBehvaior) { // and the random number is zero, no pingback is sent. pingback_client()->SetPingbackReportingFraction(0.0f); pingback_client()->OverrideRandom(true, 0.0f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectBucketCount(kHistogramAttempted, false, 2); test_fetcher = factory()->GetFetcherByID(0); EXPECT_FALSE(test_fetcher); @@ -341,7 +386,9 @@ TEST_F(DataReductionProxyPingbackClientTest, VerifyReportingBehvaior) { data_reduction_proxy::switches::kEnableDataReductionProxyForcePingback); pingback_client()->SetPingbackReportingFraction(0.0f); pingback_client()->OverrideRandom(true, 1.0f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectBucketCount(kHistogramAttempted, true, 2); test_fetcher = factory()->GetFetcherByID(0); EXPECT_TRUE(test_fetcher); @@ -354,7 +401,9 @@ TEST_F(DataReductionProxyPingbackClientTest, FailedPingback) { EXPECT_FALSE(factory()->GetFetcherByID(0)); pingback_client()->OverrideRandom(true, 0.5f); pingback_client()->SetPingbackReportingFraction(1.0f); - CreateAndSendPingback(); + CreateAndSendPingback(false /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); EXPECT_TRUE(test_fetcher); @@ -365,4 +414,166 @@ TEST_F(DataReductionProxyPingbackClientTest, FailedPingback) { histogram_tester().ExpectUniqueSample(kHistogramSucceeded, false, 1); } +TEST_F(DataReductionProxyPingbackClientTest, VerifyLoFiContentNoOptOut) { + Init(); + EXPECT_FALSE(factory()->GetFetcherByID(0)); + pingback_client()->OverrideRandom(true, 0.5f); + pingback_client()->SetPingbackReportingFraction(1.0f); + base::Time current_time = base::Time::UnixEpoch(); + pingback_client()->set_current_time(current_time); + CreateAndSendPingback(true /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); + histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); + net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); + EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); + RecordPageloadMetricsRequest batched_request; + batched_request.ParseFromString(test_fetcher->upload_data()); + EXPECT_EQ(batched_request.pageloads_size(), 1); + PageloadMetrics pageload_metrics = batched_request.pageloads(0); + EXPECT_EQ(PageloadMetrics_PreviewsType_LOFI, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_NON_OPT_OUT, + pageload_metrics.previews_opt_out()); + + test_fetcher->delegate()->OnURLFetchComplete(test_fetcher); + EXPECT_FALSE(factory()->GetFetcherByID(0)); +} + +TEST_F(DataReductionProxyPingbackClientTest, VerifyLoFiContentOptOut) { + Init(); + EXPECT_FALSE(factory()->GetFetcherByID(0)); + pingback_client()->OverrideRandom(true, 0.5f); + pingback_client()->SetPingbackReportingFraction(1.0f); + base::Time current_time = base::Time::UnixEpoch(); + pingback_client()->set_current_time(current_time); + pingback_client()->AddOptOut(NavigationID(page_id(), kSessionKey)); + CreateAndSendPingback(true /* lofi_received */, + false /* lite_page_received */, + false /* app_background_occurred */); + histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); + net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); + EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); + RecordPageloadMetricsRequest batched_request; + batched_request.ParseFromString(test_fetcher->upload_data()); + EXPECT_EQ(batched_request.pageloads_size(), 1); + PageloadMetrics pageload_metrics = batched_request.pageloads(0); + EXPECT_EQ(PageloadMetrics_PreviewsType_LOFI, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_OPT_OUT, + pageload_metrics.previews_opt_out()); + + test_fetcher->delegate()->OnURLFetchComplete(test_fetcher); + EXPECT_FALSE(factory()->GetFetcherByID(0)); +} + +TEST_F(DataReductionProxyPingbackClientTest, VerifyLoFiContentBackground) { + Init(); + EXPECT_FALSE(factory()->GetFetcherByID(0)); + pingback_client()->OverrideRandom(true, 0.5f); + pingback_client()->SetPingbackReportingFraction(1.0f); + base::Time current_time = base::Time::UnixEpoch(); + pingback_client()->set_current_time(current_time); + pingback_client()->AddOptOut(NavigationID(page_id(), kSessionKey)); + CreateAndSendPingback(true /* lofi_received */, + false /* lite_page_received */, + true /* app_background_occurred */); + histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); + net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); + EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); + RecordPageloadMetricsRequest batched_request; + batched_request.ParseFromString(test_fetcher->upload_data()); + EXPECT_EQ(batched_request.pageloads_size(), 1); + PageloadMetrics pageload_metrics = batched_request.pageloads(0); + EXPECT_EQ(PageloadMetrics_PreviewsType_LOFI, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_UNKNOWN, + pageload_metrics.previews_opt_out()); + + test_fetcher->delegate()->OnURLFetchComplete(test_fetcher); + EXPECT_FALSE(factory()->GetFetcherByID(0)); +} + +TEST_F(DataReductionProxyPingbackClientTest, VerifyLitePageContent) { + Init(); + EXPECT_FALSE(factory()->GetFetcherByID(0)); + pingback_client()->OverrideRandom(true, 0.5f); + pingback_client()->SetPingbackReportingFraction(1.0f); + base::Time current_time = base::Time::UnixEpoch(); + pingback_client()->set_current_time(current_time); + pingback_client()->AddOptOut(NavigationID(page_id(), kSessionKey)); + CreateAndSendPingback(false /* lofi_received */, + true /* lite_page_received */, + false /* app_background_occurred */); + histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 1); + net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); + EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); + RecordPageloadMetricsRequest batched_request; + batched_request.ParseFromString(test_fetcher->upload_data()); + EXPECT_EQ(batched_request.pageloads_size(), 1); + PageloadMetrics pageload_metrics = batched_request.pageloads(0); + EXPECT_EQ(PageloadMetrics_PreviewsType_LITE_PAGE, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_OPT_OUT, + pageload_metrics.previews_opt_out()); + + test_fetcher->delegate()->OnURLFetchComplete(test_fetcher); + EXPECT_FALSE(factory()->GetFetcherByID(0)); +} + +TEST_F(DataReductionProxyPingbackClientTest, VerifyTwoLitePagePingbacks) { + Init(); + EXPECT_FALSE(factory()->GetFetcherByID(0)); + pingback_client()->OverrideRandom(true, 0.5f); + pingback_client()->SetPingbackReportingFraction(1.0f); + base::Time current_time = base::Time::UnixEpoch(); + pingback_client()->set_current_time(current_time); + pingback_client()->AddOptOut(NavigationID(page_id(), kSessionKey)); + CreateAndSendPingback(false /* lofi_received */, + true /* lite_page_received */, + false /* app_background_occurred */); + pingback_client()->AddOptOut(NavigationID(page_id(), kSessionKey)); + CreateAndSendPingback(false /* lofi_received */, + true /* lite_page_received */, + false /* app_background_occurred */); + histogram_tester().ExpectUniqueSample(kHistogramAttempted, true, 2); + net::TestURLFetcher* test_fetcher = factory()->GetFetcherByID(0); + EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); + RecordPageloadMetricsRequest batched_request; + batched_request.ParseFromString(test_fetcher->upload_data()); + EXPECT_EQ(batched_request.pageloads_size(), 1); + PageloadMetrics pageload_metrics = batched_request.pageloads(0); + EXPECT_EQ(PageloadMetrics_PreviewsType_LITE_PAGE, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_OPT_OUT, + pageload_metrics.previews_opt_out()); + test_fetcher->delegate()->OnURLFetchComplete(test_fetcher); + test_fetcher = factory()->GetFetcherByID(0); + EXPECT_EQ(test_fetcher->upload_content_type(), "application/x-protobuf"); + batched_request.ParseFromString(test_fetcher->upload_data()); + EXPECT_EQ(batched_request.pageloads_size(), 1); + pageload_metrics = batched_request.pageloads(0); + EXPECT_EQ(PageloadMetrics_PreviewsType_LITE_PAGE, + pageload_metrics.previews_type()); + EXPECT_EQ(PageloadMetrics_PreviewsOptOut_OPT_OUT, + pageload_metrics.previews_opt_out()); + test_fetcher->delegate()->OnURLFetchComplete(test_fetcher); + EXPECT_FALSE(factory()->GetFetcherByID(0)); +} + +TEST_F(DataReductionProxyPingbackClientTest, VerifyClearingPendingLoads) { + Init(); + EXPECT_FALSE(factory()->GetFetcherByID(0)); + pingback_client()->OverrideRandom(true, 0.5f); + pingback_client()->SetPingbackReportingFraction(1.0f); + base::Time current_time = base::Time::UnixEpoch(); + pingback_client()->set_current_time(current_time); + pingback_client()->AddOptOut(NavigationID(page_id(), kSessionKey)); + EXPECT_EQ(1u, pingback_client()->OptOutsSizeForTesting()); + pingback_client()->ClearNavigationKeyAsync( + NavigationID(page_id(), kSessionKey)); + base::RunLoop().RunUntilIdle(); + EXPECT_EQ(0u, pingback_client()->OptOutsSizeForTesting()); +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc index beabdfb174a..85671575bb3 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc @@ -39,8 +39,7 @@ class DataReductionProxyPrefsTest : public testing::Test { PrefService* pref_service) { ListPrefUpdate list(local_state_prefs(), pref_name); for (int64_t i = 0; i < 10L; ++i) { - list->Set(i, new base::StringValue( - base::Int64ToString(i + starting_value))); + list->Set(i, new base::Value(base::Int64ToString(i + starting_value))); } } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc index 3b66f062cfa..470a67597e5 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc @@ -7,6 +7,7 @@ #include "base/bind.h" #include "base/command_line.h" #include "base/single_thread_task_runner.h" +#include "base/strings/safe_sprintf.h" #include "base/strings/string_piece.h" #include "base/strings/string_split.h" #include "base/strings/string_tokenizer.h" @@ -46,6 +47,7 @@ const char kBuildNumberHeaderOption[] = "b"; const char kPatchNumberHeaderOption[] = "p"; const char kClientHeaderOption[] = "c"; const char kExperimentsOption[] = "exp"; +const char kPageIdOption[] = "pid"; // The empty version for the authentication protocol. Currently used by // Android webview. @@ -74,7 +76,8 @@ DataReductionProxyRequestOptions::DataReductionProxyRequestOptions( DataReductionProxyConfig* config) : client_(util::GetStringForClient(client)), use_assigned_credentials_(false), - data_reduction_proxy_config_(config) { + data_reduction_proxy_config_(config), + current_page_id_(0u) { DCHECK(data_reduction_proxy_config_); util::GetChromiumBuildAndPatch(version, &build_, &patch_); } @@ -150,8 +153,10 @@ void DataReductionProxyRequestOptions::RandBytes(void* output, } void DataReductionProxyRequestOptions::AddRequestHeader( - net::HttpRequestHeaders* request_headers) { + net::HttpRequestHeaders* request_headers, + base::Optional<uint64_t> page_id) { DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(!page_id || page_id.value() > 0u); base::Time now = Now(); // Authorization credentials must be regenerated if they are expired. if (!use_assigned_credentials_ && (now > credentials_expiration_time_)) @@ -164,6 +169,14 @@ void DataReductionProxyRequestOptions::AddRequestHeader( header_value += ", "; } header_value += header_value_; + + if (page_id) { + char page_id_buffer[16]; + if (base::strings::SafeSPrintf(page_id_buffer, "%x", page_id.value()) > 0) { + header_value += ", " + FormatOption(kPageIdOption, page_id_buffer); + } + } + request_headers->SetHeader(kChromeProxyHeader, header_value); } @@ -209,6 +222,7 @@ void DataReductionProxyRequestOptions::SetSecureSession( session_.clear(); credentials_.clear(); secure_session_ = secure_session; + ResetPageId(); // Force skipping of credential regeneration. It should be handled by the // caller. use_assigned_credentials_ = true; @@ -298,4 +312,14 @@ std::string DataReductionProxyRequestOptions::GetSessionKeyFromRequestHeaders( return ""; } +uint64_t DataReductionProxyRequestOptions::GeneratePageId() { + // Caller should not depend on order. + return ++current_page_id_; +} + +void DataReductionProxyRequestOptions::ResetPageId() { + // Caller should not depend on reset setting the page ID to 0. + current_page_id_ = 0; +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.h index b18dee44472..08700fd2d18 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.h @@ -13,6 +13,7 @@ #include "base/gtest_prod_util.h" #include "base/macros.h" +#include "base/optional.h" #include "base/strings/string16.h" #include "base/threading/thread_checker.h" #include "base/time/time.h" @@ -56,8 +57,10 @@ class DataReductionProxyRequestOptions { void Init(); // Adds a 'Chrome-Proxy' header to |request_headers| with the data reduction - // proxy authentication credentials. - void AddRequestHeader(net::HttpRequestHeaders* request_headers); + // proxy authentication credentials. |page_id| should only be non-empty for + // main frame requests. + void AddRequestHeader(net::HttpRequestHeaders* request_headers, + base::Optional<uint64_t> page_id); // Stores the supplied key and sets up credentials suitable for authenticating // with the data reduction proxy. @@ -81,6 +84,9 @@ class DataReductionProxyRequestOptions { std::string GetSessionKeyFromRequestHeaders( const net::HttpRequestHeaders& request_headers) const; + // Creates and returns a new unique page ID (unique per session). + uint64_t GeneratePageId(); + protected: // Returns a UTF16 string that's the hash of the configured authentication // |key| and |salt|. Returns an empty UTF16 string if no key is configured or @@ -106,6 +112,11 @@ class DataReductionProxyRequestOptions { FRIEND_TEST_ALL_PREFIXES(DataReductionProxyRequestOptionsTest, AuthHashForSalt); + // Resets the page ID for a new session. + // TODO(ryansturm): Create a session object to store this and other data saver + // session info. crbug.com/709624 + void ResetPageId(); + // Updates the value of the experiments to be run and regenerate the header if // necessary. void UpdateExperiments(); @@ -152,6 +163,9 @@ class DataReductionProxyRequestOptions { // Must outlive |this|. DataReductionProxyConfig* data_reduction_proxy_config_; + // The page identifier that was last generated for data saver proxy server. + uint64_t current_page_id_; + // Enforce usage on the IO thread. base::ThreadChecker thread_checker_; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc index 40cc55ec0df..1e58b8a5452 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc @@ -39,11 +39,15 @@ const char kExpectedBuild[] = "2"; const char kExpectedPatch[] = "3"; const char kExpectedCredentials[] = "96bd72ec4a050ba60981743d41787768"; const char kExpectedSession[] = "0-1633771873-1633771873-1633771873"; +const char kPageId[] = "1"; +const uint64_t kPageIdValue = 1; const char kTestKey2[] = "test-key2"; const char kExpectedCredentials2[] = "c911fdb402f578787562cf7f00eda972"; const char kExpectedSession2[] = "0-1633771873-1633771873-1633771873"; const char kDataReductionProxyKey[] = "12345"; +const char kPageId2[] = "f"; +const uint64_t kPageIdValue2 = 15; const char kSecureSession[] = "TestSecureSessionKey"; } // namespace @@ -93,6 +97,7 @@ void SetHeaderExpectations(const std::string& session, const std::string& client, const std::string& build, const std::string& patch, + const std::string& page_id, const std::vector<std::string> experiments, std::string* expected_header) { std::vector<std::string> expected_options; @@ -123,6 +128,10 @@ void SetHeaderExpectations(const std::string& session, expected_options.push_back( std::string(kExperimentsOption) + "=" + experiment); } + + EXPECT_FALSE(page_id.empty()); + expected_options.push_back("pid=" + page_id); + if (!expected_options.empty()) *expected_header = base::JoinString(expected_options, ", "); } @@ -153,10 +162,11 @@ class DataReductionProxyRequestOptionsTest : public testing::Test { return request_options_.get(); } - void VerifyExpectedHeader(const std::string& expected_header) { + void VerifyExpectedHeader(const std::string& expected_header, + uint64_t page_id) { test_context_->RunUntilIdle(); net::HttpRequestHeaders headers; - request_options_->AddRequestHeader(&headers); + request_options_->AddRequestHeader(&headers, page_id); if (expected_header.empty()) { EXPECT_FALSE(headers.HasHeader(kChromeProxyHeader)); return; @@ -184,13 +194,13 @@ TEST_F(DataReductionProxyRequestOptionsTest, AuthHashForSalt) { TEST_F(DataReductionProxyRequestOptionsTest, AuthorizationOnIOThread) { std::string expected_header; SetHeaderExpectations(kExpectedSession2, kExpectedCredentials2, std::string(), - kClientStr, kExpectedBuild, kExpectedPatch, + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, std::vector<std::string>(), &expected_header); std::string expected_header2; SetHeaderExpectations("86401-1633771873-1633771873-1633771873", "d7c1c34ef6b90303b01c48a6c1db6419", std::string(), - kClientStr, kExpectedBuild, kExpectedPatch, + kClientStr, kExpectedBuild, kExpectedPatch, kPageId2, std::vector<std::string>(), &expected_header2); CreateRequestOptions(kVersion); @@ -200,40 +210,40 @@ TEST_F(DataReductionProxyRequestOptionsTest, AuthorizationOnIOThread) { request_options()->SetKeyOnIO(kTestKey2); // Write headers. - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); // Fast forward 24 hours. The header should be the same. request_options()->set_offset(base::TimeDelta::FromSeconds(24 * 60 * 60)); - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); // Fast forward one more second. The header should be new. request_options()->set_offset(base::TimeDelta::FromSeconds(24 * 60 * 60 + 1)); - VerifyExpectedHeader(expected_header2); + VerifyExpectedHeader(expected_header2, kPageIdValue2); } TEST_F(DataReductionProxyRequestOptionsTest, AuthorizationIgnoresEmptyKey) { std::string expected_header; SetHeaderExpectations(kExpectedSession, kExpectedCredentials, std::string(), - kClientStr, kExpectedBuild, kExpectedPatch, + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, std::vector<std::string>(), &expected_header); CreateRequestOptions(kVersion); - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); // Now set an empty key. The auth handler should ignore that, and the key // remains |kTestKey|. request_options()->SetKeyOnIO(std::string()); - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); } TEST_F(DataReductionProxyRequestOptionsTest, SecureSession) { std::string expected_header; SetHeaderExpectations(std::string(), std::string(), kSecureSession, - kClientStr, kExpectedBuild, kExpectedPatch, + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, std::vector<std::string>(), &expected_header); CreateRequestOptions(kVersion); request_options()->SetSecureSession(kSecureSession); - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); } TEST_F(DataReductionProxyRequestOptionsTest, ParseExperiments) { @@ -245,11 +255,11 @@ TEST_F(DataReductionProxyRequestOptionsTest, ParseExperiments) { expected_experiments.push_back("\"foo,bar\""); std::string expected_header; SetHeaderExpectations(kExpectedSession, kExpectedCredentials, std::string(), - kClientStr, kExpectedBuild, kExpectedPatch, + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, expected_experiments, &expected_header); CreateRequestOptions(kVersion); - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); } TEST_F(DataReductionProxyRequestOptionsTest, ParseExperimentsFromFieldTrial) { @@ -312,11 +322,11 @@ TEST_F(DataReductionProxyRequestOptionsTest, ParseExperimentsFromFieldTrial) { expected_experiments.push_back(test.expected_experiment); SetHeaderExpectations(kExpectedSession, kExpectedCredentials, std::string(), - kClientStr, kExpectedBuild, kExpectedPatch, + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, expected_experiments, &expected_header); CreateRequestOptions(kVersion); - VerifyExpectedHeader(expected_header); + VerifyExpectedHeader(expected_header, kPageIdValue); } } @@ -356,4 +366,17 @@ TEST_F(DataReductionProxyRequestOptionsTest, GetSessionKeyFromRequestHeaders) { } } +TEST_F(DataReductionProxyRequestOptionsTest, PageIdIncrementing) { + CreateRequestOptions(kVersion); + DCHECK_EQ(1u, request_options()->GeneratePageId()); + DCHECK_EQ(2u, request_options()->GeneratePageId()); + DCHECK_EQ(3u, request_options()->GeneratePageId()); + + request_options()->SetSecureSession("blah"); + + DCHECK_EQ(1u, request_options()->GeneratePageId()); + DCHECK_EQ(2u, request_options()->GeneratePageId()); + DCHECK_EQ(3u, request_options()->GeneratePageId()); +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.cc index 6269291be06..b07e7dd8c8b 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.cc @@ -8,6 +8,7 @@ #include "base/bind.h" #include "base/command_line.h" +#include "base/feature_list.h" #include "base/metrics/histogram_macros.h" #include "base/time/clock.h" #include "base/time/default_clock.h" @@ -15,6 +16,7 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_pref_names.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h" @@ -103,6 +105,12 @@ void DataReductionProxySettings::InitDataReductionProxySettings( UpdateConfigValues(); RecordDataReductionInit(); data_reduction_proxy_service_->InitializeLoFiPrefs(); + + if (base::FeatureList::IsEnabled(features::kDataReductionSiteBreakdown) && + spdy_proxy_auth_enabled_.GetValue()) { + data_reduction_proxy_service_->compression_stats() + ->SetDataUsageReportingEnabled(true); + } } void DataReductionProxySettings::OnServiceInitialized() { @@ -139,9 +147,14 @@ bool DataReductionProxySettings::IsDataReductionProxyManaged() { void DataReductionProxySettings::SetDataReductionProxyEnabled(bool enabled) { DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(data_reduction_proxy_service_->compression_stats()); if (spdy_proxy_auth_enabled_.GetValue() != enabled) { spdy_proxy_auth_enabled_.SetValue(enabled); OnProxyEnabledPrefChange(); + if (base::FeatureList::IsEnabled(features::kDataReductionSiteBreakdown)) { + data_reduction_proxy_service_->compression_stats() + ->SetDataUsageReportingEnabled(enabled); + } } } @@ -152,6 +165,13 @@ int64_t DataReductionProxySettings::GetDataReductionLastUpdateTime() { data_reduction_proxy_service_->compression_stats()->GetLastUpdateTime(); } +void DataReductionProxySettings::ClearDataSavingStatistics() { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(data_reduction_proxy_service_->compression_stats()); + data_reduction_proxy_service_->compression_stats() + ->ClearDataSavingStatistics(); +} + int64_t DataReductionProxySettings::GetTotalHttpContentLengthSaved() { DCHECK(thread_checker_.CalledOnValidThread()); return data_reduction_proxy_service_->compression_stats() @@ -391,15 +411,4 @@ void DataReductionProxySettings::GetContentLengths( days, original_content_length, received_content_length, last_update_time); } -bool DataReductionProxySettings::UpdateDataSavings( - const std::string& data_usage_host, - int64_t data_used, - int64_t original_size) { - if (!IsDataReductionProxyEnabled()) - return false; - data_reduction_proxy_service_->compression_stats()->UpdateDataSavings( - data_usage_host, data_used, original_size); - return true; -} - } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h index ea4bcb767e4..b329f5f1cb3 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h @@ -17,7 +17,6 @@ #include "base/threading/thread_checker.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_service_observer.h" -#include "components/data_reduction_proxy/core/common/data_savings_recorder.h" #include "components/prefs/pref_member.h" #include "url/gurl.h" @@ -73,20 +72,14 @@ enum DataReductionSettingsEnabledAction { // Central point for configuring the data reduction proxy. // This object lives on the UI thread and all of its methods are expected to // be called from there. -class DataReductionProxySettings : public DataReductionProxyServiceObserver, - public DataSavingsRecorder { +class DataReductionProxySettings : public DataReductionProxyServiceObserver { public: - typedef base::Callback<bool(const std::string&, const std::string&)> - SyntheticFieldTrialRegistrationCallback; + using SyntheticFieldTrialRegistrationCallback = + base::Callback<bool(base::StringPiece, base::StringPiece)>; DataReductionProxySettings(); virtual ~DataReductionProxySettings(); - // DataSavingsRecorder implementation: - bool UpdateDataSavings(const std::string& data_usage_host, - int64_t data_used, - int64_t original_size) override; - // Initializes the Data Reduction Proxy with the name of the preference that // controls enabling it, profile prefs and a |DataReductionProxyIOData|. The // caller must ensure that all parameters remain alive for the lifetime of @@ -97,8 +90,6 @@ class DataReductionProxySettings : public DataReductionProxyServiceObserver, DataReductionProxyIOData* io_data, std::unique_ptr<DataReductionProxyService> data_reduction_proxy_service); - base::WeakPtr<DataReductionProxyCompressionStats> compression_stats(); - // Sets the |register_synthetic_field_trial_| callback and runs to register // the DataReductionProxyEnabled and the DataReductionProxyLoFiEnabled // synthetic field trial. @@ -146,6 +137,9 @@ class DataReductionProxySettings : public DataReductionProxyServiceObserver, // daily original and received content lengths. int64_t GetDataReductionLastUpdateTime(); + // Clears all data saving statistics. + void ClearDataSavingStatistics(); + // Returns the difference between the total original size of all HTTP content // received from the network and the actual size of the HTTP content received. int64_t GetTotalHttpContentLengthSaved(); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.cc index 744a0f3b939..90476c4db2b 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.cc @@ -62,9 +62,9 @@ void DataReductionProxySettingsTestBase::SetUp() { prefs::kDailyHttpReceivedContentLength); for (int64_t i = 0; i < kNumDaysInHistory; i++) { original_update->Insert( - 0, base::MakeUnique<base::StringValue>(base::Int64ToString(2 * i))); + 0, base::MakeUnique<base::Value>(base::Int64ToString(2 * i))); received_update->Insert( - 0, base::MakeUnique<base::StringValue>(base::Int64ToString(i))); + 0, base::MakeUnique<base::Value>(base::Int64ToString(i))); } last_update_time_ = base::Time::Now().LocalMidnight(); pref_service->SetInt64(prefs::kDailyHttpContentLengthLastUpdateDate, @@ -124,7 +124,7 @@ void DataReductionProxySettingsTestBase::CheckOnPrefChange( if (managed) { test_context_->pref_service()->SetManagedPref( test_context_->GetDataReductionProxyEnabledPrefName(), - new base::Value(enabled)); + base::MakeUnique<base::Value>(enabled)); } else { test_context_->SetDataReductionProxyEnabled(enabled); } @@ -140,10 +140,9 @@ void DataReductionProxySettingsTestBase::InitDataReductionProxy( test_context_->CreateDataReductionProxyService(settings_.get())); settings_->data_reduction_proxy_service()->SetIOData( test_context_->io_data()->GetWeakPtr()); - settings_->SetCallbackToRegisterSyntheticFieldTrial( - base::Bind(&DataReductionProxySettingsTestBase:: - SyntheticFieldTrialRegistrationCallback, - base::Unretained(this))); + settings_->SetCallbackToRegisterSyntheticFieldTrial(base::Bind( + &DataReductionProxySettingsTestBase::OnSyntheticFieldTrialRegistration, + base::Unretained(this))); test_context_->RunUntilIdle(); } @@ -154,4 +153,11 @@ void DataReductionProxySettingsTestBase::CheckDataReductionProxySyntheticTrial( synthetic_field_trials_["SyntheticDataReductionProxySetting"]); } +bool DataReductionProxySettingsTestBase::OnSyntheticFieldTrialRegistration( + base::StringPiece trial_name, + base::StringPiece group_name) { + synthetic_field_trials_[trial_name.as_string()] = group_name.as_string(); + return true; +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.h index 987de6337db..6846173a474 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_test_utils.h @@ -8,8 +8,10 @@ #include <map> #include <memory> #include <string> +#include <utility> #include "base/message_loop/message_loop.h" +#include "base/strings/string_piece.h" #include "base/time/clock.h" #include "base/time/time.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h" @@ -41,7 +43,6 @@ class DataReductionProxySettingsTestBase : public testing::Test { static void AddTestProxyToCommandLine(); DataReductionProxySettingsTestBase(); - DataReductionProxySettingsTestBase(bool promo_allowed); ~DataReductionProxySettingsTestBase() override; void AddProxyToCommandLine(); @@ -68,12 +69,10 @@ class DataReductionProxySettingsTestBase : public testing::Test { void InitWithStatisticsPrefs(); void InitDataReductionProxy(bool enabled_at_startup); void CheckDataReductionProxySyntheticTrial(bool enabled); - bool SyntheticFieldTrialRegistrationCallback(const std::string& trial_name, - const std::string& group_name) { - synthetic_field_trials_[trial_name] = group_name; - return true; - } + bool OnSyntheticFieldTrialRegistration(base::StringPiece trial_name, + base::StringPiece group_name); + protected: base::MessageLoopForIO message_loop_; std::unique_ptr<DataReductionProxyTestContext> test_context_; std::unique_ptr<DataReductionProxySettings> settings_; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc index d2ef4375ad6..8af34b7db5c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc @@ -66,17 +66,14 @@ TEST_F(DataReductionProxySettingsTest, TestIsProxyEnabledOrManaged) { test_context_->config()->UpdateConfigForTesting(false, true); EXPECT_FALSE(settings_->IsDataReductionProxyEnabled()); - EXPECT_FALSE(settings_->UpdateDataSavings(std::string(), 0, 0)); EXPECT_FALSE(settings_->IsDataReductionProxyManaged()); CheckOnPrefChange(true, true, false); EXPECT_TRUE(settings_->IsDataReductionProxyEnabled()); - EXPECT_TRUE(settings_->UpdateDataSavings(std::string(), 0, 0)); EXPECT_FALSE(settings_->IsDataReductionProxyManaged()); CheckOnPrefChange(true, true, true); EXPECT_TRUE(settings_->IsDataReductionProxyEnabled()); - EXPECT_TRUE(settings_->UpdateDataSavings(std::string(), 0, 0)); EXPECT_TRUE(settings_->IsDataReductionProxyManaged()); test_context_->RunUntilIdle(); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_tamper_detection.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_tamper_detection.cc index c28f9e0fa55..f8483575138 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_tamper_detection.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_tamper_detection.cc @@ -299,9 +299,8 @@ bool DataReductionProxyTamperDetection::ValidateViaHeader( base::StringPiece fingerprint, bool* has_chrome_proxy_via_header) const { bool has_intermediary; - *has_chrome_proxy_via_header = HasDataReductionProxyViaHeader( - response_headers_, - &has_intermediary); + *has_chrome_proxy_via_header = + HasDataReductionProxyViaHeader(*response_headers_, &has_intermediary); if (*has_chrome_proxy_via_header) return !has_intermediary; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.cc index eb267fb172a..f5a610b1dfd 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.cc @@ -287,6 +287,12 @@ DataStore::Status TestDataStore::Delete(base::StringPiece key) { return OK; } +DataStore::Status TestDataStore::RecreateDB() { + map_.clear(); + + return OK; +} + DataReductionProxyTestContext::Builder::Builder() : params_flags_(DataReductionProxyParams::kPromoAllowed), params_definitions_(TestDataReductionProxyParams::HAS_EVERYTHING), diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h index e42557f3a0e..a6eb4039e03 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h @@ -261,6 +261,8 @@ class TestDataStore : public data_reduction_proxy::DataStore { DataStore::Status Delete(base::StringPiece key) override; + DataStore::Status RecreateDB() override; + std::map<std::string, std::string>* map() { return &map_; } private: diff --git a/chromium/components/data_reduction_proxy/core/browser/data_store.cc b/chromium/components/data_reduction_proxy/core/browser/data_store.cc index eacf50ca185..782a1944b47 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_store.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_store.cc @@ -28,4 +28,8 @@ DataStore::Status DataStore::Delete(base::StringPiece key) { return DataStore::Status::OK; } +DataStore::Status DataStore::RecreateDB() { + return DataStore::Status::OK; +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_store.h b/chromium/components/data_reduction_proxy/core/browser/data_store.h index 1c254c2f137..f275da781e2 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_store.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_store.h @@ -36,6 +36,9 @@ class DataStore { virtual Status Delete(base::StringPiece key); + // Deletes the LevelDB and recreates it. + virtual Status RecreateDB(); + private: DISALLOW_COPY_AND_ASSIGN(DataStore); }; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_store_impl.cc b/chromium/components/data_reduction_proxy/core/browser/data_store_impl.cc index 76903bee1df..3c7c180fe96 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_store_impl.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_store_impl.cc @@ -117,7 +117,9 @@ DataStore::Status DataStoreImpl::OpenDB() { leveldb::Options options; options.create_if_missing = true; options.paranoid_checks = true; - options.reuse_logs = leveldb_env::kDefaultLogReuseOptionValue; + // Deletes to buckets not found are stored in the log. Use a new log so that + // these log entries are deleted. + options.reuse_logs = false; std::string db_name = profile_path_.Append(kDBName).AsUTF8Unsafe(); leveldb::DB* dbptr = nullptr; Status status = @@ -144,14 +146,13 @@ DataStore::Status DataStoreImpl::OpenDB() { return status; } -void DataStoreImpl::RecreateDB() { +DataStore::Status DataStoreImpl::RecreateDB() { DCHECK(sequence_checker_.CalledOnValidSequence()); - LOG(WARNING) << "Deleting corrupt Data Reduction Proxy LevelDB"; db_.reset(nullptr); base::DeleteFile(profile_path_.Append(kDBName), true); - OpenDB(); + return OpenDB(); } } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_store_impl.h b/chromium/components/data_reduction_proxy/core/browser/data_store_impl.h index b7aeca2bbd2..950de079ae0 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_store_impl.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_store_impl.h @@ -36,14 +36,14 @@ class DataStoreImpl : public DataStore { Status Delete(base::StringPiece key) override; + // Deletes the LevelDB and recreates it. This method is called if any DB call + // returns a |CORRUPTED| status or the database is cleared. + Status RecreateDB() override; + private: // Opens the underlying LevelDB for read and write. Status OpenDB(); - // Deletes the LevelDB and recreates it. This method is called if any DB call - // returns a |CORRUPTED| status. - void RecreateDB(); - // The underlying LevelDB used by this implementation. std::unique_ptr<leveldb::DB> db_; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_usage_store.cc b/chromium/components/data_reduction_proxy/core/browser/data_usage_store.cc index 98bcbbd0ed4..41f5fb65332 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_usage_store.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_usage_store.cc @@ -22,23 +22,23 @@ #include "base/time/time.h" #include "components/data_reduction_proxy/proto/data_store.pb.h" +namespace data_reduction_proxy { + namespace { + const char kCurrentBucketIndexKey[] = "current_bucket_index"; const char kBucketKeyPrefix[] = "data_usage_bucket:"; const int kMinutesInHour = 60; const int kMinutesInDay = 24 * kMinutesInHour; -// Time interval for each DataUsageBucket. -const int kDataUsageBucketLengthInMinutes = 15; -static_assert(kDataUsageBucketLengthInMinutes > 0, +static_assert(data_reduction_proxy::kDataUsageBucketLengthInMinutes > 0, "Length of time should be positive"); -static_assert(kMinutesInHour % kDataUsageBucketLengthInMinutes == 0, +static_assert(kMinutesInHour % + data_reduction_proxy::kDataUsageBucketLengthInMinutes == + 0, "kDataUsageBucketLengthMins must be a factor of kMinsInHour"); -// Number of days for which to maintain data usage history. -const int kDataUsageHistoryNumDays = 60; - // Total number of buckets persisted to DB. const int kNumDataUsageBuckets = kDataUsageHistoryNumDays * kMinutesInDay / kDataUsageBucketLengthInMinutes; @@ -65,8 +65,6 @@ base::Time BucketLowerBoundary(base::Time time) { } // namespace -namespace data_reduction_proxy { - DataUsageStore::DataUsageStore(DataStore* db) : db_(db), current_bucket_index_(-1) { sequence_checker_.DetachFromSequence(); @@ -154,10 +152,16 @@ void DataUsageStore::StoreCurrentDataUsageBucket( } void DataUsageStore::DeleteHistoricalDataUsage() { - for (int i = 0; i < kNumDataUsageBuckets; ++i) - db_->Delete(DbKeyForBucketIndex(i)); + std::string current_index_string; + DataStore::Status index_read_status = + db_->Get(kCurrentBucketIndexKey, ¤t_index_string); + + // If the index doesn't exist, then no buckets have been written and the + // data usage doesn't need to be deleted. + if (index_read_status != DataStore::Status::OK) + return; - db_->Delete(kCurrentBucketIndexKey); + db_->RecreateDB(); } void DataUsageStore::DeleteBrowsingHistory(const base::Time& start, diff --git a/chromium/components/data_reduction_proxy/core/browser/data_usage_store.h b/chromium/components/data_reduction_proxy/core/browser/data_usage_store.h index c346408374b..9ef594a55bd 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_usage_store.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_usage_store.h @@ -21,6 +21,12 @@ namespace data_reduction_proxy { class DataStore; class DataUsageBucket; +// Time interval for each DataUsageBucket. +constexpr int kDataUsageBucketLengthInMinutes = 15; + +// Number of days for which to maintain data usage history. +constexpr int kDataUsageHistoryNumDays = 60; + // Store for detailed data usage stats. Data usage from every // |kDataUsageBucketLengthMins| interval is stored in a DataUsageBucket. class DataUsageStore { diff --git a/chromium/components/data_reduction_proxy/core/common/BUILD.gn b/chromium/components/data_reduction_proxy/core/common/BUILD.gn index ba5918c21ed..d4a8642349b 100644 --- a/chromium/components/data_reduction_proxy/core/common/BUILD.gn +++ b/chromium/components/data_reduction_proxy/core/common/BUILD.gn @@ -17,6 +17,8 @@ template("common_tmpl") { "data_reduction_proxy_event_storage_delegate.h", "data_reduction_proxy_event_store.cc", "data_reduction_proxy_event_store.h", + "data_reduction_proxy_features.cc", + "data_reduction_proxy_features.h", "data_reduction_proxy_headers.cc", "data_reduction_proxy_headers.h", "data_reduction_proxy_page_load_timing.cc", @@ -31,7 +33,6 @@ template("common_tmpl") { "data_reduction_proxy_switches.h", "data_reduction_proxy_util.cc", "data_reduction_proxy_util.h", - "data_savings_recorder.h", "lofi_decider.h", "lofi_ui_service.h", "resource_type_provider.h", diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_bypass_type_list.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_bypass_type_list.h index 8010fbf87c2..0c67e60f5ec 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_bypass_type_list.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_bypass_type_list.h @@ -45,5 +45,8 @@ BYPASS_EVENT_TYPE(STATUS_503_HTTP_SERVICE_UNAVAILABLE, 9) // Bypass due to any network error. BYPASS_EVENT_TYPE(NETWORK_ERROR, 10) +// Bypass due to URL redirect cycle. +BYPASS_EVENT_TYPE(URL_REDIRECT_CYCLE, 11) + // This must always be last. -BYPASS_EVENT_TYPE(MAX, 11) +BYPASS_EVENT_TYPE(MAX, 12) diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_event_store.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_event_store.cc index 8f7db517adb..fd6ddc7bd21 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_event_store.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_event_store.cc @@ -12,6 +12,7 @@ #include "base/memory/ptr_util.h" #include "base/stl_util.h" #include "base/strings/string_number_conversions.h" +#include "base/strings/string_piece.h" #include "base/strings/string_util.h" #include "base/time/time.h" #include "base/values.h" @@ -51,13 +52,13 @@ const StringToConstant kDataReductionProxyBypassActionTypeTable[] = { }; std::string JoinListValueStrings(base::ListValue* list_value) { - std::vector<std::string> values; + std::vector<base::StringPiece> values; for (const auto& value : *list_value) { - std::string value_string; - if (!value->GetAsString(&value_string)) + base::StringPiece value_string; + if (!value.GetAsString(&value_string)) return std::string(); - values.push_back(std::move(value_string)); + values.push_back(value_string); } return base::JoinString(values, ";"); diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc new file mode 100644 index 00000000000..6e09db8dc84 --- /dev/null +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc @@ -0,0 +1,20 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h" + +namespace data_reduction_proxy { +namespace features { + +// Enables the Data Reduction Proxy menu item in the main menu rather than under +// Settings on Android. +const base::Feature kDataReductionMainMenu{"DataReductionProxyMainMenu", + base::FEATURE_DISABLED_BY_DEFAULT}; + +// Enables the site breakdown on the Data Reduction Proxy settings page. +const base::Feature kDataReductionSiteBreakdown{ + "DataReductionProxySiteBreakdown", base::FEATURE_DISABLED_BY_DEFAULT}; + +} // namespace features +} // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h new file mode 100644 index 00000000000..9913b055a51 --- /dev/null +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h @@ -0,0 +1,18 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_REDUCTION_PROXY_FEATURES_H_ +#define COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_REDUCTION_PROXY_FEATURES_H_ + +#include "base/feature_list.h" + +namespace data_reduction_proxy { +namespace features { + +extern const base::Feature kDataReductionMainMenu; +extern const base::Feature kDataReductionSiteBreakdown; + +} // namespace features +} // namespace data_reduction_proxy +#endif // COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_REDUCTION_PROXY_FEATURES_H_ diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc index 375416a081c..3a697aecaf8 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc @@ -20,6 +20,8 @@ #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h" #include "net/http/http_response_headers.h" #include "net/http/http_status_code.h" +#include "net/http/http_util.h" +#include "net/url_request/url_request.h" using base::StringPiece; using base::TimeDelta; @@ -40,6 +42,9 @@ const char kLitePageDirective[] = "lite-page"; const char kCompressedVideoDirective[] = "compressed-video"; const char kIdentityDirective[] = "identity"; +// The legacy Chrome-Proxy response header directive for LoFi images. +const char kLegacyChromeProxyLoFiResponseDirective[] = "q=low"; + const char kChromeProxyLitePageIngoreBlacklistDirective[] = "exp=ignore_preview_blacklist"; @@ -80,29 +85,37 @@ bool StartsWithActionPrefix(base::StringPiece header_value, // Returns true if the provided transform type is specified in the provided // Chrome-Proxy-Content-Transform header value. -bool IsPreviewTypeInHeaderValue(const std::string& header_value, - const std::string& transform_type) { - std::vector<std::string> tokens = - base::SplitString(base::ToLowerASCII(header_value), ";", - base::KEEP_WHITESPACE, base::SPLIT_WANT_ALL); - if (tokens.empty()) - return false; - std::string header_transform_type; - base::TrimWhitespaceASCII(tokens[0], base::TRIM_ALL, &header_transform_type); - return header_transform_type == transform_type; +bool IsPreviewTypeInHeaderValue(base::StringPiece header_value, + base::StringPiece transform_type) { + DCHECK_EQ(transform_type, base::ToLowerASCII(transform_type)); + + // The Chrome-Proxy-Content-Transform header consists of a single + // transformation type string followed by zero or more semicolon-delimited + // options, e.g. "empty-image", "empty-image;foo-option". + base::StringPiece token = base::TrimWhitespaceASCII( + header_value.substr(0, header_value.find(';')), base::TRIM_ALL); + return base::LowerCaseEqualsASCII(token, transform_type); } // Returns true if the provided transform type is specified in the // Chrome-Proxy-Content-Transform-Header. bool IsPreviewType(const net::HttpResponseHeaders& headers, - const std::string& transform_type) { - std::string header_value; - if (!headers.GetNormalizedHeader( - data_reduction_proxy::chrome_proxy_content_transform_header(), - &header_value)) { + base::StringPiece transform_type) { + std::string value; + return headers.EnumerateHeader(nullptr, kChromeProxyContentTransformHeader, + &value) && + IsPreviewTypeInHeaderValue(value, transform_type); +} + +// Returns true if there is a cycle in |url_chain|. +bool HasURLRedirectCycle(const std::vector<GURL>& url_chain) { + if (url_chain.size() <= 1) return false; - } - return IsPreviewTypeInHeaderValue(header_value, transform_type); + + // If the last entry occurs earlier in the |url_chain|, then very likely there + // is a redirect cycle. + return std::find(url_chain.rbegin() + 1, url_chain.rend(), + url_chain.back()) != url_chain.rend(); } } // namespace @@ -146,12 +159,27 @@ const char* if_heavy_qualifier() { } bool IsEmptyImagePreview(const net::HttpResponseHeaders& headers) { - return IsPreviewType(headers, kEmptyImageDirective); + return IsPreviewType(headers, kEmptyImageDirective) || + headers.HasHeaderValue(kChromeProxyHeader, + kLegacyChromeProxyLoFiResponseDirective); } -bool IsEmptyImagePreview(const std::string& content_transform_value) { - return IsPreviewTypeInHeaderValue(content_transform_value, - kEmptyImageDirective); +bool IsEmptyImagePreview(const std::string& content_transform_value, + const std::string& chrome_proxy_value) { + if (IsPreviewTypeInHeaderValue(content_transform_value, kEmptyImageDirective)) + return true; + + // Look for "q=low" in the "Chrome-Proxy" response header. + net::HttpUtil::ValuesIterator values(chrome_proxy_value.begin(), + chrome_proxy_value.end(), ','); + while (values.GetNext()) { + base::StringPiece value(values.value_begin(), values.value_end()); + if (base::LowerCaseEqualsASCII(value, + kLegacyChromeProxyLoFiResponseDirective)) { + return true; + } + } + return false; } bool IsLitePagePreview(const net::HttpResponseHeaders& headers) { @@ -175,14 +203,13 @@ bool GetDataReductionProxyActionValue(const net::HttpResponseHeaders* headers, return false; } -bool ParseHeadersAndSetBypassDuration(const net::HttpResponseHeaders* headers, +bool ParseHeadersAndSetBypassDuration(const net::HttpResponseHeaders& headers, base::StringPiece action_prefix, base::TimeDelta* bypass_duration) { - DCHECK(headers); size_t iter = 0; std::string value; - while (headers->EnumerateHeader(&iter, kChromeProxyHeader, &value)) { + while (headers.EnumerateHeader(&iter, kChromeProxyHeader, &value)) { if (StartsWithActionPrefix(value, action_prefix)) { int64_t seconds; if (!base::StringToInt64( @@ -203,7 +230,7 @@ bool ParseHeadersAndSetBypassDuration(const net::HttpResponseHeaders* headers, return false; } -bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders* headers, +bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders& headers, DataReductionProxyInfo* proxy_info) { DCHECK(proxy_info); @@ -240,8 +267,7 @@ bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders* headers, // reduction proxies. Unlike 'block', 'block-once' does not cause data // reduction proxies to be bypassed for an extended period of time; // 'block-once' only affects the retry of the current request. - if (headers->HasHeaderValue(kChromeProxyHeader, - kChromeProxyActionBlockOnce)) { + if (headers.HasHeaderValue(kChromeProxyHeader, kChromeProxyActionBlockOnce)) { proxy_info->bypass_all = true; proxy_info->mark_proxies_as_bad = false; proxy_info->bypass_duration = TimeDelta(); @@ -252,7 +278,7 @@ bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders* headers, return false; } -bool HasDataReductionProxyViaHeader(const net::HttpResponseHeaders* headers, +bool HasDataReductionProxyViaHeader(const net::HttpResponseHeaders& headers, bool* has_intermediary) { static const size_t kVersionSize = 4; static const char kDataReductionProxyViaValue[] = "Chrome-Compression-Proxy"; @@ -262,14 +288,14 @@ bool HasDataReductionProxyViaHeader(const net::HttpResponseHeaders* headers, // Case-sensitive comparison of |value|. Assumes the received protocol and the // space following it are always |kVersionSize| characters. E.g., // 'Via: 1.1 Chrome-Compression-Proxy' - while (headers->EnumerateHeader(&iter, "via", &value)) { + while (headers.EnumerateHeader(&iter, "via", &value)) { if (base::StringPiece(value).substr( kVersionSize, arraysize(kDataReductionProxyViaValue) - 1) == kDataReductionProxyViaValue) { if (has_intermediary) // We assume intermediary exists if there is another Via header after // the data reduction proxy's Via header. - *has_intermediary = !(headers->EnumerateHeader(&iter, "via", &value)); + *has_intermediary = !(headers.EnumerateHeader(&iter, "via", &value)); return true; } } @@ -278,9 +304,21 @@ bool HasDataReductionProxyViaHeader(const net::HttpResponseHeaders* headers, } DataReductionProxyBypassType GetDataReductionProxyBypassType( - const net::HttpResponseHeaders* headers, + const std::vector<GURL>& url_chain, + const net::HttpResponseHeaders& headers, DataReductionProxyInfo* data_reduction_proxy_info) { DCHECK(data_reduction_proxy_info); + + bool has_via_header = HasDataReductionProxyViaHeader(headers, nullptr); + + if (has_via_header && HasURLRedirectCycle(url_chain)) { + data_reduction_proxy_info->bypass_all = true; + data_reduction_proxy_info->mark_proxies_as_bad = false; + data_reduction_proxy_info->bypass_duration = base::TimeDelta(); + data_reduction_proxy_info->bypass_action = BYPASS_ACTION_TYPE_BLOCK_ONCE; + return BYPASS_EVENT_TYPE_URL_REDIRECT_CYCLE; + } + if (ParseHeadersForBypassInfo(headers, data_reduction_proxy_info)) { // A chrome-proxy response header is only present in a 502. For proper // reporting, this check must come before the 5xx checks below. @@ -302,20 +340,19 @@ DataReductionProxyBypassType GetDataReductionProxyBypassType( data_reduction_proxy_info->bypass_duration = GetDefaultBypassDuration(); // Fall back if a 500, 502 or 503 is returned. - if (headers->response_code() == net::HTTP_INTERNAL_SERVER_ERROR) + if (headers.response_code() == net::HTTP_INTERNAL_SERVER_ERROR) return BYPASS_EVENT_TYPE_STATUS_500_HTTP_INTERNAL_SERVER_ERROR; - if (headers->response_code() == net::HTTP_BAD_GATEWAY) + if (headers.response_code() == net::HTTP_BAD_GATEWAY) return BYPASS_EVENT_TYPE_STATUS_502_HTTP_BAD_GATEWAY; - if (headers->response_code() == net::HTTP_SERVICE_UNAVAILABLE) + if (headers.response_code() == net::HTTP_SERVICE_UNAVAILABLE) return BYPASS_EVENT_TYPE_STATUS_503_HTTP_SERVICE_UNAVAILABLE; // TODO(kundaji): Bypass if Proxy-Authenticate header value cannot be // interpreted by data reduction proxy. - if (headers->response_code() == net::HTTP_PROXY_AUTHENTICATION_REQUIRED && - !headers->HasHeader("Proxy-Authenticate")) { + if (headers.response_code() == net::HTTP_PROXY_AUTHENTICATION_REQUIRED && + !headers.HasHeader("Proxy-Authenticate")) { return BYPASS_EVENT_TYPE_MALFORMED_407; } - if (!HasDataReductionProxyViaHeader(headers, NULL) && - (headers->response_code() != net::HTTP_NOT_MODIFIED)) { + if (!has_via_header && (headers.response_code() != net::HTTP_NOT_MODIFIED)) { // A Via header might not be present in a 304. Since the goal of a 304 // response is to minimize information transfer, a sender in general // should not generate representation metadata other than Cache-Control, @@ -323,8 +360,8 @@ DataReductionProxyBypassType GetDataReductionProxyBypassType( // The proxy Via header might also not be present in a 4xx response. // Separate this case from other responses that are missing the header. - if (headers->response_code() >= net::HTTP_BAD_REQUEST && - headers->response_code() < net::HTTP_INTERNAL_SERVER_ERROR) { + if (headers.response_code() >= net::HTTP_BAD_REQUEST && + headers.response_code() < net::HTTP_INTERNAL_SERVER_ERROR) { // At this point, any 4xx response that is missing the via header // indicates an issue that is scoped to only the current request, so only // bypass the data reduction proxy for the current request. diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h index e86cb5af22a..e059d641d74 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h @@ -6,16 +6,17 @@ #define COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_REDUCTION_PROXY_HEADERS_H_ #include <string> +#include <vector> #include "base/macros.h" #include "base/strings/string_piece.h" #include "base/time/time.h" #include "net/proxy/proxy_service.h" -namespace net { +class GURL; +namespace net { class HttpResponseHeaders; - } // namespace net namespace data_reduction_proxy { @@ -110,7 +111,8 @@ bool IsEmptyImagePreview(const net::HttpResponseHeaders& headers); // Returns true if the provided value of the Chrome-Proxy-Content-Transform // response header that is provided in |content_transform_value| indicates that // an empty image has been provided. -bool IsEmptyImagePreview(const std::string& content_transform_value); +bool IsEmptyImagePreview(const std::string& content_transform_value, + const std::string& chrome_proxy_value); // Returns true if the Chrome-Proxy-Content-Transform response header indicates // that a lite page has been provided. @@ -122,7 +124,7 @@ bool IsLitePagePreview(const net::HttpResponseHeaders& headers); // (as specified in |ProxyList::UpdateRetryInfoOnFallback|) should be used. // If all available data reduction proxies should by bypassed, |bypass_all| is // set to true. |proxy_info| must be non-NULL. -bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders* headers, +bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders& headers, DataReductionProxyInfo* proxy_info); // Returns true if the response contains the data reduction proxy Via header @@ -130,14 +132,15 @@ bool ParseHeadersForBypassInfo(const net::HttpResponseHeaders* headers, // a Via header after the data reduction proxy, and to false otherwise. Used to // check the integrity of data reduction proxy responses and whether there are // other middleboxes between the data reduction proxy and the client. -bool HasDataReductionProxyViaHeader(const net::HttpResponseHeaders* headers, +bool HasDataReductionProxyViaHeader(const net::HttpResponseHeaders& headers, bool* has_intermediary); // Returns the reason why the Chrome proxy should be bypassed or not, and // populates |proxy_info| with information on how long to bypass if -// applicable. +// applicable. |url_chain| is the chain of URLs traversed by the request. DataReductionProxyBypassType GetDataReductionProxyBypassType( - const net::HttpResponseHeaders* headers, + const std::vector<GURL>& url_chain, + const net::HttpResponseHeaders& headers, DataReductionProxyInfo* proxy_info); // Searches for the specified Chrome-Proxy action, and if present saves its diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers_unittest.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers_unittest.cc index e2f88449332..2e10c016f9e 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers_unittest.cc @@ -18,6 +18,7 @@ #include "net/http/http_response_headers.h" #include "net/proxy/proxy_service.h" #include "testing/gtest/include/gtest/gtest.h" +#include "url/gurl.h" namespace data_reduction_proxy { @@ -76,6 +77,28 @@ TEST_F(DataReductionProxyHeadersTest, IsEmptyImagePreview) { "Another-Header: empty-image\n", false, }, + { + "HTTP/1.1 200 OK\n" + "Chrome-Proxy: q=low\n", + true, + }, + { + "HTTP/1.1 200 OK\n" + "Chrome-Proxy: foo=bar, Q=LOW\n", + true, + }, + { + "HTTP/1.1 200 OK\n" + "Chrome-Proxy-Content-Transform: q=low\n" + "Chrome-Proxy: empty-image\n", + false, + }, + { + "HTTP/1.1 200 OK\n" + "Chrome-Proxy-Content-Transform: foo\n" + "Chrome-Proxy: q=low\n", + true, + }, }; for (size_t i = 0; i < arraysize(tests); ++i) { std::string headers(tests[i].headers); @@ -88,30 +111,30 @@ TEST_F(DataReductionProxyHeadersTest, IsEmptyImagePreview) { TEST_F(DataReductionProxyHeadersTest, IsEmptyImagePreviewValue) { const struct { - const char* header; + const char* chrome_proxy_content_transform_header; + const char* chrome_proxy_header; bool expected_result; } tests[] = { - { - "foo", false, - }, - { - "", false, - }, - { - "empty-image", true, - }, - { - "empty-image;foo", true, - }, - { - "Empty-Image", true, - }, - { - "foo;empty-image", false, - }, + {"", "", false}, + {"foo", "", false}, + {"", "bar", false}, + {"foo", "bar", false}, + {"empty-image", "", true}, + {"empty-image;foo", "", true}, + {"Empty-Image", "", true}, + {"foo;empty-image", "", false}, + {"empty-image", "foo", true}, + {"foo;empty-image", "bar", false}, + {"", "q=low", true}, + {"foo", "q=low", true}, + {"foo", "bar, baz, Q=LOW ", true}, + {"empty-image", "q=low", true}, }; - for (size_t i = 0; i < arraysize(tests); ++i) - EXPECT_EQ(tests[i].expected_result, IsEmptyImagePreview(tests[i].header)); + for (const auto& test : tests) { + EXPECT_EQ(test.expected_result, + IsEmptyImagePreview(test.chrome_proxy_content_transform_header, + test.chrome_proxy_header)); + } } TEST_F(DataReductionProxyHeadersTest, IsLitePagePreview) { @@ -497,9 +520,8 @@ TEST_F(DataReductionProxyHeadersTest, GetProxyBypassInfo) { new net::HttpResponseHeaders(headers)); DataReductionProxyInfo data_reduction_proxy_info; - EXPECT_EQ( - tests[i].expected_result, - ParseHeadersForBypassInfo(parsed.get(), &data_reduction_proxy_info)); + EXPECT_EQ(tests[i].expected_result, + ParseHeadersForBypassInfo(*parsed, &data_reduction_proxy_info)); EXPECT_EQ(tests[i].expected_retry_delay, data_reduction_proxy_info.bypass_duration.InSeconds()); EXPECT_EQ(tests[i].expected_bypass_all, @@ -520,8 +542,7 @@ TEST_F(DataReductionProxyHeadersTest, ParseHeadersAndSetProxyInfo) { new net::HttpResponseHeaders(headers)); DataReductionProxyInfo data_reduction_proxy_info; - EXPECT_TRUE( - ParseHeadersForBypassInfo(parsed.get(), &data_reduction_proxy_info)); + EXPECT_TRUE(ParseHeadersForBypassInfo(*parsed, &data_reduction_proxy_info)); EXPECT_LE(60, data_reduction_proxy_info.bypass_duration.InSeconds()); EXPECT_GE(5 * 60, data_reduction_proxy_info.bypass_duration.InSeconds()); EXPECT_FALSE(data_reduction_proxy_info.bypass_all); @@ -612,11 +633,11 @@ TEST_F(DataReductionProxyHeadersTest, HasDataReductionProxyViaHeader) { bool has_chrome_proxy_via_header, has_intermediary; if (tests[i].ignore_intermediary) { has_chrome_proxy_via_header = - HasDataReductionProxyViaHeader(parsed.get(), NULL); + HasDataReductionProxyViaHeader(*parsed, NULL); } else { has_chrome_proxy_via_header = - HasDataReductionProxyViaHeader(parsed.get(), &has_intermediary); + HasDataReductionProxyViaHeader(*parsed, &has_intermediary); } EXPECT_EQ(tests[i].expected_result, has_chrome_proxy_via_header); if (has_chrome_proxy_via_header && !tests[i].ignore_intermediary) { @@ -768,8 +789,87 @@ TEST_F(DataReductionProxyHeadersTest, GetDataReductionProxyBypassEventType) { tests[i].in_tamper_detection_experiment ? "TamperDetection_Enabled" : "TamperDetection_Disabled"); - EXPECT_EQ(tests[i].expected_result, GetDataReductionProxyBypassType( - parsed.get(), &chrome_proxy_info)); + EXPECT_EQ(tests[i].expected_result, + GetDataReductionProxyBypassType(std::vector<GURL>(), *parsed, + &chrome_proxy_info)); + } +} + +TEST_F(DataReductionProxyHeadersTest, + GetDataReductionProxyBypassEventTypeURLRedirectCycle) { + const struct { + const char* headers; + std::vector<GURL> url_chain; + DataReductionProxyBypassType expected_result; + } tests[] = { + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{GURL("http://google.com/1"), + GURL("http://google.com/2"), + GURL("http://google.com/1")}, + BYPASS_EVENT_TYPE_URL_REDIRECT_CYCLE, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{ + GURL("http://google.com/1"), GURL("http://google.com/2"), + GURL("http://google.com/1"), GURL("http://google.com/2")}, + BYPASS_EVENT_TYPE_URL_REDIRECT_CYCLE, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{GURL("http://google.com/1")}, BYPASS_EVENT_TYPE_MAX, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{GURL("http://google.com/1"), + GURL("http://google.com/2")}, + BYPASS_EVENT_TYPE_MAX, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{GURL("http://google.com/1"), + GURL("http://google.com/2"), + GURL("http://google.com/3")}, + BYPASS_EVENT_TYPE_MAX, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{ + GURL("http://google.com/1"), GURL("http://google.com/2"), + GURL("http://google.com/3"), GURL("http://google.com/1")}, + BYPASS_EVENT_TYPE_URL_REDIRECT_CYCLE, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>{ + GURL("http://google.com/1"), GURL("http://google.com/2"), + GURL("http://google.com/1"), GURL("http://google.com/3")}, + BYPASS_EVENT_TYPE_MAX, + }, + { + "HTTP/1.1 200 OK\n" + "Via: 1.1 Chrome-Compression-Proxy\n", + std::vector<GURL>(), BYPASS_EVENT_TYPE_MAX, + }}; + + for (const auto& test : tests) { + std::string headers(test.headers); + HeadersToRaw(&headers); + scoped_refptr<net::HttpResponseHeaders> parsed( + new net::HttpResponseHeaders(headers)); + DataReductionProxyInfo chrome_proxy_info; + + EXPECT_EQ(test.expected_result, + GetDataReductionProxyBypassType(test.url_chain, *parsed, + &chrome_proxy_info)); } } diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.cc index 22a9b101b0b..13ff57d2f35 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.cc @@ -17,7 +17,8 @@ DataReductionProxyPageLoadTiming::DataReductionProxyPageLoadTiming( parse_blocked_on_script_load_duration, const base::Optional<base::TimeDelta>& parse_stop, int64_t network_bytes, - int64_t original_network_bytes) + int64_t original_network_bytes, + bool app_background_occurred) : navigation_start(navigation_start), response_start(response_start), load_event_start(load_event_start), @@ -28,7 +29,8 @@ DataReductionProxyPageLoadTiming::DataReductionProxyPageLoadTiming( parse_blocked_on_script_load_duration), parse_stop(parse_stop), network_bytes(network_bytes), - original_network_bytes(original_network_bytes) {} + original_network_bytes(original_network_bytes), + app_background_occurred(app_background_occurred) {} DataReductionProxyPageLoadTiming::DataReductionProxyPageLoadTiming( const DataReductionProxyPageLoadTiming& other) = default; diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.h index a1224225e41..84e1357dae1 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_page_load_timing.h @@ -26,7 +26,8 @@ struct DataReductionProxyPageLoadTiming { parse_blocked_on_script_load_duration, const base::Optional<base::TimeDelta>& parse_stop, int64_t network_bytes, - int64_t original_network_bytes); + int64_t original_network_bytes, + bool app_background_occurred); DataReductionProxyPageLoadTiming( const DataReductionProxyPageLoadTiming& other); @@ -57,6 +58,8 @@ struct DataReductionProxyPageLoadTiming { // The number of bytes that would have been served over the network if the // user were not using data reduction proxy, not including headers. const int64_t original_network_bytes; + // True when android app background occurred during the page load lifetime. + const bool app_background_occurred; }; } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc index ac08fb10e9d..44bb16c9f56 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc @@ -237,6 +237,14 @@ bool IsIncludedInQuicFieldTrial() { return true; } +bool IsQuicEnabledForNonCoreProxies() { + DCHECK(IsIncludedInQuicFieldTrial()); + std::map<std::string, std::string> params; + variations::GetVariationParams(GetQuicFieldTrialName(), ¶ms); + return GetStringValueForVariationParamWithDefaultValue( + params, "enable_quic_non_core_proxies", "false") == "true"; +} + const char* GetQuicFieldTrialName() { return kQuicFieldTrial; } diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h index 02b3d9cc19c..eab391b026e 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h @@ -122,6 +122,9 @@ bool WarnIfNoDataReductionProxy(); // proxy server as quic://proxy.googlezip.net. bool IsIncludedInQuicFieldTrial(); +// Returns true if QUIC is enabled for non core data reduction proxies. +bool IsQuicEnabledForNonCoreProxies(); + const char* GetQuicFieldTrialName(); // Returns true if zero RTT for QUIC is enabled. diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc index 20de8115b95..b045ae0ddd9 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc @@ -456,6 +456,44 @@ TEST_F(DataReductionProxyParamsTest, QuicFieldTrial) { } } +// Tests if the QUIC field trial |enable_quic_non_core_proxies| is set +// correctly. +TEST_F(DataReductionProxyParamsTest, QuicEnableNonCoreProxies) { + const struct { + std::string trial_group_name; + bool expected_enabled; + std::string enable_non_core_proxies; + bool expected_enable_non_core_proxies; + } tests[] = { + {"Enabled", true, "true", true}, + {"Enabled", true, "false", false}, + {"Enabled", true, std::string(), false}, + {"Control", false, "true", false}, + {"Disabled", false, "true", false}, + }; + + for (const auto& test : tests) { + variations::testing::ClearAllVariationParams(); + std::map<std::string, std::string> variation_params; + variation_params["enable_quic_non_core_proxies"] = + test.enable_non_core_proxies; + + ASSERT_TRUE(variations::AssociateVariationParams( + params::GetQuicFieldTrialName(), test.trial_group_name, + variation_params)); + + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetQuicFieldTrialName(), + test.trial_group_name); + + EXPECT_EQ(test.expected_enabled, params::IsIncludedInQuicFieldTrial()); + if (params::IsIncludedInQuicFieldTrial()) { + EXPECT_EQ(test.expected_enable_non_core_proxies, + params::IsQuicEnabledForNonCoreProxies()); + } + } +} + TEST_F(DataReductionProxyParamsTest, HoldbackEnabledFieldTrial) { const struct { std::string trial_group_name; diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc index 8e2caccccc3..82dc96a2074 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc @@ -80,10 +80,6 @@ const char kEnableDataReductionProxyLitePage[] = const char kEnableDataReductionProxyForcePingback[] = "enable-data-reduction-proxy-force-pingback"; -// If set, enables use of QUIC with non core data reduction proxies. -const char kDataReductionProxyEnableQuicOnNonCoreProxies[] = - "data-reduction-proxy-enable-quic-on-non-core-proxies"; - // Enables a 1 MB savings promo for the data reduction proxy. const char kEnableDataReductionProxySavingsPromo[] = "enable-data-reduction-proxy-savings-promo"; diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h index c3ab54b10b1..cbc063d6836 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h @@ -31,7 +31,6 @@ extern const char kEnableDataReductionProxyBypassWarning[]; extern const char kEnableDataReductionProxyCarrierTest[]; extern const char kEnableDataReductionProxyForcePingback[]; extern const char kEnableDataReductionProxyLitePage[]; -extern const char kDataReductionProxyEnableQuicOnNonCoreProxies[]; extern const char kEnableDataReductionProxySavingsPromo[]; } // namespace switches diff --git a/chromium/components/data_reduction_proxy/core/common/data_savings_recorder.h b/chromium/components/data_reduction_proxy/core/common/data_savings_recorder.h deleted file mode 100644 index a96cf5984be..00000000000 --- a/chromium/components/data_reduction_proxy/core/common/data_savings_recorder.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_SAVINGS_RECORDER_H_ -#define COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_SAVINGS_RECORDER_H_ - -#include <stdint.h> - -#include <string> - -namespace data_reduction_proxy { - -// An interface that will record data savings based on the amount of data used, -// the original size of the data, and the request type. -class DataSavingsRecorder { - public: - // Records detailed data usage. Assumes |data_used| is recorded already by - // previous handling of URLRequests. Also records daily data savings - // statistics to prefs and reports data savings UMA. |data_used| and - // |original_size| are measured in bytes. - // Returns true if data savings was recorded. - virtual bool UpdateDataSavings(const std::string& data_usage_host, - int64_t data_used, - int64_t original_size) = 0; -}; - -} // namespace data_reduction_proxy - -#endif // COMPONENTS_DATA_REDUCTION_PROXY_CORE_COMMON_DATA_SAVINGS_RECORDER_H_ diff --git a/chromium/components/data_reduction_proxy/proto/pageload_metrics.proto b/chromium/components/data_reduction_proxy/proto/pageload_metrics.proto index ab567c1b11a..1725604afae 100644 --- a/chromium/components/data_reduction_proxy/proto/pageload_metrics.proto +++ b/chromium/components/data_reduction_proxy/proto/pageload_metrics.proto @@ -34,6 +34,26 @@ message PageloadMetrics { EFFECTIVE_CONNECTION_TYPE_4G = 5; }; + // The various opt out states seen by server previews. + enum PreviewsOptOut { + // Set for non-previews navigations and app background navigations. + UNKNOWN = 0; + // Set for previews navigations that clicked "show original". + OPT_OUT = 1; + // Set for previews navigations that did not click "show original". + NON_OPT_OUT = 2; + } + + // The various server previews that can be shown. + enum PreviewsType { + // No server preview was applied. + NONE = 0; + // Image placeholders were used on the page. + LOFI = 1; + // The main resource was a lite page. + LITE_PAGE = 2; + } + // The session key used to load the page. optional string session_key = 1; // The time at which the first request of the pageload was made, according to @@ -72,4 +92,13 @@ message PageloadMetrics { // Time to first meaningful paint. This measure is unstable and will change // over time. optional Duration experimental_time_to_first_meaningful_paint = 15; + + // The unique identifier for the page load. + optional uint64 page_id = 16; + + // The opt out state of the page load. + optional PreviewsOptOut previews_opt_out = 17; + + // The previews type that was used on the page. + optional PreviewsType previews_type = 18; } |