diff options
Diffstat (limited to 'chromium/components/data_reduction_proxy')
55 files changed, 2534 insertions, 631 deletions
diff --git a/chromium/components/data_reduction_proxy/DEPS b/chromium/components/data_reduction_proxy/DEPS index 20847ff1716..d2649794511 100644 --- a/chromium/components/data_reduction_proxy/DEPS +++ b/chromium/components/data_reduction_proxy/DEPS @@ -2,6 +2,7 @@ include_rules = [ "+components/data_use_measurement/core", "+components/pref_registry", "+components/prefs", + "+components/previews", "+components/variations", "+crypto", "+google_apis", diff --git a/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.cc b/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.cc index f600474209e..ac945240865 100644 --- a/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.cc +++ b/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.cc @@ -180,22 +180,6 @@ void ContentLoFiDecider::RemoveAcceptTransformHeader( headers->RemoveHeader(chrome_proxy_accept_transform_header()); } -void ContentLoFiDecider::MaybeSetIgnorePreviewsBlacklistDirective( - net::HttpRequestHeaders* headers) const { - if (!headers || !params::AreLitePagesEnabledViaFlags() || - !IsLitePagePreviewRequested(*headers)) { - return; - } - std::string chrome_proxy_header_value; - headers->GetHeader(chrome_proxy_header(), &chrome_proxy_header_value); - headers->RemoveHeader(chrome_proxy_header()); - if (!chrome_proxy_header_value.empty()) - chrome_proxy_header_value += ", "; - chrome_proxy_header_value += - chrome_proxy_lite_page_ignore_blacklist_directive(); - headers->SetHeader(chrome_proxy_header(), chrome_proxy_header_value); -} - bool ContentLoFiDecider::ShouldRecordLoFiUMA( const net::URLRequest& request) const { const content::ResourceRequestInfo* request_info = @@ -212,4 +196,20 @@ bool ContentLoFiDecider::ShouldRecordLoFiUMA( params::IsIncludedInLoFiControlFieldTrial(); } +bool ContentLoFiDecider::IsClientLoFiImageRequest( + const net::URLRequest& request) const { + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(&request); + return request_info && + request_info->GetResourceType() == content::RESOURCE_TYPE_IMAGE && + (request_info->GetPreviewsState() & content::CLIENT_LOFI_ON); +} + +bool ContentLoFiDecider::IsClientLoFiAutoReloadRequest( + const net::URLRequest& request) const { + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(&request); + return request_info && + (request_info->GetPreviewsState() & content::CLIENT_LOFI_AUTO_RELOAD); +} } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.h b/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.h index 40f6dc32814..327278e9548 100644 --- a/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.h +++ b/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider.h @@ -38,9 +38,10 @@ class ContentLoFiDecider : public LoFiDecider { const net::HttpRequestHeaders& headers) const override; void RemoveAcceptTransformHeader( net::HttpRequestHeaders* headers) const override; - void MaybeSetIgnorePreviewsBlacklistDirective( - net::HttpRequestHeaders* headers) const override; bool ShouldRecordLoFiUMA(const net::URLRequest& request) const override; + bool IsClientLoFiImageRequest(const net::URLRequest& request) const override; + bool IsClientLoFiAutoReloadRequest( + const net::URLRequest& request) const override; private: DISALLOW_COPY_AND_ASSIGN(ContentLoFiDecider); diff --git a/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider_unittest.cc b/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider_unittest.cc index 4e5dcf9cd6a..d033a521d8f 100644 --- a/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider_unittest.cc +++ b/chromium/components/data_reduction_proxy/content/browser/content_lofi_decider_unittest.cc @@ -33,6 +33,7 @@ #include "net/proxy/proxy_info.h" #include "net/proxy/proxy_retry_info.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context.h" #include "net/url_request/url_request_test_util.h" @@ -110,8 +111,9 @@ class ContentLoFiDeciderTest : public testing::Test { std::unique_ptr<net::URLRequest> CreateRequest( bool is_main_frame, content::PreviewsState previews_state) { - std::unique_ptr<net::URLRequest> request = context_.CreateRequest( - GURL("http://www.google.com/"), net::IDLE, &delegate_); + std::unique_ptr<net::URLRequest> request = + context_.CreateRequest(GURL("http://www.google.com/"), net::IDLE, + &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); AllocateRequestInfoForTesting( request.get(), (is_main_frame ? content::RESOURCE_TYPE_MAIN_FRAME @@ -124,10 +126,10 @@ class ContentLoFiDeciderTest : public testing::Test { content::ResourceType resource_type, bool scheme_is_https, content::PreviewsState previews_state) { - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(GURL(scheme_is_https ? "https://www.google.com/" - : "http://www.google.com/"), - net::IDLE, &delegate_); + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + GURL(scheme_is_https ? "https://www.google.com/" + : "http://www.google.com/"), + net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); AllocateRequestInfoForTesting(request.get(), resource_type, previews_state); return request; } @@ -201,18 +203,6 @@ class ContentLoFiDeciderTest : public testing::Test { header_value.find(compressed_video_directive()) != std::string::npos); } - static void VerifyLitePageIgnoreBlacklistHeader( - bool expected_blacklist_directive_added, - const net::HttpRequestHeaders& headers) { - EXPECT_TRUE(headers.HasHeader(chrome_proxy_header())); - std::string header_value; - headers.GetHeader(chrome_proxy_header(), &header_value); - EXPECT_EQ(expected_blacklist_directive_added, - header_value.find( - chrome_proxy_lite_page_ignore_blacklist_directive()) != - std::string::npos); - } - protected: base::MessageLoopForIO message_loop_; net::TestURLRequestContext context_; @@ -256,8 +246,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiFlags) { NotifyBeforeSendHeaders(&headers, request.get(), true); VerifyLoFiHeader(false, false, headers); VerifyLitePageHeader(false, false, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); - // The Lo-Fi flag is "always-on", Lo-Fi is being used, and it's a main frame // request. Lo-Fi or lite page header should be added. command_line->AppendSwitchASCII( @@ -271,8 +259,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiFlags) { headers); VerifyLitePageHeader(tests[i].is_using_lite_page && tests[i].is_main_frame, false, headers); - VerifyLitePageIgnoreBlacklistHeader( - tests[i].is_using_lite_page && tests[i].is_main_frame, headers); DataReductionProxyData* data = DataReductionProxyData::GetData(*request); // |lofi_requested| should be set to false when Lo-Fi is enabled using // flags. @@ -287,7 +273,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiFlags) { VerifyLoFiHeader(!tests[i].is_using_lite_page, !tests[i].is_using_lofi, headers); VerifyLitePageHeader(false, false, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); // The Lo-Fi flag is "cellular-only" and Lo-Fi is being used. Lo-Fi header // should be added. @@ -299,7 +284,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiFlags) { VerifyLoFiHeader(!tests[i].is_using_lite_page, !tests[i].is_using_lofi, headers); VerifyLitePageHeader(false, false, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); data = DataReductionProxyData::GetData(*request); // |lofi_requested| should be set to false when Lo-Fi is enabled using // flags. @@ -315,7 +299,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiFlags) { VerifyLoFiHeader(!tests[i].is_using_lite_page, !tests[i].is_using_lofi, headers); VerifyLitePageHeader(false, false, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); data = DataReductionProxyData::GetData(*request); // |lofi_requested| should be set to false when Lo-Fi is enabled using // flags. @@ -391,7 +374,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiEnabledFieldTrial) { VerifyLoFiHeader(is_lofi_resource_type, !tests[i].is_using_lofi, headers); VerifyLitePageHeader(false, false, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); DataReductionProxyData* data = DataReductionProxyData::GetData(*request); EXPECT_EQ(tests[i].is_using_lofi, data->lofi_requested()) << i; } @@ -415,7 +397,6 @@ TEST_F(ContentLoFiDeciderTest, LoFiControlFieldTrial) { NotifyBeforeSendHeaders(&headers, request.get(), true); VerifyLoFiHeader(false, false, headers); VerifyLitePageHeader(false, false, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); DataReductionProxyData* data = DataReductionProxyData::GetData(*request); EXPECT_EQ(tests[i].is_using_lofi, data->lofi_requested()) << i; } @@ -443,7 +424,6 @@ TEST_F(ContentLoFiDeciderTest, LitePageFieldTrial) { VerifyLoFiHeader(false, false, headers); VerifyLitePageHeader(tests[i].is_main_frame, !tests[i].is_using_lite_page, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); DataReductionProxyData* data = DataReductionProxyData::GetData(*request); EXPECT_EQ(tests[i].is_using_lite_page, data->lofi_requested()) << i; } @@ -487,7 +467,6 @@ TEST_F(ContentLoFiDeciderTest, LitePageFieldTrialFallbackEnabled) { VerifyLitePageHeader(tests[i].is_main_frame, tests[i].is_main_frame && !tests[i].is_using_lite_page, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); DataReductionProxyData* data = DataReductionProxyData::GetData(*request); EXPECT_EQ(tests[i].is_using_lofi || tests[i].is_using_lite_page, data->lofi_requested()) @@ -527,7 +506,6 @@ TEST_F(ContentLoFiDeciderTest, LitePageFieldTrialFallbackDisabled) { VerifyLitePageHeader(tests[i].is_main_frame, tests[i].is_main_frame && !tests[i].is_using_lite_page, headers); - VerifyLitePageIgnoreBlacklistHeader(false, headers); DataReductionProxyData* data = DataReductionProxyData::GetData(*request); EXPECT_EQ(tests[i].is_using_lofi || tests[i].is_using_lite_page, data->lofi_requested()) @@ -747,40 +725,6 @@ TEST_F(ContentLoFiDeciderTest, RemoveAcceptTransformHeader) { EXPECT_FALSE(headers.HasHeader(chrome_proxy_accept_transform_header())); } -TEST_F(ContentLoFiDeciderTest, MaybeIgnoreBlacklist) { - base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); - command_line->InitFromArgv(command_line->argv()); - std::unique_ptr<data_reduction_proxy::ContentLoFiDecider> lofi_decider( - new data_reduction_proxy::ContentLoFiDecider()); - net::HttpRequestHeaders headers; - lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(&headers); - EXPECT_FALSE(headers.HasHeader(chrome_proxy_header())); - - headers.SetHeader(chrome_proxy_header(), "Foo"); - lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(&headers); - std::string header_value; - headers.GetHeader(chrome_proxy_header(), &header_value); - EXPECT_EQ("Foo", header_value); - - headers.RemoveHeader(chrome_proxy_header()); - command_line->AppendSwitch(switches::kEnableDataReductionProxyLitePage); - headers.SetHeader(chrome_proxy_accept_transform_header(), "empty-image"); - lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(&headers); - EXPECT_FALSE(headers.HasHeader(chrome_proxy_header())); - - headers.SetHeader(chrome_proxy_accept_transform_header(), "lite-page"); - lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(&headers); - EXPECT_TRUE(headers.HasHeader(chrome_proxy_header())); - headers.GetHeader(chrome_proxy_header(), &header_value); - EXPECT_EQ("exp=ignore_preview_blacklist", header_value); - - headers.SetHeader(chrome_proxy_header(), "Foo"); - lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(&headers); - EXPECT_TRUE(headers.HasHeader(chrome_proxy_header())); - headers.GetHeader(chrome_proxy_header(), &header_value); - EXPECT_EQ("Foo, exp=ignore_preview_blacklist", header_value); -} - TEST_F(ContentLoFiDeciderTest, NoTransformDoesNotAddHeader) { base::FieldTrialList field_trial_list(nullptr); base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), @@ -792,4 +736,56 @@ TEST_F(ContentLoFiDeciderTest, NoTransformDoesNotAddHeader) { EXPECT_FALSE(headers.HasHeader(chrome_proxy_accept_transform_header())); } +TEST_F(ContentLoFiDeciderTest, RequestIsClientSideLoFiMainFrameTest) { + std::unique_ptr<net::URLRequest> request = CreateRequestByType( + content::RESOURCE_TYPE_MAIN_FRAME, true, content::CLIENT_LOFI_ON); + std::unique_ptr<data_reduction_proxy::ContentLoFiDecider> lofi_decider( + new data_reduction_proxy::ContentLoFiDecider()); + EXPECT_FALSE(lofi_decider->IsClientLoFiImageRequest(*request)); +} + +TEST_F(ContentLoFiDeciderTest, RequestIsNotClientSideLoFiImageTest) { + std::unique_ptr<net::URLRequest> request = CreateRequestByType( + content::RESOURCE_TYPE_IMAGE, true, content::PREVIEWS_NO_TRANSFORM); + std::unique_ptr<data_reduction_proxy::ContentLoFiDecider> lofi_decider( + new data_reduction_proxy::ContentLoFiDecider()); + EXPECT_FALSE(lofi_decider->IsClientLoFiImageRequest(*request)); +} + +TEST_F(ContentLoFiDeciderTest, RequestIsClientSideLoFiImageTest) { + std::unique_ptr<net::URLRequest> request = CreateRequestByType( + content::RESOURCE_TYPE_IMAGE, true, content::CLIENT_LOFI_ON); + std::unique_ptr<data_reduction_proxy::ContentLoFiDecider> lofi_decider( + new data_reduction_proxy::ContentLoFiDecider()); + EXPECT_TRUE(lofi_decider->IsClientLoFiImageRequest(*request)); +} + +TEST_F(ContentLoFiDeciderTest, RequestIsClientLoFiAutoReload) { + // IsClientLoFiAutoReloadRequest() should return true for any request with the + // CLIENT_LOFI_AUTO_RELOAD bit set. + + EXPECT_TRUE(ContentLoFiDecider().IsClientLoFiAutoReloadRequest( + *CreateRequestByType(content::RESOURCE_TYPE_IMAGE, false, + content::CLIENT_LOFI_AUTO_RELOAD))); + + EXPECT_TRUE( + ContentLoFiDecider().IsClientLoFiAutoReloadRequest(*CreateRequestByType( + content::RESOURCE_TYPE_IMAGE, true, + content::CLIENT_LOFI_AUTO_RELOAD | content::PREVIEWS_NO_TRANSFORM))); + + EXPECT_TRUE(ContentLoFiDecider().IsClientLoFiAutoReloadRequest( + *CreateRequestByType(content::RESOURCE_TYPE_MAIN_FRAME, true, + content::CLIENT_LOFI_AUTO_RELOAD))); + + EXPECT_TRUE(ContentLoFiDecider().IsClientLoFiAutoReloadRequest( + *CreateRequestByType(content::RESOURCE_TYPE_SCRIPT, true, + content::CLIENT_LOFI_AUTO_RELOAD))); + + // IsClientLoFiAutoReloadRequest() should return false for any request without + // the CLIENT_LOFI_AUTO_RELOAD bit set. + EXPECT_FALSE(ContentLoFiDecider().IsClientLoFiAutoReloadRequest( + *CreateRequestByType(content::RESOURCE_TYPE_IMAGE, false, + content::PREVIEWS_NO_TRANSFORM))); +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/content/browser/content_lofi_ui_service_unittest.cc b/chromium/components/data_reduction_proxy/content/browser/content_lofi_ui_service_unittest.cc index e42be3df3d4..52d6503347e 100644 --- a/chromium/components/data_reduction_proxy/content/browser/content_lofi_ui_service_unittest.cc +++ b/chromium/components/data_reduction_proxy/content/browser/content_lofi_ui_service_unittest.cc @@ -20,6 +20,7 @@ #include "content/public/common/previews_state.h" #include "content/public/test/test_renderer_host.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_test_util.h" #include "testing/gtest/include/gtest/gtest.h" @@ -66,8 +67,9 @@ class ContentLoFiUIServiceTest : public content::RenderViewHostTestHarness { EXPECT_TRUE( content::BrowserThread::CurrentlyOn(content::BrowserThread::IO)); - std::unique_ptr<net::URLRequest> request = context.CreateRequest( - GURL("http://www.google.com/"), net::IDLE, delegate); + std::unique_ptr<net::URLRequest> request = + context.CreateRequest(GURL("http://www.google.com/"), net::IDLE, + delegate, TRAFFIC_ANNOTATION_FOR_TESTS); content::ResourceRequestInfo::AllocateForTesting( request.get(), content::RESOURCE_TYPE_SUB_FRAME, NULL, diff --git a/chromium/components/data_reduction_proxy/content/browser/content_resource_type_provider_unittest.cc b/chromium/components/data_reduction_proxy/content/browser/content_resource_type_provider_unittest.cc index 099aab54a4f..44429e35613 100644 --- a/chromium/components/data_reduction_proxy/content/browser/content_resource_type_provider_unittest.cc +++ b/chromium/components/data_reduction_proxy/content/browser/content_resource_type_provider_unittest.cc @@ -6,6 +6,7 @@ #include "base/command_line.h" #include "base/macros.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/run_loop.h" #include "base/test/histogram_tester.h" @@ -19,6 +20,7 @@ #include "content/public/browser/resource_request_info.h" #include "content/public/common/previews_state.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context.h" #include "net/url_request/url_request_test_util.h" @@ -109,8 +111,8 @@ class ContentResourceProviderTest : public testing::Test { std::unique_ptr<net::URLRequest> CreateRequestByType( const GURL& gurl, content::ResourceType resource_type) { - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(gurl, net::IDLE, &delegate_); + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + gurl, net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); AllocateRequestInfoForTesting(request.get(), resource_type); return request; } diff --git a/chromium/components/data_reduction_proxy/core/browser/BUILD.gn b/chromium/components/data_reduction_proxy/core/browser/BUILD.gn index 6b6c428c388..dc9c93e0631 100644 --- a/chromium/components/data_reduction_proxy/core/browser/BUILD.gn +++ b/chromium/components/data_reduction_proxy/core/browser/BUILD.gn @@ -17,6 +17,8 @@ browser_sources = [ "data_reduction_proxy_configurator.h", "data_reduction_proxy_data.cc", "data_reduction_proxy_data.h", + "data_reduction_proxy_data_use_observer.cc", + "data_reduction_proxy_data_use_observer.h", "data_reduction_proxy_delegate.cc", "data_reduction_proxy_delegate.h", "data_reduction_proxy_interceptor.cc", @@ -59,8 +61,10 @@ if (is_android) { "//components/data_reduction_proxy/core/common", "//components/data_reduction_proxy/proto:data_reduction_proxy_proto", "//components/data_use_measurement/core", + "//components/data_use_measurement/core:ascriber", "//components/pref_registry", "//components/prefs", + "//components/previews/core", "//components/variations", "//crypto", "//google_apis", @@ -84,8 +88,10 @@ static_library("browser") { deps = [ "//base", "//components/data_use_measurement/core", + "//components/data_use_measurement/core:ascriber", "//components/pref_registry", "//components/prefs", + "//components/previews/core", "//components/variations", "//crypto", "//google_apis", @@ -182,6 +188,7 @@ source_set("unit_tests") { "//components/data_reduction_proxy/core/common:test_support", "//components/data_reduction_proxy/proto:data_reduction_proxy_proto", "//components/prefs:test_support", + "//components/previews/core", "//components/variations", "//net:test_support", "//testing/gmock", diff --git a/chromium/components/data_reduction_proxy/core/browser/DEPS b/chromium/components/data_reduction_proxy/core/browser/DEPS index 743a2f3baec..16b7cb2bd4b 100644 --- a/chromium/components/data_reduction_proxy/core/browser/DEPS +++ b/chromium/components/data_reduction_proxy/core/browser/DEPS @@ -1,3 +1,4 @@ include_rules = [ + "+components/data_use_measurement/core", "+third_party/leveldatabase", ] diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc index 10f161ec10d..c20c7ea1128 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol.cc @@ -13,9 +13,9 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h" -#include "components/data_reduction_proxy/core/common/data_reduction_proxy_util.h" #include "net/base/load_flags.h" #include "net/http/http_response_headers.h" +#include "net/http/http_util.h" #include "net/proxy/proxy_config.h" #include "net/proxy/proxy_info.h" #include "net/proxy/proxy_list.h" @@ -176,7 +176,7 @@ bool DataReductionProxyBypassProtocol::MaybeBypassProxyAndPrepareToRetry( // Retry if block-once was specified or if method is idempotent. return bypass_type == BYPASS_EVENT_TYPE_CURRENT || - util::IsMethodIdempotent(request->method()); + net::HttpUtil::IsMethodIdempotent(request->method()); } } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol_unittest.cc index e44a9c2143f..ef62c610e40 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_protocol_unittest.cc @@ -25,7 +25,6 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params_test_utils.h" -#include "components/data_reduction_proxy/core/common/data_reduction_proxy_util.h" #include "net/base/completion_callback.h" #include "net/base/host_port_pair.h" #include "net/base/load_flags.h" @@ -34,9 +33,11 @@ #include "net/base/proxy_delegate.h" #include "net/http/http_response_headers.h" #include "net/http/http_transaction_test_util.h" +#include "net/http/http_util.h" #include "net/proxy/proxy_server.h" #include "net/proxy/proxy_service.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/static_http_user_agent_settings.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context.h" @@ -237,7 +238,8 @@ class DataReductionProxyProtocolTest : public testing::Test { network_delegate_->headers_received_count(); TestDelegate d; std::unique_ptr<URLRequest> r(context_->CreateRequest( - GURL("http://www.google.com/"), net::DEFAULT_PRIORITY, &d)); + GURL("http://www.google.com/"), net::DEFAULT_PRIORITY, &d, + TRAFFIC_ANNOTATION_FOR_TESTS)); r->set_method(method); r->SetLoadFlags(net::LOAD_NORMAL); @@ -340,10 +342,11 @@ TEST_F(DataReductionProxyProtocolTest, TestIdempotency) { }; for (size_t i = 0; i < arraysize(tests); ++i) { std::unique_ptr<net::URLRequest> request(context.CreateRequest( - GURL("http://www.google.com/"), net::DEFAULT_PRIORITY, NULL)); + GURL("http://www.google.com/"), net::DEFAULT_PRIORITY, NULL, + TRAFFIC_ANNOTATION_FOR_TESTS)); request->set_method(tests[i].method); EXPECT_EQ(tests[i].expected_result, - util::IsMethodIdempotent(request->method())); + net::HttpUtil::IsMethodIdempotent(request->method())); } } @@ -876,8 +879,9 @@ TEST_F(DataReductionProxyBypassProtocolEndToEndTest, mock_socket_factory()->AddSocketDataProvider(&retry_socket); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> url_request(context()->CreateRequest( - GURL("http://www.google.com"), net::IDLE, &delegate)); + std::unique_ptr<net::URLRequest> url_request( + context()->CreateRequest(GURL("http://www.google.com"), net::IDLE, + &delegate, TRAFFIC_ANNOTATION_FOR_TESTS)); url_request->Start(); drp_test_context()->RunUntilIdle(); @@ -935,8 +939,9 @@ TEST_F(DataReductionProxyBypassProtocolEndToEndTest, base::HistogramTester histogram_tester; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request(context()->CreateRequest( - GURL("http://google.com"), net::IDLE, &delegate)); + std::unique_ptr<net::URLRequest> request( + context()->CreateRequest(GURL("http://google.com"), net::IDLE, + &delegate, TRAFFIC_ANNOTATION_FOR_TESTS)); request->Start(); drp_test_context()->RunUntilIdle(); @@ -971,7 +976,8 @@ TEST_F(DataReductionProxyProtocolTest, TestDelegate d; std::unique_ptr<URLRequest> r(context_->CreateRequest( - GURL("http://www.google.com/"), net::DEFAULT_PRIORITY, &d)); + GURL("http://www.google.com/"), net::DEFAULT_PRIORITY, &d, + TRAFFIC_ANNOTATION_FOR_TESTS)); r->set_method("GET"); r->SetLoadFlags(net::LOAD_NORMAL); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc index cd7ddb4139b..7aa01480080 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats_unittest.cc @@ -16,6 +16,7 @@ #include "base/logging.h" #include "base/macros.h" #include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/histogram.h" #include "base/test/histogram_tester.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h" @@ -40,6 +41,7 @@ #include "net/http/http_util.h" #include "net/proxy/proxy_server.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context_getter.h" #include "net/url_request/url_request_context_storage.h" @@ -80,14 +82,15 @@ class DataReductionProxyBypassStatsTest : public testing::Test { test_context_ = DataReductionProxyTestContext::Builder().WithMockConfig().Build(); - mock_url_request_ = context_.CreateRequest(GURL(), net::IDLE, &delegate_); + mock_url_request_ = context_.CreateRequest(GURL(), net::IDLE, &delegate_, + TRAFFIC_ANNOTATION_FOR_TESTS); } std::unique_ptr<net::URLRequest> CreateURLRequestWithResponseHeaders( const GURL& url, const std::string& response_headers) { - std::unique_ptr<net::URLRequest> fake_request = - context_.CreateRequest(url, net::IDLE, &delegate_); + std::unique_ptr<net::URLRequest> fake_request = context_.CreateRequest( + url, net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); // Create a test job that will fill in the given response headers for the // |fake_request|. @@ -326,8 +329,8 @@ class DataReductionProxyBypassStatsEndToEndTest : public testing::Test { retry_socket_data_provider.get()); } - std::unique_ptr<net::URLRequest> request( - context_.CreateRequest(url, net::IDLE, &delegate_)); + std::unique_ptr<net::URLRequest> request(context_.CreateRequest( + url, net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS)); request->set_method("GET"); request->SetLoadFlags(load_flags); request->Start(); @@ -385,7 +388,8 @@ class DataReductionProxyBypassStatsEndToEndTest : public testing::Test { mock_socket_factory_.AddSocketDataProvider(&response_socket_data_provider); std::unique_ptr<net::URLRequest> request( - context_.CreateRequest(GURL("http://foo.com"), net::IDLE, &delegate_)); + context_.CreateRequest(GURL("http://foo.com"), net::IDLE, &delegate_, + TRAFFIC_ANNOTATION_FOR_TESTS)); request->set_method("GET"); request->Start(); drp_test_context_->RunUntilIdle(); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc index 04280b08bc7..662892a0087 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.cc @@ -78,7 +78,8 @@ void AddInt64ToListPref(size_t index, int64_t length, base::ListValue* list_update) { int64_t value = GetInt64PrefValue(*list_update, index) + length; - list_update->Set(index, new base::Value(base::Int64ToString(value))); + list_update->Set(index, + base::MakeUnique<base::Value>(base::Int64ToString(value))); } // DailyContentLengthUpdate maintains a data saving pref. The pref is a list @@ -470,6 +471,8 @@ void DataReductionProxyCompressionStats::RecordData( IncreaseInt64Pref(data_reduction_proxy::prefs::kHttpOriginalContentLength, original_size); + // TODO(rajendrant): Remove RecordDataUsage once data use ascriber based per + // domain data usage recording is enabled. RecordDataUsage(data_use_host, data_used, original_size, base::Time::Now()); RecordRequestSizePrefs(data_used, original_size, data_saver_enabled, request_type, mime_type, base::Time::Now()); @@ -1189,7 +1192,7 @@ void DataReductionProxyCompressionStats::RecordDataUsage( const std::string& data_usage_host, int64_t data_used, int64_t original_size, - const base::Time& time) { + const base::Time time) { if (current_data_usage_load_status_ != LOADED) return; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h index b3cf152cc31..324a1da2a28 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h @@ -64,6 +64,9 @@ class DataReductionProxyCompressionStats { // Records detailed data usage broken down by connection type and domain. Also // records daily data savings statistics to prefs and reports data savings // UMA. |compressed_size| and |original_size| are measured in bytes. + // TODO(rajendrant): This can be changed to RecordRequestMimeType and + // |data_use_group| param removed. It records daily data savings statistics to + // prefs and reports data savings UMA. void UpdateContentLengths(int64_t compressed_size, int64_t original_size, bool data_reduction_proxy_enabled, @@ -71,6 +74,15 @@ class DataReductionProxyCompressionStats { const scoped_refptr<DataUseGroup>& data_use_group, const std::string& mime_type); + // Record data usage and original size of request broken down by host. + // |original_request_size| and |data_used| are in bytes. |time| is the time at + // which the data usage occurred. This method should be called in real time, + // so |time| is expected to be |Time::Now()|. + void RecordDataUsage(const std::string& data_usage_host, + int64_t original_request_size, + int64_t data_used, + const base::Time time); + // Creates a |Value| summary of the persistent state of the network // statistics. // Must be called on the UI thread. @@ -203,14 +215,6 @@ class DataReductionProxyCompressionStats { const char* original_size_via_proxy_pref, const char* received_size_via_proxy_pref); - // Record data usage and original size of request broken down by host. |time| - // is the time at which the data usage occurred. This method should be called - // in real time, so |time| is expected to be |Time::Now()|. - void RecordDataUsage(const std::string& data_usage_host, - int64_t original_request_size, - int64_t data_used, - const base::Time& time); - // Persists the in memory data usage information to storage and clears all // in-memory data usage. Do not call this method unless |data_usage_loaded_| // is |LOADED|. diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc index e9a5338c84c..d05f679441f 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats_unittest.cc @@ -15,6 +15,7 @@ #include "base/run_loop.h" #include "base/strings/string_number_conversions.h" #include "base/test/histogram_tester.h" +#include "base/test/scoped_task_environment.h" #include "base/time/time.h" #include "base/values.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs.h" @@ -112,7 +113,9 @@ const char kLastUpdateTime[] = "Wed, 18 Sep 2013 03:45:26"; class DataReductionProxyCompressionStatsTest : public testing::Test { protected: - DataReductionProxyCompressionStatsTest() { + DataReductionProxyCompressionStatsTest() + : scoped_task_environment_( + base::test::ScopedTaskEnvironment::MainThreadType::UI) { EXPECT_TRUE(base::Time::FromString(kLastUpdateTime, &now_)); } @@ -163,13 +166,12 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { for (size_t i = 0; i < kNumDaysInHistory; ++i) { original_daily_content_length_list->Set( - i, new base::Value(base::SizeTToString(i))); + i, base::MakeUnique<base::Value>(base::SizeTToString(i))); } received_daily_content_length_list->Clear(); for (size_t i = 0; i < kNumDaysInHistory / 2; ++i) { - received_daily_content_length_list->Set( - i, new base::Value(base::SizeTToString(i))); + received_daily_content_length_list->AppendString(base::SizeTToString(i)); } } @@ -474,7 +476,7 @@ class DataReductionProxyCompressionStatsTest : public testing::Test { } private: - base::MessageLoopForUI loop_; + base::test::ScopedTaskEnvironment scoped_task_environment_; std::unique_ptr<DataReductionProxyTestContext> drp_test_context_; std::unique_ptr<DataReductionProxyCompressionStats> compression_stats_; base::Time now_; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc index 9ffb7f87fd1..16cf8fef44f 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.cc @@ -27,14 +27,18 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_config_values.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_event_creator.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h" #include "components/data_use_measurement/core/data_use_user_data.h" +#include "components/previews/core/previews_decider.h" #include "components/variations/variations_associated_data.h" #include "net/base/host_port_pair.h" #include "net/base/load_flags.h" #include "net/base/network_change_notifier.h" #include "net/log/net_log_source_type.h" +#include "net/nqe/effective_connection_type.h" #include "net/proxy/proxy_server.h" +#include "net/traffic_annotation/network_traffic_annotation.h" #include "net/url_request/url_fetcher.h" #include "net/url_request/url_fetcher_delegate.h" #include "net/url_request/url_request.h" @@ -244,12 +248,38 @@ class SecureProxyChecker : public net::URLFetcherDelegate { void CheckIfSecureProxyIsAllowed(const GURL& secure_proxy_check_url, FetcherResponseCallback fetcher_callback) { - fetcher_ = net::URLFetcher::Create(secure_proxy_check_url, - net::URLFetcher::GET, this); + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation( + "data_reduction_proxy_secure_proxy_check", R"( + semantics { + sender: "Data Reduction Proxy" + description: + "Sends a request to the Data Reduction Proxy server. Proceeds " + "with using a secure connection to the proxy only if the " + "response is not blocked or modified by an intermediary." + trigger: + "A request can be sent whenever the browser is determining how " + "to configure its connection to the data reduction proxy. This " + "happens on startup and network changes." + data: "A specific URL, not related to user data." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: + "Users can control Data Saver on Android via the 'Data Saver' " + "setting. Data Saver is not available on iOS, and on desktop " + "it is enabled by installing the Data Saver extension." + policy_exception_justification: "Not implemented." + })"); + fetcher_ = net::URLFetcher::Create( + secure_proxy_check_url, net::URLFetcher::GET, this, traffic_annotation); data_use_measurement::DataUseUserData::AttachToFetcher( fetcher_.get(), data_use_measurement::DataUseUserData::DATA_REDUCTION_PROXY); - fetcher_->SetLoadFlags(net::LOAD_DISABLE_CACHE | net::LOAD_BYPASS_PROXY); + fetcher_->SetLoadFlags(net::LOAD_DISABLE_CACHE | net::LOAD_BYPASS_PROXY | + net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); fetcher_->SetRequestContext(url_request_context_getter_.get()); // Configure max retries to be at most kMaxRetries times for 5xx errors. static const int kMaxRetries = 5; @@ -298,17 +328,42 @@ class WarmupURLFetcher : public net::URLFetcherDelegate { void FetchWarmupURL() { UMA_HISTOGRAM_EXACT_LINEAR("DataReductionProxy.WarmupURL.FetchInitiated", 1, 2); - - fetcher_ = net::URLFetcher::Create(params::GetWarmupURL(), - net::URLFetcher::GET, this); + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("data_reduction_proxy_warmup", R"( + semantics { + sender: "Data Reduction Proxy" + description: + "Sends a request to the Data Reduction Proxy server to warm up " + "the connection to the proxy." + trigger: + "A network change while the data reduction proxy is enabled will " + "trigger this request." + data: "A specific URL, not related to user data." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: + "Users can control Data Saver on Android via the 'Data Saver' " + "setting. Data Saver is not available on iOS, and on desktop it " + "is enabled by installing the Data Saver extension." + policy_exception_justification: "Not implemented." + })"); + fetcher_ = net::URLFetcher::Create( + params::GetWarmupURL(), net::URLFetcher::GET, this, traffic_annotation); data_use_measurement::DataUseUserData::AttachToFetcher( fetcher_.get(), data_use_measurement::DataUseUserData::DATA_REDUCTION_PROXY); + // Do not disable cookies. This allows the warmup connection to be reused + // for fetching user initiated requests. fetcher_->SetLoadFlags(net::LOAD_BYPASS_CACHE); fetcher_->SetRequestContext(url_request_context_getter_.get()); - // |fetcher| should not retry on 5xx errors. + // |fetcher| should not retry on 5xx errors. |fetcher_| should retry on + // network changes since the network stack may receive the connection change + // event later than |this|. + static const int kMaxRetries = 5; fetcher_->SetAutomaticallyRetryOn5xx(false); - fetcher_->SetAutomaticallyRetryOnNetworkChanges(0); + fetcher_->SetAutomaticallyRetryOnNetworkChanges(kMaxRetries); fetcher_->Start(); } @@ -951,17 +1006,18 @@ bool DataReductionProxyConfig::IsEffectiveConnectionTypeSlowerThanThreshold( } bool DataReductionProxyConfig::ShouldEnableLoFi( - const net::URLRequest& request) { + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK((request.load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) != 0); DCHECK(!request.url().SchemeIsCryptographic()); - net::NetworkQualityEstimator* network_quality_estimator; - network_quality_estimator = - request.context() ? request.context()->network_quality_estimator() - : nullptr; + if (base::FeatureList::IsEnabled( + features::kDataReductionProxyDecidesTransform)) { + return ShouldAcceptServerLoFi(request, previews_decider); + } - bool enable_lofi = ShouldEnableLoFiInternal(network_quality_estimator); + bool enable_lofi = ShouldEnableLoFiInternal(request, previews_decider); if (params::IsLoFiSlowConnectionsOnlyViaFlags() || params::IsIncludedInLoFiEnabledFieldTrial()) { @@ -974,14 +1030,18 @@ bool DataReductionProxyConfig::ShouldEnableLoFi( } bool DataReductionProxyConfig::ShouldEnableLitePages( - const net::URLRequest& request) { + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK((request.load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) != 0); DCHECK(!request.url().SchemeIsCryptographic()); - return ShouldEnableLitePagesInternal( - request.context() ? request.context()->network_quality_estimator() - : nullptr); + if (base::FeatureList::IsEnabled( + features::kDataReductionProxyDecidesTransform)) { + return ShouldAcceptLitePages(request, previews_decider); + } + + return ShouldEnableLitePagesInternal(request, previews_decider); } bool DataReductionProxyConfig::enabled_by_user_and_reachable() const { @@ -989,16 +1049,111 @@ bool DataReductionProxyConfig::enabled_by_user_and_reachable() const { return enabled_by_user_ && !unreachable_; } +bool DataReductionProxyConfig::IsBlackListedOrDisabled( + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider, + previews::PreviewsType previews_type) const { + // Make sure request is not locally blacklisted. + if (params::IsBlackListEnabledForServerPreviews()) { + // Pass in net::EFFECTIVE_CONNECTION_TYPE_4G as the thresold as network + // speed is checked in IsNetworkQualityProhibitivelySlow(). + // TODO(ryansturm): Use the correct ECT value (or add new method to + // just check blacklist). crbug.com/720102 + return !previews_decider.ShouldAllowPreviewAtECT( + request, previews_type, net::EFFECTIVE_CONNECTION_TYPE_4G); + } else { + // If Lo-Fi has been turned off, its status can't change. This Lo-Fi bit + // will be removed when Lo-Fi and Lite Pages are moved over to using the + // PreviewsBlackList. + return lofi_off_; + } +} + +bool DataReductionProxyConfig::ShouldAcceptServerLoFi( + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) const { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(base::FeatureList::IsEnabled( + features::kDataReductionProxyDecidesTransform)); + + if (IsBlackListedOrDisabled(request, previews_decider, + previews::PreviewsType::LOFI)) { + return false; + } + + if (params::IsLoFiAlwaysOnViaFlags()) { + return true; + } + + if (params::IsLoFiCellularOnlyViaFlags()) { + return net::NetworkChangeNotifier::IsConnectionCellular(connection_type_); + } + + if (params::IsLoFiSlowConnectionsOnlyViaFlags() || + params::IsIncludedInLoFiEnabledFieldTrial()) { + // Accept Lo-Fi from the data reduction proxy (it will handle the effective + // connection type check). + return true; + } + + return false; +} + +bool DataReductionProxyConfig::ShouldAcceptLitePages( + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) const { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(base::FeatureList::IsEnabled( + features::kDataReductionProxyDecidesTransform)); + + if (IsBlackListedOrDisabled(request, previews_decider, + previews::PreviewsType::LITE_PAGE)) { + return false; + } + + if (params::IsLoFiAlwaysOnViaFlags() && + params::AreLitePagesEnabledViaFlags()) { + return true; + } + + if (params::IsLoFiCellularOnlyViaFlags() && + params::AreLitePagesEnabledViaFlags()) { + return net::NetworkChangeNotifier::IsConnectionCellular(connection_type_); + } + + if ((params::IsLoFiSlowConnectionsOnlyViaFlags() && + params::AreLitePagesEnabledViaFlags()) || + params::IsIncludedInLitePageFieldTrial()) { + // Accept LitePages from the data reduction proxy (it will handle the + // effective connection type check). + return true; + } + + return false; +} + bool DataReductionProxyConfig::ShouldEnableLoFiInternal( - const net::NetworkQualityEstimator* network_quality_estimator) { + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) { DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(!base::FeatureList::IsEnabled( + features::kDataReductionProxyDecidesTransform)); last_query_ = GetTicksNow(); network_quality_at_last_query_ = NETWORK_QUALITY_AT_LAST_QUERY_UNKNOWN; - // If Lo-Fi has been turned off, its status can't change. - if (lofi_off_) + // LitePages overrides Server Lo-Fi. No fallback to Lo-Fi supported + // on this code path (not using DataReductionProxyDecidesTransform feature). + // TODO(dougarnett): Delete this surrounding method and related code once the + // DataReductionProxyDecidesTransform feature is launched to stable [725645]. + if (ShouldEnableLitePages(request, previews_decider)) { + return false; + } + + if (IsBlackListedOrDisabled(request, previews_decider, + previews::PreviewsType::LOFI)) { return false; + } if (params::IsLoFiAlwaysOnViaFlags()) return true; @@ -1007,27 +1162,31 @@ bool DataReductionProxyConfig::ShouldEnableLoFiInternal( return net::NetworkChangeNotifier::IsConnectionCellular(connection_type_); } + net::NetworkQualityEstimator* network_quality_estimator; + network_quality_estimator = + request.context() ? request.context()->network_quality_estimator() + : nullptr; + if (params::IsLoFiSlowConnectionsOnlyViaFlags() || params::IsIncludedInLoFiEnabledFieldTrial() || params::IsIncludedInLoFiControlFieldTrial()) { return IsNetworkQualityProhibitivelySlow(network_quality_estimator); } - // If Lo-Fi is not enabled through command line and the user is not in - // Lo-Fi field trials, set Lo-Fi to off. - lofi_off_ = true; return false; } bool DataReductionProxyConfig::ShouldEnableLitePagesInternal( - const net::NetworkQualityEstimator* network_quality_estimator) { + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) { DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(!base::FeatureList::IsEnabled( + features::kDataReductionProxyDecidesTransform)); - // If Lo-Fi has been turned off, its status can't change. This Lo-Fi bit will - // be removed when Lo-Fi and Lite Pages are moved over to using the Previews - // blacklist. - if (lofi_off_) + if (IsBlackListedOrDisabled(request, previews_decider, + previews::PreviewsType::LITE_PAGE)) { return false; + } if (params::IsLoFiAlwaysOnViaFlags() && params::AreLitePagesEnabledViaFlags()) return true; @@ -1037,6 +1196,10 @@ bool DataReductionProxyConfig::ShouldEnableLitePagesInternal( return net::NetworkChangeNotifier::IsConnectionCellular( net::NetworkChangeNotifier::GetConnectionType()); } + net::NetworkQualityEstimator* network_quality_estimator; + network_quality_estimator = + request.context() ? request.context()->network_quality_estimator() + : nullptr; if ((params::IsLoFiSlowConnectionsOnlyViaFlags() && params::AreLitePagesEnabledViaFlags()) || diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h index 5bc7b57a8b9..e4529cbf34f 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h @@ -19,6 +19,7 @@ #include "base/threading/thread_checker.h" #include "base/time/time.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_server.h" +#include "components/previews/core/previews_experiments.h" #include "net/base/net_errors.h" #include "net/base/network_change_notifier.h" #include "net/base/network_interfaces.h" @@ -42,6 +43,10 @@ class URLRequestContextGetter; class URLRequestStatus; } +namespace previews { +class PreviewsDecider; +} + namespace data_reduction_proxy { typedef base::Callback<void(const std::string&, @@ -193,12 +198,16 @@ class DataReductionProxyConfig // Returns true when Lo-Fi Previews should be activated. Records metrics for // Lo-Fi state changes. |request| is used to get the network quality estimator - // from the URLRequestContext. - bool ShouldEnableLoFi(const net::URLRequest& request); + // from the URLRequestContext. |previews_decider| is used to check if + // |request| is locally blacklisted. + bool ShouldEnableLoFi(const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider); // Returns true when Lite Page Previews should be activated. |request| is used // to get the network quality estimator from the URLRequestContext. - bool ShouldEnableLitePages(const net::URLRequest& request); + // |previews_decider| is used to check if |request| is locally blacklisted. + bool ShouldEnableLitePages(const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider); // Returns true if the data saver has been enabled by the user, and the data // saver proxy is reachable. @@ -245,6 +254,9 @@ class DataReductionProxyConfig FRIEND_TEST_ALL_PREFIXES(DataReductionProxyConfigTest, LoFiAccuracyNonZeroDelay); FRIEND_TEST_ALL_PREFIXES(DataReductionProxyConfigTest, WarmupURL); + FRIEND_TEST_ALL_PREFIXES(DataReductionProxyConfigTest, + ShouldAcceptServerLoFi); + FRIEND_TEST_ALL_PREFIXES(DataReductionProxyConfigTest, ShouldAcceptLitePages); // Values of the estimated network quality at the beginning of the most // recent query of the Network Quality Estimator. @@ -291,18 +303,52 @@ class DataReductionProxyConfig bool is_https, base::TimeDelta* min_retry_delay) const; + // Returns whether the request is blacklisted (or if Lo-Fi is disabled). + bool IsBlackListedOrDisabled( + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider, + previews::PreviewsType previews_type) const; + + // Returns whether the client should report to the data reduction proxy that + // it is willing to accept the Server Lo-Fi optimization for |request|. + // |previews_decider| is used to check if |request| is locally blacklisted. + // Should only be used if the kDataReductionProxyDecidesTransform feature is + // enabled. + bool ShouldAcceptServerLoFi( + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) const; + + // Returns whether the client should report to the data reduction proxy that + // it is willing to accept a LitePage optimization for |request|. + // |previews_decider| is used to check if |request| is locally blacklisted. + // Should only be used if the kDataReductionProxyDecidesTransform feature is + // enabled. + bool ShouldAcceptLitePages( + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider) const; + // Returns true when Lo-Fi Previews should be activated. Determines if Lo-Fi // Previews should be activated by checking the Lo-Fi flags and if the network // quality is prohibitively slow. |network_quality_estimator| may be NULL. + // |previews_decider| is a non-null object that determines eligibility of the + // showing the preview based on past opt outs. + // Should NOT be used if the kDataReductionProxyDecidesTransform feature is + // enabled. bool ShouldEnableLoFiInternal( - const net::NetworkQualityEstimator* network_quality_estimator); + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider); // Returns true when Lite Page Previews should be activated. Determines if // Lite Page Previewsmode should be activated by checking the Lite Page // Previews flags and if the network quality is prohibitively slow. - // |network_quality_estimator| may be NULL. + // |network_quality_estimator| may be NULL. |previews_decider| is a non-null + // object that determines eligibility of showing the preview based on past opt + // outs. + // Should NOT be used if the kDataReductionProxyDecidesTransform feature is + // enabled. bool ShouldEnableLitePagesInternal( - const net::NetworkQualityEstimator* network_quality_estimator); + const net::URLRequest& request, + const previews::PreviewsDecider& previews_decider); // Returns true if the network quality is at least as poor as the one // specified in the Auto Lo-Fi field trial parameters. diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.cc index 2b58c6b14f5..ef0b077e08c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client.cc @@ -38,6 +38,7 @@ #include "net/http/http_status_code.h" #include "net/log/net_log_source_type.h" #include "net/proxy/proxy_server.h" +#include "net/traffic_annotation/network_traffic_annotation.h" #include "net/url_request/url_fetcher.h" #include "net/url_request/url_request_status.h" @@ -407,20 +408,44 @@ DataReductionProxyConfigServiceClient::GetURLFetcherForConfig( const GURL& secure_proxy_check_url, const std::string& request_body) { DCHECK(thread_checker_.CalledOnValidThread()); + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("data_reduction_proxy_config", R"( + semantics { + sender: "Data Reduction Proxy" + description: + "Requests a configuration that specifies how to connect to the " + "data reduction proxy." + trigger: + "Requested when Data Saver is enabled and the browser does not " + "have a configuration that is not older than a threshold set by " + "the server." + data: "None." + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: + "Users can control Data Saver on Android via 'Data Saver' setting. " + "Data Saver is not available on iOS, and on desktop it is enabled " + "by insalling the Data Saver extension." + policy_exception_justification: "Not implemented." + })"); std::unique_ptr<net::URLFetcher> fetcher(net::URLFetcher::Create( - secure_proxy_check_url, net::URLFetcher::POST, this)); + secure_proxy_check_url, net::URLFetcher::POST, this, traffic_annotation)); data_use_measurement::DataUseUserData::AttachToFetcher( fetcher.get(), data_use_measurement::DataUseUserData::DATA_REDUCTION_PROXY); - fetcher->SetLoadFlags(net::LOAD_BYPASS_PROXY); + fetcher->SetLoadFlags(net::LOAD_BYPASS_PROXY | net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); fetcher->SetUploadData("application/x-protobuf", request_body); DCHECK(url_request_context_getter_); fetcher->SetRequestContext(url_request_context_getter_); // |fetcher| should not retry on 5xx errors since the server may already be // overloaded. Spurious 5xx errors are still retried on exponential backoff. - // |fetcher| should not retry on network changes since a new fetch will be - // initiated. - fetcher->SetAutomaticallyRetryOnNetworkChanges(0); + // |fetcher| should retry on network changes since the network stack may + // receive the connection change event later than |this|. + static const int kMaxRetries = 5; + fetcher->SetAutomaticallyRetryOnNetworkChanges(kMaxRetries); return fetcher; } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client_unittest.cc index 1a5cffc1935..2c798fa7c3b 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_service_client_unittest.cc @@ -14,6 +14,7 @@ #include "base/command_line.h" #include "base/macros.h" #include "base/memory/ptr_util.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/run_loop.h" #include "base/test/histogram_tester.h" @@ -35,6 +36,7 @@ #include "net/http/http_response_headers.h" #include "net/proxy/proxy_server.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request_context_storage.h" #include "net/url_request/url_request_test_util.h" #include "testing/gmock/include/gmock/gmock.h" @@ -1091,8 +1093,9 @@ TEST_F(DataReductionProxyConfigServiceClientTest, HTTPRequests) { net::TestDelegate test_delegate; std::unique_ptr<net::URLRequest> request( - test_url_request_context()->CreateRequest(GURL(tests[i].url), net::IDLE, - &test_delegate)); + test_url_request_context()->CreateRequest( + GURL(tests[i].url), net::IDLE, &test_delegate, + TRAFFIC_ANNOTATION_FOR_TESTS)); request->Start(); RunUntilIdle(); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h index 05b59b391e3..9a96c341552 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h @@ -117,6 +117,11 @@ class TestDataReductionProxyConfig : public DataReductionProxyConfig { // Sets if the captive portal probe has been blocked for the current network. void SetIsCaptivePortal(bool is_captive_portal); + void SetConnectionTypeForTesting( + net::NetworkChangeNotifier::ConnectionType connection_type) { + connection_type_ = connection_type; + } + using DataReductionProxyConfig::UpdateConfigForTesting; private: diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc index f70b5e700b8..ed3784b1b06 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_config_unittest.cc @@ -19,12 +19,14 @@ #include "base/macros.h" #include "base/memory/ptr_util.h" #include "base/memory/ref_counted.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/run_loop.h" #include "base/strings/safe_sprintf.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_util.h" #include "base/test/histogram_tester.h" +#include "base/test/scoped_feature_list.h" #include "base/test/simple_test_tick_clock.h" #include "base/threading/platform_thread.h" #include "base/time/time.h" @@ -34,10 +36,13 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_event_creator.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params_test_utils.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_server.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h" #include "components/data_reduction_proxy/proto/client_config.pb.h" +#include "components/previews/core/previews_decider.h" +#include "components/previews/core/previews_experiments.h" #include "components/variations/variations_associated_data.h" #include "net/base/load_flags.h" #include "net/base/net_errors.h" @@ -49,6 +54,7 @@ #include "net/nqe/network_quality_estimator_test_util.h" #include "net/proxy/proxy_server.h" #include "net/test/embedded_test_server/embedded_test_server.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/test_url_fetcher_factory.h" #include "net/url_request/url_fetcher.h" #include "net/url_request/url_request.h" @@ -78,6 +84,28 @@ std::string GetRetryMapKeyFromOrigin(const std::string& origin) { return origin; } +class TestPreviewsDecider : public previews::PreviewsDecider { + public: + TestPreviewsDecider(bool allow_previews) : allow_previews_(allow_previews) {} + ~TestPreviewsDecider() override {} + + // previews::PreviewsDecider: + bool ShouldAllowPreviewAtECT( + const net::URLRequest& request, + previews::PreviewsType type, + net::EffectiveConnectionType effective_connection_type_threshold) + const override { + return allow_previews_; + } + bool ShouldAllowPreview(const net::URLRequest& request, + previews::PreviewsType type) const override { + return allow_previews_; + } + + private: + bool allow_previews_; +}; + } // namespace class DataReductionProxyConfigTest : public testing::Test { @@ -811,90 +839,199 @@ TEST_F(DataReductionProxyConfigTest, IsDataReductionProxyWithMutableConfig) { TEST_F(DataReductionProxyConfigTest, LoFiOn) { const struct { - bool lofi_switch_enabled; + bool lofi_enabled; + bool previews_black_list_used; const std::string lofi_field_trial_group_name; bool network_prohibitively_slow; bool expect_lofi_header; int bucket_to_check_for_auto_lofi_uma; int expect_bucket_count; + bool is_opted_out; } tests[] = { { // The Lo-Fi switch is off and the user is not in the enabled field // trial group. Lo-Fi should not be used. - false, std::string(), false, false, 0, + false, false, std::string(), false, false, 0, 0, // not in enabled field trial, UMA is not recorded + false, }, { // The Lo-Fi switch is off and the user is not in enabled field trial // group and the network quality is bad. Lo-Fi should not be used. - false, std::string(), true, false, 0, + false, false, std::string(), true, false, 0, + 0, // not in enabled field trial, UMA is not recorded + false, + + }, + { + // Lo-Fi is enabled through command line switch, but opted out. LoFi + // should not be used. + true, false, std::string(), false, false, 0, 0, // not in enabled field trial, UMA is not recorded + true, }, { // Lo-Fi is enabled through command line switch. LoFi should be used. - true, std::string(), false, true, 0, + true, false, std::string(), false, true, 0, 0, // not in enabled field trial, UMA is not recorded + false, }, { // The user is in the enabled field trial group but the network // quality is not bad. Lo-Fi should not be used. - false, "Enabled", false, false, + false, false, "Enabled", false, false, 0, // Lo-Fi request header is not used (state change: empty to empty) - 1, + 1, false, }, { // The user is in the enabled field trial group but the network // quality is not bad. Lo-Fi should not be used. - false, "Enabled_Control", false, false, + false, false, "Enabled_Control", false, false, 0, // Lo-Fi request header is not used (state change: empty to empty) - 1, + 1, false, }, { // The user is in the enabled field trial group and the network // quality is bad. Lo-Fi should be used. - false, "Enabled", true, true, + false, false, "Enabled", true, true, 1, // Lo-Fi request header is now used (state change: empty to low) - 1, + 1, false, }, { // The user is in the enabled field trial group and the network // quality is bad. Lo-Fi should be used. - false, "Enabled_Control", true, true, + false, false, "Enabled_Control", true, true, 3, // Lo-Fi request header is now used (state change: low to low) - 1, + 1, false, }, { // The user is in the enabled field trial group and the network // quality is bad. Lo-Fi should be used again. - false, "Enabled", true, true, + false, false, "Enabled", true, true, 3, // Lo-Fi request header is now used (state change: low to low) - 1, + 1, false, }, { // The user is in the enabled field trial group and the network // quality is bad. Lo-Fi should be used again. - false, "Enabled_Control", true, true, + false, false, "Enabled_Control", true, true, 3, // Lo-Fi request header is now used (state change: low to low) - 1, + 1, false, }, { // The user is in the enabled field trial group but the network // quality is not bad. Lo-Fi should not be used. - false, "Enabled", false, false, + false, false, "Enabled", false, false, 2, // Lo-Fi request header is not used (state change: low to empty) - 1, + 1, false, }, { // The user is in the enabled field trial group but the network // quality is not bad. Lo-Fi should not be used. - false, "Enabled_Control", false, false, + false, false, "Enabled_Control", false, false, 0, // Lo-Fi request header is not used (state change: empty to empty) - 1, + 1, false, + }, + { + // The Lo-Fi switch is off and the user is not in the enabled field + // trial group. Lo-Fi should not be used. + false, true, std::string(), false, false, 0, + 0, // not in enabled field trial, UMA is not recorded + false, + }, + { + // The Lo-Fi switch is off and the user is not in enabled field trial + // group and the network quality is bad. Lo-Fi should not be used. + false, true, std::string(), true, false, 0, + 0, // not in enabled field trial, UMA is not recorded + false, + }, + { + // Lo-Fi is enabled through command line switch. LoFi should be used. + true, true, std::string(), false, true, 0, + 0, // not in enabled field trial, UMA is not recorded + false, + }, + { + // Lo-Fi is enabled through command line switch, but opted out. LoFi + // should not be used. + true, true, std::string(), false, false, 0, + 0, // not in enabled field trial, UMA is not recorded + true, + }, + { + // The user is in the enabled field trial group but the network + // quality is not bad. Lo-Fi should not be used. + false, true, "Enabled", false, false, + 0, // Lo-Fi request header is not used (state change: empty to empty) + 1, false, + }, + { + // The user is in the enabled field trial group but the network + // quality is not bad. Lo-Fi should not be used. + false, true, "Enabled_Control", false, false, + 0, // Lo-Fi request header is not used (state change: empty to empty) + 1, false, + }, + { + // The user is in the enabled field trial group and the network + // quality is bad. Lo-Fi should be used. + false, true, "Enabled", true, true, + 1, // Lo-Fi request header is now used (state change: empty to low) + 1, false, + }, + { + // The user is in the enabled field trial group and the network + // quality is bad. Lo-Fi should be used. + false, true, "Enabled_Control", true, true, + 3, // Lo-Fi request header is now used (state change: low to low) + 1, false, + }, + { + // The user is in the enabled field trial group and the network + // quality is bad. Lo-Fi should be used again. + false, true, "Enabled", true, true, + 3, // Lo-Fi request header is now used (state change: low to low) + 1, false, + }, + { + // The user is in the enabled field trial group and the network + // quality is bad. Lo-Fi should be used again. + false, true, "Enabled_Control", true, true, + 3, // Lo-Fi request header is now used (state change: low to low) + 1, false, + }, + { + // The user is in the enabled field trial group but the network + // quality is not bad. Lo-Fi should not be used. + false, true, "Enabled", false, false, + 2, // Lo-Fi request header is not used (state change: low to empty) + 1, false, + }, + { + // The user is in the enabled field trial group but the network + // quality is not bad. Lo-Fi should not be used. + false, true, "Enabled_Control", false, false, + 0, // Lo-Fi request header is not used (state change: empty to empty) + 1, false, }, }; for (size_t i = 0; i < arraysize(tests); ++i) { config()->ResetLoFiStatusForTest(); - if (tests[i].lofi_switch_enabled) { + + // Ensure not using proxy-decides-transform feature. + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndDisableFeature( + features::kDataReductionProxyDecidesTransform); + + base::FieldTrialList field_trial_list(nullptr); + if (tests[i].previews_black_list_used) { + base::FieldTrialList::CreateFieldTrial( + "DataReductionProxyPreviewsBlackListTransition", "Enabled_"); + } else if (tests[i].is_opted_out) { + config()->SetLoFiModeOff(); + } + if (tests[i].lofi_enabled) { base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( switches::kDataReductionProxyLoFi, switches::kDataReductionProxyLoFiValueAlwaysOn); @@ -903,7 +1040,6 @@ TEST_F(DataReductionProxyConfigTest, LoFiOn) { switches::kDataReductionProxyLoFi, std::string()); } - base::FieldTrialList field_trial_list(nullptr); if (!tests[i].lofi_field_trial_group_name.empty()) { base::FieldTrialList::CreateFieldTrial( params::GetLoFiFieldTrialName(), @@ -916,11 +1052,14 @@ TEST_F(DataReductionProxyConfigTest, LoFiOn) { base::HistogramTester histogram_tester; net::TestURLRequestContext context_; net::TestDelegate delegate_; - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(GURL(), net::IDLE, &delegate_); + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + GURL(), net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); request->SetLoadFlags(request->load_flags() | net::LOAD_MAIN_FRAME_DEPRECATED); - bool should_enable_lofi = config()->ShouldEnableLoFi(*request.get()); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(!tests[i].is_opted_out); + bool should_enable_lofi = + config()->ShouldEnableLoFi(*request.get(), *previews_decider.get()); if (tests[i].expect_bucket_count != 0) { histogram_tester.ExpectBucketCount( "DataReductionProxy.AutoLoFiRequestHeaderState.Unknown", @@ -1262,4 +1401,346 @@ TEST_F(DataReductionProxyConfigTest, LoFiAccuracyNonZeroDelay) { "DataReductionProxy.LoFi.Accuracy.1.Unknown", 0, 1); } +TEST_F(DataReductionProxyConfigTest, ShouldEnableLoFiDoesNotFallback) { + // Turn off proxy-decides-transform feature (path for client ECT check). + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndDisableFeature( + features::kDataReductionProxyDecidesTransform); + + // Enable Server Lo-Fi. + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + + // Also enable LitePages + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + + net::TestURLRequestContext context; + net::TestDelegate delegate; + std::unique_ptr<net::URLRequest> request = context.CreateRequest( + GURL(), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + EXPECT_TRUE( + config()->ShouldEnableLitePages(*request.get(), *previews_decider.get())); + EXPECT_FALSE( + config()->ShouldEnableLoFi(*request.get(), *previews_decider.get())); +} + +TEST_F(DataReductionProxyConfigTest, ShouldEnableLoFiWithECTCheck) { + // Turn off proxy-decides-transform feature (so will locally check ECT). + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndDisableFeature( + features::kDataReductionProxyDecidesTransform); + + // Enable Server Lo-Fi field trial. + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Enabled"); + + EXPECT_CALL(*config(), IsNetworkQualityProhibitivelySlow(_)) + .WillRepeatedly(testing::Return(true)); + + net::TestURLRequestContext context; + net::TestDelegate delegate; + std::unique_ptr<net::URLRequest> request = context.CreateRequest( + GURL(), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + EXPECT_TRUE( + config()->ShouldEnableLoFi(*request.get(), *previews_decider.get())); + + // Now verify should not enable with good network connection. + EXPECT_CALL(*config(), IsNetworkQualityProhibitivelySlow(_)) + .WillRepeatedly(testing::Return(false)); + + EXPECT_FALSE( + config()->ShouldEnableLoFi(*request.get(), *previews_decider.get())); +} + +TEST_F(DataReductionProxyConfigTest, ShouldEnableLoFiWithoutECTCheck) { + // Turn on proxy-decides-transform feature (so will not locally check ECT). + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndEnableFeature( + features::kDataReductionProxyDecidesTransform); + + // Enable Server Lo-Fi field trial. + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Enabled"); + + // Expect network quality check is never called. + EXPECT_CALL(*config(), IsNetworkQualityProhibitivelySlow(_)).Times(0); + + net::TestURLRequestContext context; + net::TestDelegate delegate; + std::unique_ptr<net::URLRequest> request = context.CreateRequest( + GURL(), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + EXPECT_TRUE( + config()->ShouldEnableLoFi(*request.get(), *previews_decider.get())); +} + +TEST_F(DataReductionProxyConfigTest, ShouldEnableLitePagesWithECTCheck) { + // Turn off proxy-decides-transform feature (so will locally check ECT). + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndDisableFeature( + features::kDataReductionProxyDecidesTransform); + + // Enable Server Lo-Fi field trial. + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Enabled_Preview"); + + EXPECT_CALL(*config(), IsNetworkQualityProhibitivelySlow(_)) + .WillRepeatedly(testing::Return(true)); + + net::TestURLRequestContext context; + net::TestDelegate delegate; + std::unique_ptr<net::URLRequest> request = context.CreateRequest( + GURL(), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + EXPECT_TRUE( + config()->ShouldEnableLitePages(*request.get(), *previews_decider.get())); + + // Now verify should not enable with good network connection. + EXPECT_CALL(*config(), IsNetworkQualityProhibitivelySlow(_)) + .WillRepeatedly(testing::Return(false)); + + EXPECT_FALSE( + config()->ShouldEnableLitePages(*request.get(), *previews_decider.get())); +} + +TEST_F(DataReductionProxyConfigTest, ShouldEnableLitePagesWithoutECTCheck) { + // Turn on proxy-decides-transform feature (so will not locally check ECT). + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndEnableFeature( + features::kDataReductionProxyDecidesTransform); + + // Enable Server Lo-Fi field trial. + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Enabled_Preview"); + + // Expect network quality check is never called. + EXPECT_CALL(*config(), IsNetworkQualityProhibitivelySlow(_)).Times(0); + + net::TestURLRequestContext context_; + net::TestDelegate delegate_; + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + GURL(), net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + EXPECT_TRUE( + config()->ShouldEnableLitePages(*request.get(), *previews_decider.get())); +} + +TEST_F(DataReductionProxyConfigTest, ShouldAcceptServerLoFi) { + // Turn on proxy-decides-transform feature to satisfy DCHECK. + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndEnableFeature( + features::kDataReductionProxyDecidesTransform); + + net::TestURLRequestContext context_; + net::TestDelegate delegate_; + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + GURL(), net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + // Verify false for no flags. + EXPECT_FALSE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + + // Verify true for Always-On flag. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + EXPECT_TRUE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + + // Verify true for Always-On with LitePages enabled too. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + EXPECT_TRUE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + + // Verify true for Slow Connection flag. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueSlowConnectionsOnly); + EXPECT_TRUE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + + // Verify false for Cellular Only flag and WIFI connection. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueCellularOnly); + config()->SetConnectionTypeForTesting( + net::NetworkChangeNotifier::ConnectionType::CONNECTION_WIFI); + EXPECT_FALSE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + + // Verify true for Cellular Only flag and 3G connection. + config()->SetConnectionTypeForTesting( + net::NetworkChangeNotifier::ConnectionType::CONNECTION_3G); + EXPECT_TRUE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + + // Verify true for field trial. + { + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Enabled"); + EXPECT_TRUE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + } + + // Verify false for control field trial. + { + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Control"); + EXPECT_FALSE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + } + + // Verify PreviewsDecider check. + { + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial( + "DataReductionProxyPreviewsBlackListTransition", "Enabled"); + EXPECT_FALSE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + previews_decider = base::MakeUnique<TestPreviewsDecider>(true); + EXPECT_TRUE(config()->ShouldAcceptServerLoFi(*request.get(), + *previews_decider.get())); + } +} + +TEST_F(DataReductionProxyConfigTest, ShouldAcceptLitePages) { + // Turn on proxy-decides-transform feature to satisfy DCHECK. + base::test::ScopedFeatureList scoped_feature_list; + scoped_feature_list.InitAndEnableFeature( + features::kDataReductionProxyDecidesTransform); + + net::TestURLRequestContext context_; + net::TestDelegate delegate_; + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + GURL(), net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS); + request->SetLoadFlags(request->load_flags() | + net::LOAD_MAIN_FRAME_DEPRECATED); + std::unique_ptr<TestPreviewsDecider> previews_decider = + base::MakeUnique<TestPreviewsDecider>(false); + + // Verify false for no flags. + EXPECT_FALSE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + + // Verify true for Always-On flag and LitePage flag. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + EXPECT_FALSE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + EXPECT_TRUE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + + // Verify true for Slow Connection flag and LitePage flag. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueSlowConnectionsOnly); + EXPECT_FALSE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + EXPECT_TRUE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + + // Verify true for Cellular Only flag and 3G connection. + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueCellularOnly); + config()->SetConnectionTypeForTesting( + net::NetworkChangeNotifier::ConnectionType::CONNECTION_3G); + EXPECT_FALSE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + EXPECT_TRUE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + + // Verify false for Cellular Only flag and WIFI connection. + config()->SetConnectionTypeForTesting( + net::NetworkChangeNotifier::ConnectionType::CONNECTION_WIFI); + EXPECT_FALSE( + config()->ShouldAcceptLitePages(*request.get(), *previews_decider.get())); + + // Verify true for field trial. + { + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial(params::GetLoFiFieldTrialName(), + "Enabled_Preview"); + EXPECT_TRUE(config()->ShouldAcceptLitePages(*request.get(), + *previews_decider.get())); + } + + // Verify PreviewsDecider check. + { + base::CommandLine::ForCurrentProcess()->InitFromArgv(0, NULL); + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial( + "DataReductionProxyPreviewsBlackListTransition", "Enabled"); + EXPECT_FALSE(config()->ShouldAcceptLitePages(*request.get(), + *previews_decider.get())); + previews_decider = base::MakeUnique<TestPreviewsDecider>(true); + EXPECT_TRUE(config()->ShouldAcceptLitePages(*request.get(), + *previews_decider.get())); + } +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc index 425d1be797f..8d134ee644e 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.cc @@ -4,6 +4,7 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h" +#include "base/memory/ptr_util.h" #include "net/url_request/url_request.h" namespace data_reduction_proxy { @@ -14,6 +15,7 @@ const void* const kDataReductionProxyUserDataKey = DataReductionProxyData::DataReductionProxyData() : used_data_reduction_proxy_(false), lofi_requested_(false), + client_lofi_requested_(false), lite_page_received_(false), lofi_received_(false), effective_connection_type_(net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN) {} @@ -25,6 +27,7 @@ std::unique_ptr<DataReductionProxyData> DataReductionProxyData::DeepCopy() std::unique_ptr<DataReductionProxyData> copy(new DataReductionProxyData()); copy->used_data_reduction_proxy_ = used_data_reduction_proxy_; copy->lofi_requested_ = lofi_requested_; + copy->client_lofi_requested_ = client_lofi_requested_; copy->lite_page_received_ = lite_page_received_; copy->lofi_received_ = lofi_received_; copy->session_key_ = session_key_; @@ -49,7 +52,7 @@ DataReductionProxyData* DataReductionProxyData::GetDataAndCreateIfNecessary( if (data) return data; data = new DataReductionProxyData(); - request->SetUserData(kDataReductionProxyUserDataKey, data); + request->SetUserData(kDataReductionProxyUserDataKey, base::WrapUnique(data)); return data; } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h index 1f12d47e502..55b792b0239 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data.h @@ -53,6 +53,14 @@ class DataReductionProxyData : public base::SupportsUserData::Data { bool lofi_received() const { return lofi_received_; } void set_lofi_received(bool lofi_received) { lofi_received_ = lofi_received; } + // Whether client Lo-Fi was requested for this request. This is only set on + // image requests that have added a range header to attempt to get a smaller + // file size image. + bool client_lofi_requested() const { return client_lofi_requested_; } + void set_client_lofi_requested(bool client_lofi_requested) { + client_lofi_requested_ = client_lofi_requested; + } + // The session key used for this request. Only set for main frame requests. std::string session_key() const { return session_key_; } void set_session_key(const std::string& session_key) { @@ -99,11 +107,16 @@ class DataReductionProxyData : public base::SupportsUserData::Data { // Whether the DataReductionProxy was used for this request or navigation. bool used_data_reduction_proxy_; - // Whether Lo-Fi was requested for this request or navigation. True if the - // session is in Lo-Fi control or enabled group, and the network quality is - // slow. + // Whether server Lo-Fi was requested for this request or navigation. True if + // the session is in Lo-Fi control or enabled group, and the network quality + // is slow. bool lofi_requested_; + // Whether client Lo-Fi was requested for this request. This is only set on + // image requests that have added a range header to attempt to get a smaller + // file size image. + bool client_lofi_requested_; + // Whether a lite page response was seen for the request or navigation. bool lite_page_received_; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc index 8ba21079bf2..a7deafb2582 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_unittest.cc @@ -13,6 +13,7 @@ #include "base/message_loop/message_loop.h" #include "net/base/request_priority.h" #include "net/nqe/effective_connection_type.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context.h" #include "testing/gtest/include/gtest/gtest.h" @@ -81,7 +82,8 @@ TEST_F(DataReductionProxyDataTest, BasicSettersAndGetters) { TEST_F(DataReductionProxyDataTest, AddToURLRequest) { std::unique_ptr<net::URLRequestContext> context(new net::URLRequestContext()); std::unique_ptr<net::URLRequest> fake_request(context->CreateRequest( - GURL("http://www.google.com"), net::RequestPriority::IDLE, nullptr)); + GURL("http://www.google.com"), net::RequestPriority::IDLE, nullptr, + TRAFFIC_ANNOTATION_FOR_TESTS)); DataReductionProxyData* data = DataReductionProxyData::GetData(*fake_request.get()); EXPECT_FALSE(data); @@ -142,7 +144,8 @@ TEST_F(DataReductionProxyDataTest, DeepCopy) { TEST_F(DataReductionProxyDataTest, ClearData) { std::unique_ptr<net::URLRequestContext> context(new net::URLRequestContext()); std::unique_ptr<net::URLRequest> fake_request(context->CreateRequest( - GURL("http://www.google.com"), net::RequestPriority::IDLE, nullptr)); + GURL("http://www.google.com"), net::RequestPriority::IDLE, nullptr, + TRAFFIC_ANNOTATION_FOR_TESTS)); DataReductionProxyData* data = DataReductionProxyData::GetDataAndCreateIfNecessary(fake_request.get()); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.cc new file mode 100644 index 00000000000..8d433aefdbc --- /dev/null +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.cc @@ -0,0 +1,167 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.h" + +#include "base/memory/ptr_util.h" +#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h" +#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_configurator.h" +#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h" +#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_util.h" +#include "components/data_reduction_proxy/core/common/lofi_decider.h" +#include "components/data_use_measurement/core/data_use.h" +#include "components/data_use_measurement/core/data_use_ascriber.h" +#include "net/http/http_response_headers.h" +#include "net/url_request/url_request.h" +#include "url/gurl.h" + +namespace { + +class DataUseUserDataBytes : public base::SupportsUserData::Data { + public: + // Key used to store data usage in userdata until the page URL is available. + static const void* kUserDataKey; + + DataUseUserDataBytes(int64_t network_bytes, int64_t original_bytes) + : network_bytes_(network_bytes), original_bytes_(original_bytes) {} + + int64_t network_bytes() const { return network_bytes_; } + int64_t original_bytes() const { return original_bytes_; } + + void IncrementBytes(int64_t network_bytes, int64_t original_bytes) { + network_bytes_ += network_bytes; + original_bytes_ += original_bytes; + } + + private: + int64_t network_bytes_; + int64_t original_bytes_; +}; + +// static +const void* DataUseUserDataBytes::kUserDataKey = + &DataUseUserDataBytes::kUserDataKey; + +// Estimate the size of the original headers of |request|. If |used_drp| is +// true, then it's assumed that the original request would have used HTTP/1.1, +// otherwise it assumes that the original request would have used the same +// protocol as |request| did. This is to account for stuff like HTTP/2 header +// compression. +int64_t EstimateOriginalHeaderBytes(const net::URLRequest& request, + bool used_drp) { + if (used_drp) { + // TODO(sclittle): Remove headers added by Data Reduction Proxy when + // computing original size. https://crbug.com/535701. + return request.response_headers()->raw_headers().size(); + } + return std::max<int64_t>(0, request.GetTotalReceivedBytes() - + request.received_response_content_length()); +} + +// Given a |request| that went through the Data Reduction Proxy if |used_drp| is +// true, this function estimates how many bytes would have been received if the +// response had been received directly from the origin without any data saver +// optimizations. +int64_t EstimateOriginalReceivedBytes( + const net::URLRequest& request, + bool used_drp, + const data_reduction_proxy::LoFiDecider* lofi_decider) { + if (request.was_cached() || !request.response_headers()) + return request.GetTotalReceivedBytes(); + + if (lofi_decider) { + if (lofi_decider->IsClientLoFiAutoReloadRequest(request)) + return 0; + + int64_t first, last, length; + if (lofi_decider->IsClientLoFiImageRequest(request) && + request.response_headers()->GetContentRangeFor206(&first, &last, + &length) && + length > request.received_response_content_length()) { + return EstimateOriginalHeaderBytes(request, used_drp) + length; + } + } + + return used_drp + ? EstimateOriginalHeaderBytes(request, used_drp) + + data_reduction_proxy::util::CalculateEffectiveOCL(request) + : request.GetTotalReceivedBytes(); +} + +} // namespace + +namespace data_reduction_proxy { + +DataReductionProxyDataUseObserver::DataReductionProxyDataUseObserver( + DataReductionProxyIOData* data_reduction_proxy_io_data, + data_use_measurement::DataUseAscriber* data_use_ascriber) + : data_reduction_proxy_io_data_(data_reduction_proxy_io_data), + data_use_ascriber_(data_use_ascriber) { + DCHECK(data_reduction_proxy_io_data_); + data_use_ascriber_->AddObserver(this); +} + +DataReductionProxyDataUseObserver::~DataReductionProxyDataUseObserver() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + data_use_ascriber_->RemoveObserver(this); +} + +void DataReductionProxyDataUseObserver::OnPageLoadCommit( + data_use_measurement::DataUse* data_use) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + if (!data_use->url().is_valid()) + return; + DataUseUserDataBytes* bytes = reinterpret_cast<DataUseUserDataBytes*>( + data_use->GetUserData(DataUseUserDataBytes::kUserDataKey)); + if (bytes) { + // Record the data use bytes saved in user data to database. + data_reduction_proxy_io_data_->UpdateDataUseForHost( + bytes->network_bytes(), bytes->original_bytes(), + data_use->url().HostNoBrackets()); + data_use->RemoveUserData(DataUseUserDataBytes::kUserDataKey); + } +} + +void DataReductionProxyDataUseObserver::OnPageResourceLoad( + const net::URLRequest& request, + data_use_measurement::DataUse* data_use) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + if (data_use->traffic_type() != + data_use_measurement::DataUse::TrafficType::USER_TRAFFIC) + return; + + int64_t network_bytes = request.GetTotalReceivedBytes(); + DataReductionProxyRequestType request_type = GetDataReductionProxyRequestType( + request, data_reduction_proxy_io_data_->configurator()->GetProxyConfig(), + *data_reduction_proxy_io_data_->config()); + + // Estimate how many bytes would have been used if the DataReductionProxy was + // not used, and record the data usage. + int64_t original_bytes = EstimateOriginalReceivedBytes( + request, request_type == VIA_DATA_REDUCTION_PROXY, + data_reduction_proxy_io_data_->lofi_decider()); + + if (!data_use->url().is_valid()) { + // URL will be empty until pageload navigation commits. Save the data use of + // these mainframe, subresource, redirected requests in user data until + // then. + DataUseUserDataBytes* bytes = reinterpret_cast<DataUseUserDataBytes*>( + data_use->GetUserData(DataUseUserDataBytes::kUserDataKey)); + if (bytes) { + bytes->IncrementBytes(network_bytes, original_bytes); + } else { + data_use->SetUserData(DataUseUserDataBytes::kUserDataKey, + base::MakeUnique<DataUseUserDataBytes>( + network_bytes, original_bytes)); + } + } else { + data_reduction_proxy_io_data_->UpdateDataUseForHost( + network_bytes, original_bytes, data_use->url().HostNoBrackets()); + } +} + +} // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.h new file mode 100644 index 00000000000..aec49c3ec62 --- /dev/null +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_data_use_observer.h @@ -0,0 +1,57 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_DATA_USE_OBSERVER_H_ +#define COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_DATA_USE_OBSERVER_H_ + +#include "base/macros.h" +#include "base/threading/thread_checker.h" +#include "components/data_use_measurement/core/data_use.h" +#include "components/data_use_measurement/core/data_use_ascriber.h" + +namespace net { +class URLRequest; +} + +namespace data_reduction_proxy { + +class DataReductionProxyIOData; + +// Observers the page load events from DataUseAscriber and records the data +// usage per site to database. +class DataReductionProxyDataUseObserver + : public data_use_measurement::DataUseAscriber::PageLoadObserver { + public: + // |data_reduction_proxy_io_data| is used to record the bytes to database. + // |data_use_ascriber| is used to listen for events. + // |this| is owned by |data_reduction_proxy_io_data|. + DataReductionProxyDataUseObserver( + DataReductionProxyIOData* data_reduction_proxy_io_data, + data_use_measurement::DataUseAscriber* data_use_ascriber); + + ~DataReductionProxyDataUseObserver() override; + + private: + // PageLoadObserver methods. + void OnPageLoadCommit(data_use_measurement::DataUse* data_use) override; + void OnPageResourceLoad(const net::URLRequest& request, + data_use_measurement::DataUse* data_use) override; + void OnPageLoadComplete(data_use_measurement::DataUse* data_use) override {} + + // |data_reduction_proxy_io_data_| owns |this| and is destroyed only after + // |this| is destroyed in the IO thread. + DataReductionProxyIOData* data_reduction_proxy_io_data_; + + // Used to register for pageload events. |data_use_ascriber_| is owned by IO + // thread globals, and is destroyed after |this|. + data_use_measurement::DataUseAscriber* data_use_ascriber_; + + THREAD_CHECKER(thread_checker_); + + DISALLOW_COPY_AND_ASSIGN(DataReductionProxyDataUseObserver); +}; + +} // namespace data_reduction_proxy + +#endif // COMPONENTS_DATA_REDUCTION_PROXY_CORE_BROWSER_DATA_REDUCTION_PROXY_DATA_USE_OBSERVER_H_ diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc index 12c809f54e4..4985301b706 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.cc @@ -24,6 +24,7 @@ #include "net/http/http_request_headers.h" #include "net/http/http_response_headers.h" #include "net/proxy/proxy_info.h" +#include "net/proxy/proxy_server.h" #include "net/proxy/proxy_service.h" namespace data_reduction_proxy { @@ -85,7 +86,9 @@ void DataReductionProxyDelegate::OnResolveProxy( } std::vector<DataReductionProxyServer> proxies_for_http = - config_->GetProxiesForHttp(); + params::IsIncludedInHoldbackFieldTrial() + ? std::vector<DataReductionProxyServer>() + : config_->GetProxiesForHttp(); // Remove the proxies that are unsupported for this request. proxies_for_http.erase( @@ -216,30 +219,6 @@ void DataReductionProxyDelegate::OnAlternativeProxyBroken( 1); } -net::ProxyServer DataReductionProxyDelegate::GetDefaultAlternativeProxy() - const { - DCHECK(thread_checker_.CalledOnValidThread()); - if (!params::IsZeroRttQuicEnabled()) - return net::ProxyServer(); - - if (alternative_proxies_broken_) { - RecordGetDefaultAlternativeProxy(DEFAULT_ALTERNATIVE_PROXY_STATUS_BROKEN); - return net::ProxyServer(); - } - - net::ProxyServer proxy_server( - net::ProxyServer::SCHEME_QUIC, - net::HostPortPair(kDataReductionCoreProxy, 443)); - if (!config_ || !config_->IsDataReductionProxy(proxy_server, NULL)) { - RecordGetDefaultAlternativeProxy( - DEFAULT_ALTERNATIVE_PROXY_STATUS_UNAVAILABLE); - return net::ProxyServer(); - } - - RecordGetDefaultAlternativeProxy(DEFAULT_ALTERNATIVE_PROXY_STATUS_AVAILABLE); - return proxy_server; -} - bool DataReductionProxyDelegate::SupportsQUIC( const net::ProxyServer& proxy_server) const { DCHECK(thread_checker_.CalledOnValidThread()); @@ -257,13 +236,6 @@ void DataReductionProxyDelegate::RecordQuicProxyStatus( QUIC_PROXY_STATUS_BOUNDARY); } -void DataReductionProxyDelegate::RecordGetDefaultAlternativeProxy( - DefaultAlternativeProxyStatus status) const { - DCHECK(thread_checker_.CalledOnValidThread()); - UMA_HISTOGRAM_ENUMERATION("DataReductionProxy.Quic.DefaultAlternativeProxy", - status, DEFAULT_ALTERNATIVE_PROXY_STATUS_BOUNDARY); -} - void DataReductionProxyDelegate::OnIPAddressChanged() { DCHECK(thread_checker_.CalledOnValidThread()); first_data_saver_request_recorded_ = false; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h index d412dc3b1cc..7179998df49 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate.h @@ -12,7 +12,6 @@ #include "net/base/network_change_notifier.h" #include "net/base/proxy_delegate.h" #include "net/proxy/proxy_retry_info.h" -#include "net/proxy/proxy_server.h" #include "url/gurl.h" namespace base { @@ -26,6 +25,7 @@ class HttpResponseHeaders; class NetLog; class ProxyConfig; class ProxyInfo; +class ProxyServer; class ProxyService; } @@ -82,7 +82,6 @@ class DataReductionProxyDelegate net::ProxyServer* alternative_proxy_server) const override; void OnAlternativeProxyBroken( const net::ProxyServer& alternative_proxy_server) override; - net::ProxyServer GetDefaultAlternativeProxy() const override; // Protected so that it can be overridden during testing. // Returns true if |proxy_server| supports QUIC. @@ -97,24 +96,10 @@ class DataReductionProxyDelegate QUIC_PROXY_STATUS_BOUNDARY }; - // Availability status of data reduction proxy that supports 0-RTT QUIC. - // Protected so that the enum values are accessible for testing. - enum DefaultAlternativeProxyStatus { - DEFAULT_ALTERNATIVE_PROXY_STATUS_AVAILABLE, - DEFAULT_ALTERNATIVE_PROXY_STATUS_BROKEN, - DEFAULT_ALTERNATIVE_PROXY_STATUS_UNAVAILABLE, - DEFAULT_ALTERNATIVE_PROXY_STATUS_BOUNDARY, - }; - private: // Records the availability status of data reduction proxy. void RecordQuicProxyStatus(QuicProxyStatus status) const; - // Records the availability status of data reduction proxy that supports 0-RTT - // QUIC. - void RecordGetDefaultAlternativeProxy( - DefaultAlternativeProxyStatus status) const; - // NetworkChangeNotifier::IPAddressObserver: void OnIPAddressChanged() override; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc index f7a2306fd34..5c4f2c15a4a 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_delegate_unittest.cc @@ -40,7 +40,6 @@ #include "components/data_reduction_proxy/core/common/data_reduction_proxy_server.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h" #include "components/data_reduction_proxy/proto/client_config.pb.h" -#include "components/variations/variations_associated_data.h" #include "net/base/host_port_pair.h" #include "net/base/net_errors.h" #include "net/base/network_change_notifier.h" @@ -51,6 +50,7 @@ #include "net/proxy/proxy_config.h" #include "net/proxy/proxy_server.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_test_util.h" #include "testing/gtest/include/gtest/gtest.h" @@ -132,49 +132,9 @@ class TestDataReductionProxyDelegate : public DataReductionProxyDelegate { } } - void VerifyGetDefaultAlternativeProxyHistogram( - const base::HistogramTester& histogram_tester, - bool is_in_quic_field_trial, - bool use_proxyzip_proxy_as_first_proxy, - bool alternative_proxy_broken) { - static const char kHistogram[] = - "DataReductionProxy.Quic.DefaultAlternativeProxy"; - if (is_in_quic_field_trial && use_proxyzip_proxy_as_first_proxy && - !alternative_proxy_broken) { - histogram_tester.ExpectUniqueSample( - kHistogram, - TestDataReductionProxyDelegate::DefaultAlternativeProxyStatus:: - DEFAULT_ALTERNATIVE_PROXY_STATUS_AVAILABLE, - 1); - return; - } - - if (is_in_quic_field_trial && alternative_proxy_broken) { - histogram_tester.ExpectUniqueSample( - kHistogram, - TestDataReductionProxyDelegate::DefaultAlternativeProxyStatus:: - DEFAULT_ALTERNATIVE_PROXY_STATUS_BROKEN, - 1); - return; - } - - if (is_in_quic_field_trial && !use_proxyzip_proxy_as_first_proxy) { - histogram_tester.ExpectUniqueSample( - kHistogram, - TestDataReductionProxyDelegate::DefaultAlternativeProxyStatus:: - DEFAULT_ALTERNATIVE_PROXY_STATUS_UNAVAILABLE, - 1); - return; - } - - histogram_tester.ExpectTotalCount(kHistogram, 0); - } - using DataReductionProxyDelegate::GetAlternativeProxy; using DataReductionProxyDelegate::OnAlternativeProxyBroken; - using DataReductionProxyDelegate::GetDefaultAlternativeProxy; using DataReductionProxyDelegate::QuicProxyStatus; - using DataReductionProxyDelegate::DefaultAlternativeProxyStatus; private: const bool proxy_supports_quic_; @@ -474,109 +434,6 @@ TEST(DataReductionProxyDelegate, AlternativeProxy) { } } -// Verifies that DataReductionProxyDelegate correctly returns the proxy server -// that supports 0-RTT. -TEST(DataReductionProxyDelegate, DefaultAlternativeProxyStatus) { - base::MessageLoopForIO message_loop_; - std::unique_ptr<DataReductionProxyTestContext> test_context = - DataReductionProxyTestContext::Builder() - .WithConfigClient() - .WithMockDataReductionProxyService() - .Build(); - - const struct { - bool is_in_quic_field_trial; - bool zero_rtt_param_set; - bool use_proxyzip_proxy_as_first_proxy; - } tests[] = {{false, false, false}, - {false, false, true}, - {true, false, false}, - {true, false, true}, - {true, true, true}}; - for (const auto test : tests) { - std::vector<DataReductionProxyServer> proxies_for_http; - net::ProxyServer first_proxy; - net::ProxyServer second_proxy = - GetProxyWithScheme(net::ProxyServer::SCHEME_HTTP); - - if (test.use_proxyzip_proxy_as_first_proxy) { - first_proxy = - net::ProxyServer(net::ProxyServer::SCHEME_QUIC, - net::HostPortPair("proxy.googlezip.net", 443)); - } else { - first_proxy = GetProxyWithScheme(net::ProxyServer::SCHEME_HTTPS); - } - - proxies_for_http.push_back( - DataReductionProxyServer(first_proxy, ProxyServer::CORE)); - proxies_for_http.push_back( - DataReductionProxyServer(second_proxy, ProxyServer::UNSPECIFIED_TYPE)); - - std::unique_ptr<DataReductionProxyMutableConfigValues> config_values = - DataReductionProxyMutableConfigValues::CreateFromParams( - test_context->test_params()); - config_values->UpdateValues(proxies_for_http); - - std::unique_ptr<DataReductionProxyConfig> config( - new DataReductionProxyConfig( - message_loop_.task_runner(), test_context->net_log(), - std::move(config_values), test_context->configurator(), - test_context->event_creator())); - - TestDataReductionProxyDelegate delegate( - config.get(), test_context->io_data()->configurator(), - test_context->io_data()->event_creator(), - test_context->io_data()->bypass_stats(), true, - test_context->io_data()->net_log()); - - variations::testing::ClearAllVariationParams(); - std::map<std::string, std::string> variation_params; - if (test.zero_rtt_param_set) - variation_params["enable_zero_rtt"] = "true"; - ASSERT_TRUE(variations::AssociateVariationParams( - params::GetQuicFieldTrialName(), - test.is_in_quic_field_trial ? "Enabled" : "Control", variation_params)); - base::FieldTrialList field_trial_list(nullptr); - base::FieldTrialList::CreateFieldTrial( - params::GetQuicFieldTrialName(), - test.is_in_quic_field_trial ? "Enabled" : "Control"); - - { - // Test if the QUIC supporting proxy is correctly set. - base::HistogramTester histogram_tester; - if (test.is_in_quic_field_trial && test.zero_rtt_param_set && - test.use_proxyzip_proxy_as_first_proxy) { - EXPECT_EQ(delegate.GetDefaultAlternativeProxy(), first_proxy); - EXPECT_TRUE(first_proxy.is_quic()); - - } else { - EXPECT_FALSE(delegate.GetDefaultAlternativeProxy().is_valid()); - } - - delegate.VerifyGetDefaultAlternativeProxyHistogram( - histogram_tester, - test.is_in_quic_field_trial && test.zero_rtt_param_set, - test.use_proxyzip_proxy_as_first_proxy, false); - } - - { - // Test if the QUIC supporting proxy is correctly set if the proxy is - // marked as broken. - base::HistogramTester histogram_tester; - - if (test.is_in_quic_field_trial && test.zero_rtt_param_set && - test.use_proxyzip_proxy_as_first_proxy) { - delegate.OnAlternativeProxyBroken(first_proxy); - EXPECT_FALSE(delegate.GetDefaultAlternativeProxy().is_quic()); - delegate.VerifyGetDefaultAlternativeProxyHistogram( - histogram_tester, - test.is_in_quic_field_trial && test.zero_rtt_param_set, - test.use_proxyzip_proxy_as_first_proxy, true); - } - } - } -} - #if defined(OS_ANDROID) const Client kClient = Client::CHROME_ANDROID; #elif defined(OS_IOS) @@ -658,8 +515,8 @@ class DataReductionProxyDelegateTest : public testing::Test { mock_socket_factory_.AddSocketDataProvider(&socket); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(url, net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = context_.CreateRequest( + url, net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); if (request_headers) request->SetExtraRequestHeaders(*request_headers); @@ -1003,6 +860,36 @@ TEST_F(DataReductionProxyDelegateTest, TimeToFirstHttpDataSaverRequest) { } } +TEST_F(DataReductionProxyDelegateTest, Holdback) { + const char kResponseHeaders[] = + "HTTP/1.1 200 OK\r\n" + "Via: 1.1 Chrome-Compression-Proxy-Suffix\r\n" + "Content-Length: 10\r\n\r\n"; + + const struct { + bool holdback; + } tests[] = { + { + true, + }, + { + false, + }, + }; + for (const auto& test : tests) { + base::FieldTrialList field_trial_list(nullptr); + ASSERT_TRUE(base::FieldTrialList::CreateFieldTrial( + "DataCompressionProxyHoldback", test.holdback ? "Enabled" : "Control")); + + base::HistogramTester histogram_tester; + FetchURLRequest(GURL("http://example.com/path/"), nullptr, kResponseHeaders, + 10); + histogram_tester.ExpectTotalCount( + "DataReductionProxy.SuccessfulRequestCompletionCounts", + test.holdback ? 0 : 1); + } +} + TEST_F(DataReductionProxyDelegateTest, OnCompletedSizeFor304) { int64_t baseline_received_bytes = total_received_bytes(); int64_t baseline_original_received_bytes = total_original_received_bytes(); @@ -1037,8 +924,9 @@ TEST_F(DataReductionProxyDelegateTest, OnCompletedSizeForWriteError) { mock_socket_factory()->AddSocketDataProvider(&socket); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL("http://example.com/path/"), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL("http://example.com/path/"), net::IDLE, + &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); request->Start(); base::RunLoop().RunUntilIdle(); @@ -1058,8 +946,9 @@ TEST_F(DataReductionProxyDelegateTest, OnCompletedSizeForReadError) { mock_socket_factory()->AddSocketDataProvider(&socket); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL("http://example.com/path/"), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL("http://example.com/path/"), net::IDLE, + &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); request->Start(); base::RunLoop().RunUntilIdle(); @@ -1150,8 +1039,9 @@ TEST_F(DataReductionProxyDelegateTest, PartialRangeSavings) { mock_socket_factory()->AddSocketDataProvider(&socket); net::TestDelegate test_delegate; - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL("http://example.com"), net::IDLE, &test_delegate); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL("http://example.com"), net::IDLE, + &test_delegate, TRAFFIC_ANNOTATION_FOR_TESTS); request->Start(); base::RunLoop().RunUntilIdle(); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc index 4d8f08897a2..5790d4403d4 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_interceptor_unittest.cc @@ -25,6 +25,7 @@ #include "components/data_reduction_proxy/core/common/data_reduction_proxy_server.h" #include "components/data_reduction_proxy/proto/client_config.pb.h" #include "components/prefs/pref_service.h" +#include "net/base/load_flags.h" #include "net/base/net_errors.h" #include "net/base/request_priority.h" #include "net/http/http_response_headers.h" @@ -32,6 +33,7 @@ #include "net/proxy/proxy_server.h" #include "net/socket/socket_test_util.h" #include "net/test/embedded_test_server/embedded_test_server.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context_storage.h" #include "net/url_request/url_request_intercepting_job_factory.h" @@ -172,8 +174,9 @@ TEST_F(DataReductionProxyInterceptorTest, MAYBE_TestJobFactoryChaining) { Init(std::move(factory1)); net::TestDelegate d; - std::unique_ptr<net::URLRequest> req(default_context_->CreateRequest( - GURL("http://foo"), net::DEFAULT_PRIORITY, &d)); + std::unique_ptr<net::URLRequest> req( + default_context_->CreateRequest(GURL("http://foo"), net::DEFAULT_PRIORITY, + &d, TRAFFIC_ANNOTATION_FOR_TESTS)); req->Start(); base::RunLoop().Run(); @@ -262,7 +265,8 @@ TEST_F(DataReductionProxyInterceptorWithServerTest, TestBypass) { // DataReductionProxyProtocolTest. net::TestDelegate delegate; std::unique_ptr<net::URLRequest> request(context().CreateRequest( - direct().GetURL("/block10.html"), net::DEFAULT_PRIORITY, &delegate)); + direct().GetURL("/block10.html"), net::DEFAULT_PRIORITY, &delegate, + TRAFFIC_ANNOTATION_FOR_TESTS)); request->Start(); EXPECT_TRUE(request->is_pending()); base::RunLoop().Run(); @@ -274,7 +278,8 @@ TEST_F(DataReductionProxyInterceptorWithServerTest, TestBypass) { TEST_F(DataReductionProxyInterceptorWithServerTest, TestNoBypass) { net::TestDelegate delegate; std::unique_ptr<net::URLRequest> request(context().CreateRequest( - direct().GetURL("/noblock.html"), net::DEFAULT_PRIORITY, &delegate)); + direct().GetURL("/noblock.html"), net::DEFAULT_PRIORITY, &delegate, + TRAFFIC_ANNOTATION_FOR_TESTS)); request->Start(); EXPECT_TRUE(request->is_pending()); base::RunLoop().Run(); @@ -312,8 +317,8 @@ class DataReductionProxyInterceptorEndToEndTest : public testing::Test { // Creates a URLRequest using the test's TestURLRequestContext and executes // it. Returns the created URLRequest. std::unique_ptr<net::URLRequest> CreateAndExecuteRequest(const GURL& url) { - std::unique_ptr<net::URLRequest> request( - context_.CreateRequest(url, net::IDLE, &delegate_)); + std::unique_ptr<net::URLRequest> request(context_.CreateRequest( + url, net::IDLE, &delegate_, TRAFFIC_ANNOTATION_FOR_TESTS)); request->Start(); drp_test_context_->RunUntilIdle(); return request; @@ -541,13 +546,7 @@ TEST_F(DataReductionProxyInterceptorEndToEndTest, RedirectWithBypassAndRetry) { EXPECT_EQ(std::vector<GURL>(1, GURL("http://foo.com")), request->url_chain()); } -// https://crbug.com/668197: Flaky on android_n5x_swarming_rel bot. -#if defined(OS_ANDROID) -#define MAYBE_RedirectChainToHttps DISABLED_RedirectChainToHttps -#else -#define MAYBE_RedirectChainToHttps RedirectChainToHttps -#endif -TEST_F(DataReductionProxyInterceptorEndToEndTest, MAYBE_RedirectChainToHttps) { +TEST_F(DataReductionProxyInterceptorEndToEndTest, RedirectChainToHttps) { // First, a redirect is successfully received through the Data Reduction // Proxy. HSTS is forced for play.google.com and prebaked into Chrome, so // http://play.google.com will automatically be redirected to @@ -579,6 +578,9 @@ TEST_F(DataReductionProxyInterceptorEndToEndTest, MAYBE_RedirectChainToHttps) { std::unique_ptr<net::URLRequest> request = CreateAndExecuteRequest(GURL("http://music.google.com")); + request->SetLoadFlags(net::LOAD_DISABLE_CACHE | + net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); EXPECT_FALSE(delegate().request_failed()); EXPECT_EQ(kBody, delegate().data_received()); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.cc index bdfbb43b48f..90318bc3eea 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.cc @@ -291,29 +291,43 @@ void DataReductionProxyIOData::SetDataReductionProxyConfiguration( } bool DataReductionProxyIOData::ShouldEnableLoFi( - const net::URLRequest& request) { + const net::URLRequest& request, + previews::PreviewsDecider* previews_decider) { + DCHECK(previews_decider); DCHECK((request.load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) != 0); if (!config_ || (config_->IsBypassedByDataReductionProxyLocalRules( request, configurator_->GetProxyConfig()))) { return false; } - return config_->ShouldEnableLoFi(request); + return config_->ShouldEnableLoFi(request, *previews_decider); } bool DataReductionProxyIOData::ShouldEnableLitePages( - const net::URLRequest& request) { + const net::URLRequest& request, + previews::PreviewsDecider* previews_decider) { + DCHECK(previews_decider); DCHECK((request.load_flags() & net::LOAD_MAIN_FRAME_DEPRECATED) != 0); if (!config_ || (config_->IsBypassedByDataReductionProxyLocalRules( request, configurator_->GetProxyConfig()))) { return false; } - return config_->ShouldEnableLitePages(request); + return config_->ShouldEnableLitePages(request, *previews_decider); } void DataReductionProxyIOData::SetLoFiModeOff() { config_->SetLoFiModeOff(); } +void DataReductionProxyIOData::UpdateDataUseForHost(int64_t network_bytes, + int64_t original_bytes, + const std::string& host) { + DCHECK(io_task_runner_->BelongsToCurrentThread()); + + ui_task_runner_->PostTask( + FROM_HERE, base::Bind(&DataReductionProxyService::UpdateDataUseForHost, + service_, network_bytes, original_bytes, host)); +} + void DataReductionProxyIOData::UpdateContentLengths( int64_t data_used, int64_t original_size, diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h index 22a90a8397e..951a32bfc92 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h @@ -36,6 +36,10 @@ class URLRequestContextGetter; class URLRequestInterceptor; } +namespace previews { +class PreviewsDecider; +} + namespace data_reduction_proxy { class DataReductionProxyBypassStats; @@ -98,17 +102,26 @@ class DataReductionProxyIOData : public DataReductionProxyEventStorageDelegate { // Returns true when Lo-Fi Previews should be activated. When Lo-Fi is // active, URL requests are modified to request low fidelity versions of the // resources, except when the user is in the Lo-Fi control group. - bool ShouldEnableLoFi(const net::URLRequest& request); + // |previews_decider| is a non-null object that determines eligibility of + // showing the preview based on past opt outs. + bool ShouldEnableLoFi(const net::URLRequest& request, + previews::PreviewsDecider* previews_decider); // Returns true when Lite Page Previews should be activated. When Lite Pages // are active, a low fidelity transcoded page is requested on the main frame - // resource, except when the user is in the control group. - bool ShouldEnableLitePages(const net::URLRequest& request); + // resource, except when the user is in the control group. |previews_decider| + // is a non-null object that determines eligibility of showing the preview + // based on past opt outs. + bool ShouldEnableLitePages(const net::URLRequest& request, + previews::PreviewsDecider* previews_decider); // Sets Lo-Fi mode off in |config_|. void SetLoFiModeOff(); // Bridge methods to safely call to the UI thread objects. + void UpdateDataUseForHost(int64_t network_bytes, + int64_t original_bytes, + const std::string& host); void UpdateContentLengths( int64_t data_used, int64_t original_size, diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data_unittest.cc index 7b09403fd4c..fb75ff2e24e 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data_unittest.cc @@ -8,6 +8,7 @@ #include "base/memory/ptr_util.h" #include "base/memory/ref_counted.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/run_loop.h" #include "base/single_thread_task_runner.h" @@ -27,6 +28,7 @@ #include "net/log/net_log_with_source.h" #include "net/proxy/proxy_info.h" #include "net/proxy/proxy_service.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request_context.h" #include "net/url_request/url_request_context_getter.h" #include "net/url_request/url_request_interceptor.h" @@ -123,8 +125,9 @@ TEST_F(DataReductionProxyIODataTest, TestConstruction) { // When creating a network delegate, expect that it properly wraps a // network delegate. Such a network delegate is thoroughly tested by // DataReductionProxyNetworkDelegateTest. - std::unique_ptr<net::URLRequest> fake_request = context().CreateRequest( - GURL("http://www.foo.com/"), net::IDLE, delegate()); + std::unique_ptr<net::URLRequest> fake_request = + context().CreateRequest(GURL("http://www.foo.com/"), net::IDLE, + delegate(), TRAFFIC_ANNOTATION_FOR_TESTS); CountingNetworkDelegate* wrapped_network_delegate = new CountingNetworkDelegate(); std::unique_ptr<DataReductionProxyNetworkDelegate> network_delegate = diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics_unittest.cc index da8d668cad6..59f6f3d8825 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_metrics_unittest.cc @@ -9,6 +9,7 @@ #include "base/compiler_specific.h" #include "base/macros.h" +#include "base/message_loop/message_loop.h" #include "base/time/time.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config_test_utils.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_test_utils.h" @@ -20,6 +21,7 @@ #include "net/proxy/proxy_server.h" #include "net/proxy/proxy_service.h" #include "net/socket/socket_test_util.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_test_util.h" #include "testing/gtest/include/gtest/gtest.h" @@ -176,8 +178,8 @@ TEST(ChromeNetworkDailyDataSavingMetricsTest, mock_socket_factory.AddSocketDataProvider(&socket_data_provider); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = - context.CreateRequest(test_case.url, net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = context.CreateRequest( + test_case.url, net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); request->SetLoadFlags(test_case.load_flags); request->Start(); test_context->RunUntilIdle(); diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc index 12d8b56770d..97fbec5be1c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.cc @@ -4,12 +4,16 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h" +#include <algorithm> #include <limits> #include <utility> #include "base/memory/ptr_util.h" +#include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" #include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" +#include "base/strings/stringprintf.h" #include "base/time/time.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_bypass_stats.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h" @@ -19,6 +23,7 @@ #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.h" #include "components/data_reduction_proxy/core/browser/data_use_group.h" #include "components/data_reduction_proxy/core/browser/data_use_group_provider.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_util.h" #include "components/data_reduction_proxy/core/common/lofi_decider.h" @@ -26,6 +31,7 @@ #include "net/base/mime_util.h" #include "net/http/http_request_headers.h" #include "net/http/http_response_headers.h" +#include "net/nqe/effective_connection_type.h" #include "net/nqe/network_quality_estimator.h" #include "net/proxy/proxy_info.h" #include "net/proxy/proxy_server.h" @@ -39,19 +45,69 @@ namespace data_reduction_proxy { namespace { -// |lofi_low_header_added| is set to true iff Lo-Fi "q=low" request header can -// be added to the Chrome proxy headers. -// |received_content_length| is the number of prefilter bytes received. -// |original_content_length| is the length of resource if accessed directly -// without data saver proxy. -// |freshness_lifetime| contains information on how long the resource will be -// fresh for and how long is the usability. +// Records the occurrence of |sample| in |name| histogram. UMA macros are not +// used because the |name| is not static. +void RecordNewContentLengthHistogram(const std::string& name, int64_t sample) { + base::UmaHistogramCustomCounts( + name, sample, + 1, // Minimum sample size in bytes. + 128 << 20, // Maximum sample size in bytes. 128MB is chosen because some + // video requests can be very large. + 50 // Bucket count. + ); +} + +void RecordNewContentLengthHistograms( + const char* prefix, + bool is_https, + bool is_video, + DataReductionProxyRequestType request_type, + int64_t content_length) { + const char* connection_type = is_https ? ".Https" : ".Http"; + const char* suffix = ".Other"; + // TODO(crbug.com/726411): Differentiate between a bypass and a disabled + // proxy config. + switch (request_type) { + case VIA_DATA_REDUCTION_PROXY: + suffix = ".ViaDRP"; + break; + case HTTPS: + suffix = ".Direct"; + break; + case SHORT_BYPASS: + case LONG_BYPASS: + suffix = ".BypassedDRP"; + break; + case UPDATE: + case UNKNOWN_TYPE: + default: + // Value already properly initialized to ".Other" + break; + } + // Record a histogram for all traffic, including video. + RecordNewContentLengthHistogram( + base::StringPrintf("%s%s%s", prefix, connection_type, suffix), + content_length); + if (is_video) { + RecordNewContentLengthHistogram( + base::StringPrintf("%s%s%s.Video", prefix, connection_type, suffix), + content_length); + } +} + +// |lofi_low_header_added| is set to true iff Lo-Fi request header +// can be added to the Chrome proxy header. |received_content_length| is +// the number of prefilter bytes received. |original_content_length| is the +// length of resource if accessed directly without data saver proxy. +// |freshness_lifetime| specifies how long the resource will +// be fresh for. void RecordContentLengthHistograms(bool lofi_low_header_added, bool is_https, bool is_video, int64_t received_content_length, int64_t original_content_length, - const base::TimeDelta& freshness_lifetime) { + const base::TimeDelta& freshness_lifetime, + DataReductionProxyRequestType request_type) { // Add the current resource to these histograms only when a valid // X-Original-Content-Length header is present. if (original_content_length >= 0) { @@ -80,17 +136,14 @@ void RecordContentLengthHistograms(bool lofi_low_header_added, original_content_length = received_content_length; } UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength", received_content_length); - if (is_https) { - UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength.Https", - received_content_length); - } else { - UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength.Http", - received_content_length); - } - if (is_video) { - UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLength.Video", - received_content_length); - } + + // Record the new histograms broken down by HTTP/HTTPS and video/non-video + RecordNewContentLengthHistograms("Net.HttpContentLength", is_https, is_video, + request_type, received_content_length); + RecordNewContentLengthHistograms("Net.HttpOriginalContentLength", is_https, + is_video, request_type, + original_content_length); + UMA_HISTOGRAM_COUNTS_1M("Net.HttpOriginalContentLength", original_content_length); UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthDifference", @@ -98,8 +151,7 @@ void RecordContentLengthHistograms(bool lofi_low_header_added, UMA_HISTOGRAM_CUSTOM_COUNTS("Net.HttpContentFreshnessLifetime", freshness_lifetime.InSeconds(), base::TimeDelta::FromHours(1).InSeconds(), - base::TimeDelta::FromDays(30).InSeconds(), - 100); + base::TimeDelta::FromDays(30).InSeconds(), 100); if (freshness_lifetime.InSeconds() <= 0) return; UMA_HISTOGRAM_COUNTS_1M("Net.HttpContentLengthCacheable", @@ -115,18 +167,52 @@ void RecordContentLengthHistograms(bool lofi_low_header_added, received_content_length); } -// Given a |request| that went through the Data Reduction Proxy, this function -// estimates how many bytes would have been received if the response had been -// received directly from the origin using HTTP/1.1 with a content length of -// |adjusted_original_content_length|. -int64_t EstimateOriginalReceivedBytes(const net::URLRequest& request) { +// Estimate the size of the original headers of |request|. If |used_drp| is +// true, then it's assumed that the original request would have used HTTP/1.1, +// otherwise it assumes that the original request would have used the same +// protocol as |request| did. This is to account for stuff like HTTP/2 header +// compression. +// TODO(rajendrant): Remove this method when data use ascriber observers are +// used to record the per-site data usage. +int64_t EstimateOriginalHeaderBytes(const net::URLRequest& request, + bool used_drp) { + if (used_drp) { + // TODO(sclittle): Remove headers added by Data Reduction Proxy when + // computing original size. https://crbug.com/535701. + return request.response_headers()->raw_headers().size(); + } + return std::max<int64_t>(0, request.GetTotalReceivedBytes() - + request.received_response_content_length()); +} + +// Given a |request| that went through the Data Reduction Proxy if |used_drp| is +// true, this function estimates how many bytes would have been received if the +// response had been received directly from the origin without any data saver +// optimizations. +// TODO(rajendrant): Remove this method when data use ascriber observers are +// used to record the per-site data usage. +int64_t EstimateOriginalReceivedBytes(const net::URLRequest& request, + bool used_drp, + const LoFiDecider* lofi_decider) { if (request.was_cached() || !request.response_headers()) return request.GetTotalReceivedBytes(); - // TODO(sclittle): Remove headers added by Data Reduction Proxy when computing - // original size. http://crbug/535701. - return request.response_headers()->raw_headers().size() + - util::CalculateEffectiveOCL(request); + if (lofi_decider) { + if (lofi_decider->IsClientLoFiAutoReloadRequest(request)) + return 0; + + int64_t first, last, length; + if (lofi_decider->IsClientLoFiImageRequest(request) && + request.response_headers()->GetContentRangeFor206(&first, &last, + &length) && + length > request.received_response_content_length()) { + return EstimateOriginalHeaderBytes(request, used_drp) + length; + } + } + + return used_drp ? EstimateOriginalHeaderBytes(request, used_drp) + + util::CalculateEffectiveOCL(request) + : request.GetTotalReceivedBytes(); } // Verifies that the chrome proxy related request headers are set correctly. @@ -134,11 +220,25 @@ int64_t EstimateOriginalReceivedBytes(const net::URLRequest& request) { // Saver proxy. void VerifyHttpRequestHeaders(bool via_chrome_proxy, const net::HttpRequestHeaders& headers) { + // If holdback is enabled, then |via_chrome_proxy| should be false. + DCHECK(!params::IsIncludedInHoldbackFieldTrial() || !via_chrome_proxy); + if (via_chrome_proxy) { - DCHECK(headers.HasHeader(chrome_proxy_header())); + DCHECK(headers.HasHeader(chrome_proxy_ect_header())); + std::string chrome_proxy_header_value; + DCHECK( + headers.GetHeader(chrome_proxy_header(), &chrome_proxy_header_value)); + // Check that only 1 "exp" directive is sent. + DCHECK_GT(3u, base::SplitStringUsingSubstr(chrome_proxy_header_value, + "exp=", base::TRIM_WHITESPACE, + base::SPLIT_WANT_ALL) + .size()); + // Silence unused variable warning in release builds. + (void)chrome_proxy_header_value; } else { DCHECK(!headers.HasHeader(chrome_proxy_header())); DCHECK(!headers.HasHeader(chrome_proxy_accept_transform_header())); + DCHECK(!headers.HasHeader(chrome_proxy_ect_header())); } } @@ -223,6 +323,8 @@ void DataReductionProxyNetworkDelegate::OnBeforeStartTransactionInternal( ->MaybeSetAcceptTransformHeader( *request, data_reduction_proxy_config_->lofi_off(), headers); } + + MaybeAddChromeProxyECTHeader(headers, *request); } void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( @@ -249,14 +351,17 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( DataReductionProxyData::ClearData(request); if (params::IsIncludedInHoldbackFieldTrial()) { - if (!WasEligibleWithoutHoldback(*request, proxy_info, proxy_retry_info)) - return; - // For the holdback field trial, still log UMA as if the proxy was used. - data = DataReductionProxyData::GetDataAndCreateIfNecessary(request); - if (data) - data->set_used_data_reduction_proxy(true); - VerifyHttpRequestHeaders(false, *headers); - return; + if (WasEligibleWithoutHoldback(*request, proxy_info, proxy_retry_info)) { + // For the holdback field trial, still log UMA as if the proxy was used. + data = DataReductionProxyData::GetDataAndCreateIfNecessary(request); + if (data) + data->set_used_data_reduction_proxy(true); + } + // If holdback is enabled, |proxy_info| must not contain a data reduction + // proxy. + DCHECK(proxy_info.is_empty() || + !data_reduction_proxy_config_->IsDataReductionProxy( + proxy_info.proxy_server(), nullptr)); } bool using_data_reduction_proxy = true; @@ -270,6 +375,9 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( proxy_info.proxy_server(), nullptr)) { using_data_reduction_proxy = false; } + // If holdback is enabled, |using_data_reduction_proxy| must be false. + DCHECK(!params::IsIncludedInHoldbackFieldTrial() || + !using_data_reduction_proxy); LoFiDecider* lofi_decider = nullptr; if (data_reduction_proxy_io_data_) @@ -281,6 +389,8 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( // Chrome-Proxy-Accept-Transform header. lofi_decider->RemoveAcceptTransformHeader(headers); } + RemoveChromeProxyECTHeader(headers); + headers->RemoveHeader(chrome_proxy_header()); VerifyHttpRequestHeaders(false, *headers); return; } @@ -329,8 +439,6 @@ void DataReductionProxyNetworkDelegate::OnBeforeSendHeadersInternal( data_reduction_proxy_request_options_->AddRequestHeader(headers, page_id); - if (lofi_decider) - lofi_decider->MaybeSetIgnorePreviewsBlacklistDirective(headers); VerifyHttpRequestHeaders(true, *headers); } @@ -373,8 +481,15 @@ void DataReductionProxyNetworkDelegate::OnCompletedInternal( net_error); net::HttpRequestHeaders request_headers; - if (data_reduction_proxy_io_data_ && request->response_headers() && - IsEmptyImagePreview(*(request->response_headers()))) { + bool server_lofi = request->response_headers() && + IsEmptyImagePreview(*(request->response_headers())); + bool client_lofi = + data_reduction_proxy_io_data_ && + data_reduction_proxy_io_data_->lofi_decider() && + data_reduction_proxy_io_data_->lofi_decider()->IsClientLoFiImageRequest( + *request); + if ((server_lofi || client_lofi) && data_reduction_proxy_io_data_ && + data_reduction_proxy_io_data_->lofi_ui_service()) { data_reduction_proxy_io_data_->lofi_ui_service()->OnLoFiReponseReceived( *request); } else if (data_reduction_proxy_io_data_ && request->response_headers() && @@ -416,7 +531,8 @@ void DataReductionProxyNetworkDelegate::OnHeadersReceivedInternal( const net::HttpResponseHeaders* original_response_headers, scoped_refptr<net::HttpResponseHeaders>* override_response_headers, GURL* allowed_unsafe_redirect_url) { - if (!original_response_headers) + if (!original_response_headers || + original_response_headers->IsRedirect(nullptr)) return; if (IsEmptyImagePreview(*original_response_headers)) { DataReductionProxyData* data = @@ -427,6 +543,14 @@ void DataReductionProxyNetworkDelegate::OnHeadersReceivedInternal( DataReductionProxyData::GetDataAndCreateIfNecessary(request); data->set_lite_page_received(true); } + if (data_reduction_proxy_io_data_ && + data_reduction_proxy_io_data_->lofi_decider() && + data_reduction_proxy_io_data_->lofi_decider()->IsClientLoFiImageRequest( + *request)) { + DataReductionProxyData* data = + DataReductionProxyData::GetDataAndCreateIfNecessary(request); + data->set_client_lofi_requested(true); + } } void DataReductionProxyNetworkDelegate::CalculateAndRecordDataUsage( @@ -437,10 +561,11 @@ void DataReductionProxyNetworkDelegate::CalculateAndRecordDataUsage( // Estimate how many bytes would have been used if the DataReductionProxy was // not used, and record the data usage. - int64_t original_size = data_used; - - if (request_type == VIA_DATA_REDUCTION_PROXY) - original_size = EstimateOriginalReceivedBytes(request); + int64_t original_size = EstimateOriginalReceivedBytes( + request, request_type == VIA_DATA_REDUCTION_PROXY, + data_reduction_proxy_io_data_ + ? data_reduction_proxy_io_data_->lofi_decider() + : nullptr); std::string mime_type; if (request.response_headers()) @@ -500,7 +625,7 @@ void DataReductionProxyNetworkDelegate::RecordContentLength( data_reduction_proxy_io_data_->lofi_decider() && data_reduction_proxy_io_data_->lofi_decider()->IsUsingLoFi(request), is_https, is_video, request.received_response_content_length(), - original_content_length, freshness_lifetime); + original_content_length, freshness_lifetime, request_type); if (data_reduction_proxy_io_data_ && data_reduction_proxy_bypass_stats_) { // Record BypassedBytes histograms for the request. @@ -596,4 +721,48 @@ void DataReductionProxyNetworkDelegate::MaybeAddBrotliToAcceptEncodingHeader( header_value); } +void DataReductionProxyNetworkDelegate::MaybeAddChromeProxyECTHeader( + net::HttpRequestHeaders* request_headers, + const net::URLRequest& request) const { + DCHECK(thread_checker_.CalledOnValidThread()); + + // This method should be called only when the resolved proxy was a data + // saver proxy. + DCHECK(request.url().is_valid()); + DCHECK(!request.url().SchemeIsCryptographic()); + DCHECK(request.url().SchemeIsHTTPOrHTTPS()); + + if (request_headers->HasHeader(chrome_proxy_ect_header())) + request_headers->RemoveHeader(chrome_proxy_ect_header()); + + if (request.context()->network_quality_estimator()) { + net::EffectiveConnectionType type = request.context() + ->network_quality_estimator() + ->GetEffectiveConnectionType(); + if (type > net::EFFECTIVE_CONNECTION_TYPE_OFFLINE) { + DCHECK_NE(net::EFFECTIVE_CONNECTION_TYPE_LAST, type); + request_headers->SetHeader(chrome_proxy_ect_header(), + net::GetNameForEffectiveConnectionType(type)); + return; + } + } + request_headers->SetHeader(chrome_proxy_ect_header(), + net::GetNameForEffectiveConnectionType( + net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN)); + + static_assert(net::EFFECTIVE_CONNECTION_TYPE_OFFLINE + 1 == + net::EFFECTIVE_CONNECTION_TYPE_SLOW_2G, + "ECT enum value is not handled."); + static_assert(net::EFFECTIVE_CONNECTION_TYPE_4G + 1 == + net::EFFECTIVE_CONNECTION_TYPE_LAST, + "ECT enum value is not handled."); +} + +void DataReductionProxyNetworkDelegate::RemoveChromeProxyECTHeader( + net::HttpRequestHeaders* request_headers) const { + DCHECK(thread_checker_.CalledOnValidThread()); + + request_headers->RemoveHeader(chrome_proxy_ect_header()); +} + } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h index 33c717f128e..600445d9eb1 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate.h @@ -166,6 +166,17 @@ class DataReductionProxyNetworkDelegate : public net::LayeredNetworkDelegate { net::HttpRequestHeaders* request_headers, const net::URLRequest& request) const; + // May add chrome-proxy-ect header to |request_headers| if adding of + // chrome-proxy-ect is enabled via field trial and a valid estimate of + // network quality is available. This method should be called only when the + // resolved proxy for |request| is a data saver proxy. + void MaybeAddChromeProxyECTHeader(net::HttpRequestHeaders* request_headers, + const net::URLRequest& request) const; + + // Removes the chrome-proxy-ect header from |request_headers|. + void RemoveChromeProxyECTHeader( + net::HttpRequestHeaders* request_headers) const; + // All raw Data Reduction Proxy pointers must outlive |this|. DataReductionProxyConfig* data_reduction_proxy_config_; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc index 5c3886d257a..fe69c111390 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_network_delegate_unittest.cc @@ -40,6 +40,8 @@ #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h" #include "components/data_reduction_proxy/core/common/lofi_decider.h" #include "components/data_reduction_proxy/proto/client_config.pb.h" +#include "components/previews/core/previews_decider.h" +#include "components/previews/core/previews_experiments.h" #include "net/base/host_port_pair.h" #include "net/base/load_flags.h" #include "net/base/net_errors.h" @@ -57,6 +59,7 @@ #include "net/test/cert_test_util.h" #include "net/test/gtest_util.h" #include "net/test/test_data_directory.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_job_factory_impl.h" #include "net/url_request/url_request_status.h" @@ -74,33 +77,99 @@ const char kOtherProxy[] = "testproxy:17"; const char kTestURL[] = "http://www.google.com/"; const char kSecureTestURL[] = "https://www.google.com/"; -const std::string kReceivedValidOCLHistogramName = +const char kReceivedValidOCLHistogramName[] = "Net.HttpContentLengthWithValidOCL"; -const std::string kOriginalValidOCLHistogramName = +const char kOriginalValidOCLHistogramName[] = "Net.HttpOriginalContentLengthWithValidOCL"; -const std::string kDifferenceValidOCLHistogramName = +const char kDifferenceValidOCLHistogramName[] = "Net.HttpContentLengthDifferenceWithValidOCL"; +// HTTP original content length +const char kOriginalInsecureDirectHistogramName[] = + "Net.HttpOriginalContentLength.Http.Direct"; +const char kOriginalInsecureViaDRPHistogramName[] = + "Net.HttpOriginalContentLength.Http.ViaDRP"; +const char kOriginalInsecureBypassedHistogramName[] = + "Net.HttpOriginalContentLength.Http.BypassedDRP"; +const char kOriginalInsecureOtherHistogramName[] = + "Net.HttpOriginalContentLength.Http.Other"; +// HTTP video original content length +const char kOriginalVideoInsecureDirectHistogramName[] = + "Net.HttpOriginalContentLength.Http.Direct.Video"; +const char kOriginalVideoInsecureViaDRPHistogramName[] = + "Net.HttpOriginalContentLength.Http.ViaDRP.Video"; +const char kOriginalVideoInsecureBypassedHistogramName[] = + "Net.HttpOriginalContentLength.Http.BypassedDRP.Video"; +const char kOriginalVideoInsecureOtherHistogramName[] = + "Net.HttpOriginalContentLength.Http.Other.Video"; +// HTTPS original content length +const char kOriginalSecureDirectHistogramName[] = + "Net.HttpOriginalContentLength.Https.Direct"; +const char kOriginalSecureViaDRPHistogramName[] = + "Net.HttpOriginalContentLength.Https.ViaDRP"; +const char kOriginalSecureBypassedHistogramName[] = + "Net.HttpOriginalContentLength.Https.BypassedDRP"; +const char kOriginalSecureOtherHistogramName[] = + "Net.HttpOriginalContentLength.Https.Other"; +// HTTPS video original content length +const char kOriginalVideoSecureDirectHistogramName[] = + "Net.HttpOriginalContentLength.Https.Direct.Video"; +const char kOriginalVideoSecureViaDRPHistogramName[] = + "Net.HttpOriginalContentLength.Https.ViaDRP.Video"; +const char kOriginalVideoSecureBypassedHistogramName[] = + "Net.HttpOriginalContentLength.Https.BypassedDRP.Video"; +const char kOriginalVideoSecureOtherHistogramName[] = + "Net.HttpOriginalContentLength.Https.Other.Video"; + // Lo-Fi histograms. -const std::string kReceivedValidOCLLoFiOnHistogramName = +const char kReceivedValidOCLLoFiOnHistogramName[] = "Net.HttpContentLengthWithValidOCL.LoFiOn"; -const std::string kOriginalValidOCLLoFiOnHistogramName = +const char kOriginalValidOCLLoFiOnHistogramName[] = "Net.HttpOriginalContentLengthWithValidOCL.LoFiOn"; -const std::string kDifferenceValidOCLLoFiOnHistogramName = +const char kDifferenceValidOCLLoFiOnHistogramName[] = "Net.HttpContentLengthDifferenceWithValidOCL.LoFiOn"; -const std::string kReceivedHistogramName = "Net.HttpContentLength"; -const std::string kReceivedInsecureHistogramName = "Net.HttpContentLength.Http"; -const std::string kReceivedSecureHistogramName = "Net.HttpContentLength.Https"; -const std::string kReceivedVideoHistogramName = "Net.HttpContentLength.Video"; -const std::string kOriginalHistogramName = "Net.HttpOriginalContentLength"; -const std::string kDifferenceHistogramName = "Net.HttpContentLengthDifference"; -const std::string kFreshnessLifetimeHistogramName = +const char kReceivedHistogramName[] = "Net.HttpContentLength"; +const char kReceivedInsecureDirectHistogramName[] = + "Net.HttpContentLength.Http.Direct"; +const char kReceivedInsecureViaDRPHistogramName[] = + "Net.HttpContentLength.Http.ViaDRP"; +const char kReceivedInsecureBypassedHistogramName[] = + "Net.HttpContentLength.Http.BypassedDRP"; +const char kReceivedInsecureOtherHistogramName[] = + "Net.HttpContentLength.Http.Other"; +const char kReceivedSecureDirectHistogramName[] = + "Net.HttpContentLength.Https.Direct"; +const char kReceivedSecureViaDRPHistogramName[] = + "Net.HttpContentLength.Https.ViaDRP"; +const char kReceivedSecureBypassedHistogramName[] = + "Net.HttpContentLength.Https.BypassedDRP"; +const char kReceivedSecureOtherHistogramName[] = + "Net.HttpContentLength.Https.Other"; +const char kReceivedVideoInsecureDirectHistogramName[] = + "Net.HttpContentLength.Http.Direct.Video"; +const char kReceivedVideoInsecureViaDRPHistogramName[] = + "Net.HttpContentLength.Http.ViaDRP.Video"; +const char kReceivedVideoInsecureBypassedHistogramName[] = + "Net.HttpContentLength.Http.BypassedDRP.Video"; +const char kReceivedVideoInsecureOtherHistogramName[] = + "Net.HttpContentLength.Http.Other.Video"; +const char kReceivedVideoSecureDirectHistogramName[] = + "Net.HttpContentLength.Https.Direct.Video"; +const char kReceivedVideoSecureViaDRPHistogramName[] = + "Net.HttpContentLength.Https.ViaDRP.Video"; +const char kReceivedVideoSecureBypassedHistogramName[] = + "Net.HttpContentLength.Https.BypassedDRP.Video"; +const char kReceivedVideoSecureOtherHistogramName[] = + "Net.HttpContentLength.Https.Other.Video"; +const char kOriginalHistogramName[] = "Net.HttpOriginalContentLength"; +const char kDifferenceHistogramName[] = "Net.HttpContentLengthDifference"; +const char kFreshnessLifetimeHistogramName[] = "Net.HttpContentFreshnessLifetime"; -const std::string kCacheableHistogramName = "Net.HttpContentLengthCacheable"; -const std::string kCacheable4HoursHistogramName = +const char kCacheableHistogramName[] = "Net.HttpContentLengthCacheable"; +const char kCacheable4HoursHistogramName[] = "Net.HttpContentLengthCacheable4Hours"; -const std::string kCacheable24HoursHistogramName = +const char kCacheable24HoursHistogramName[] = "Net.HttpContentLengthCacheable24Hours"; const int64_t kResponseContentLength = 100; const int64_t kOriginalContentLength = 200; @@ -132,7 +201,9 @@ const Client kClient = Client::UNKNOWN; class TestLoFiDecider : public LoFiDecider { public: TestLoFiDecider() - : should_request_lofi_resource_(false), + : should_be_client_lofi_(false), + should_be_client_lofi_auto_reload_(false), + should_request_lofi_resource_(false), ignore_is_using_data_reduction_proxy_check_(false) {} ~TestLoFiDecider() override {} @@ -144,6 +215,14 @@ class TestLoFiDecider : public LoFiDecider { should_request_lofi_resource_ = should_request_lofi_resource; } + void SetIsUsingClientLoFi(bool should_be_client_lofi) { + should_be_client_lofi_ = should_be_client_lofi; + } + + void SetIsClientLoFiAutoReload(bool should_be_client_lofi_auto_reload) { + should_be_client_lofi_auto_reload_ = should_be_client_lofi_auto_reload; + } + void MaybeSetAcceptTransformHeader( const net::URLRequest& request, bool is_previews_disabled, @@ -181,18 +260,26 @@ class TestLoFiDecider : public LoFiDecider { headers->RemoveHeader(chrome_proxy_accept_transform_header()); } - void MaybeSetIgnorePreviewsBlacklistDirective( - net::HttpRequestHeaders* headers) const override {} - bool ShouldRecordLoFiUMA(const net::URLRequest& request) const override { return should_request_lofi_resource_; } + bool IsClientLoFiImageRequest(const net::URLRequest& request) const override { + return should_be_client_lofi_; + } + + bool IsClientLoFiAutoReloadRequest( + const net::URLRequest& request) const override { + return should_be_client_lofi_auto_reload_; + } + void ignore_is_using_data_reduction_proxy_check() { ignore_is_using_data_reduction_proxy_check_ = true; } private: + bool should_be_client_lofi_; + bool should_be_client_lofi_auto_reload_; bool should_request_lofi_resource_; bool ignore_is_using_data_reduction_proxy_check_; }; @@ -208,17 +295,40 @@ class TestLoFiUIService : public LoFiUIService { on_lofi_response_ = true; } + void ClearResponse() { on_lofi_response_ = false; } + private: bool on_lofi_response_; }; +class TestPreviewsDecider : public previews::PreviewsDecider { + public: + TestPreviewsDecider() {} + ~TestPreviewsDecider() override {} + // previews::PreviewsDecider: + bool ShouldAllowPreviewAtECT( + const net::URLRequest& request, + previews::PreviewsType type, + net::EffectiveConnectionType effective_connection_type_threshold) + const override { + return true; + } + + // Same as ShouldAllowPreviewAtECT, but uses the previews default + // EffectiveConnectionType. + bool ShouldAllowPreview(const net::URLRequest& request, + previews::PreviewsType type) const override { + return true; + } +}; + enum ProxyTestConfig { USE_SECURE_PROXY, USE_INSECURE_PROXY, BYPASS_PROXY }; class DataReductionProxyNetworkDelegateTest : public testing::Test { public: DataReductionProxyNetworkDelegateTest() - : context_(true), - context_storage_(&context_), + : lofi_decider_(nullptr), + lofi_ui_service_(nullptr), ssl_socket_data_provider_(net::ASYNC, net::OK) { ssl_socket_data_provider_.next_proto = net::kProtoHTTP11; ssl_socket_data_provider_.cert = net::ImportCertFromFile( @@ -240,13 +350,18 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { net::ProxyServer::SCHEME_HTTP); break; } + context_.reset(new net::TestURLRequestContext(true)); + context_storage_.reset(new net::URLRequestContextStorage(context_.get())); proxy_service_ = net::ProxyService::CreateFixedFromPacResult(proxy_server.ToPacString()); - context_.set_proxy_service(proxy_service_.get()); + context_->set_proxy_service(proxy_service_.get()); + + mock_socket_factory_.reset(new net::MockClientSocketFactory()); + DataReductionProxyTestContext::Builder builder; builder = builder.WithClient(kClient) - .WithMockClientSocketFactory(&mock_socket_factory_) - .WithURLRequestContext(&context_); + .WithMockClientSocketFactory(mock_socket_factory_.get()) + .WithURLRequestContext(context_.get()); if (proxy_config != BYPASS_PROXY) { builder = builder.WithProxiesForHttp({DataReductionProxyServer( @@ -255,8 +370,8 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { test_context_ = builder.Build(); - context_.set_client_socket_factory(&mock_socket_factory_); - test_context_->AttachToURLRequestContext(&context_storage_); + context_->set_client_socket_factory(mock_socket_factory_.get()); + test_context_->AttachToURLRequestContext(context_storage_.get()); std::unique_ptr<TestLoFiDecider> lofi_decider(new TestLoFiDecider()); lofi_decider_ = lofi_decider.get(); @@ -266,13 +381,86 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { lofi_ui_service_ = lofi_ui_service.get(); test_context_->io_data()->set_lofi_ui_service(std::move(lofi_ui_service)); - context_.set_enable_brotli(enable_brotli_globally); - context_.set_network_quality_estimator(&test_network_quality_estimator_); - context_.Init(); + context_->set_enable_brotli(enable_brotli_globally); + context_->set_network_quality_estimator(&test_network_quality_estimator_); + context_->Init(); test_context_->EnableDataReductionProxyWithSecureProxyCheckSuccess(); } + // Build the sockets by adding appropriate mock data for + // |effective_connection_types.size()| number of requests. Data for + // chrome-Proxy-ect header is added to the mock data if |expect_ect_header| + // is true. |reads_list|, |mock_writes| and |writes_list| should be empty, and + // are owned by the caller. + void BuildSocket(const std::string& response_headers, + const std::string& response_body, + bool expect_ect_header, + const std::vector<net::EffectiveConnectionType>& + effective_connection_types, + std::vector<net::MockRead>* reads_list, + std::vector<std::string>* mock_writes, + std::vector<net::MockWrite>* writes_list) { + EXPECT_LT(0u, effective_connection_types.size()); + EXPECT_TRUE(reads_list->empty()); + EXPECT_TRUE(mock_writes->empty()); + EXPECT_TRUE(writes_list->empty()); + + for (size_t i = 0; i < effective_connection_types.size(); ++i) { + reads_list->push_back(net::MockRead(response_headers.c_str())); + reads_list->push_back(net::MockRead(response_body.c_str())); + } + reads_list->push_back(net::MockRead(net::SYNCHRONOUS, net::OK)); + + std::string prefix = std::string("GET ") + .append(kTestURL) + .append(" HTTP/1.1\r\n") + .append("Host: ") + .append(GURL(kTestURL).host()) + .append( + "\r\n" + "Proxy-Connection: keep-alive\r\n" + "User-Agent:\r\n" + "Accept-Encoding: gzip, deflate\r\n" + "Accept-Language: en-us,fr\r\n"); + + if (io_data()->test_request_options()->GetHeaderValueForTesting().empty()) { + // Force regeneration of Chrome-Proxy header. + io_data()->test_request_options()->SetSecureSession("123"); + } + + EXPECT_FALSE( + io_data()->test_request_options()->GetHeaderValueForTesting().empty()); + + std::string suffix = + std::string("Chrome-Proxy: ") + + io_data()->test_request_options()->GetHeaderValueForTesting() + + std::string("\r\n\r\n"); + + mock_socket_factory_->AddSSLSocketDataProvider(&ssl_socket_data_provider_); + + for (net::EffectiveConnectionType effective_connection_type : + effective_connection_types) { + std::string ect_header; + if (expect_ect_header) { + ect_header = "chrome-proxy-ect: " + + std::string(net::GetNameForEffectiveConnectionType( + effective_connection_type)) + + "\r\n"; + } + + std::string mock_write = prefix + ect_header + suffix; + mock_writes->push_back(mock_write); + writes_list->push_back(net::MockWrite(mock_writes->back().c_str())); + } + + EXPECT_FALSE(socket_); + socket_ = base::MakeUnique<net::StaticSocketDataProvider>( + reads_list->data(), reads_list->size(), writes_list->data(), + writes_list->size()); + mock_socket_factory_->AddSocketDataProvider(socket_.get()); + } + static void VerifyHeaders(bool expected_data_reduction_proxy_used, bool expected_lofi_used, const net::HttpRequestHeaders& headers) { @@ -288,6 +476,8 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { EXPECT_EQ(lofi_response, lofi_ui_service_->DidNotifyLoFiResponse()); } + void ClearLoFiUIService() { lofi_ui_service_->ClearResponse(); } + void VerifyDataReductionProxyData(const net::URLRequest& request, bool data_reduction_proxy_used, bool lofi_used) { @@ -317,11 +507,11 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { net::MockRead(response_body.c_str()), net::MockRead(net::SYNCHRONOUS, net::OK)}; net::StaticSocketDataProvider socket(reads, arraysize(reads), nullptr, 0); - mock_socket_factory_.AddSocketDataProvider(&socket); + mock_socket_factory_->AddSocketDataProvider(&socket); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(url, net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = context_->CreateRequest( + url, net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); if (request_headers) request->SetExtraRequestHeaders(*request_headers); request->SetLoadFlags(request->load_flags() | load_flags); @@ -373,7 +563,7 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { std::string(base::checked_cast<size_t>(response_body_size), ' '); } - mock_socket_factory_.AddSSLSocketDataProvider(&ssl_socket_data_provider_); + mock_socket_factory_->AddSSLSocketDataProvider(&ssl_socket_data_provider_); net::MockRead reads[] = {net::MockRead(response_headers.c_str()), net::MockRead(response_body.c_str()), @@ -399,6 +589,10 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { "User-Agent:\r\n"); std::string accept_language_header("Accept-Language: en-us,fr\r\n"); + std::string ect_header = "chrome-proxy-ect: " + + std::string(net::GetNameForEffectiveConnectionType( + net::EFFECTIVE_CONNECTION_TYPE_UNKNOWN)) + + "\r\n"; // Brotli is included in accept-encoding header only if the request went // to the network (i.e., it was not a cached response), and if data @@ -414,23 +608,24 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { std::string("\r\n\r\n"); std::string mock_write = prefix_headers + accept_language_header + - accept_encoding_header + suffix_headers; + ect_header + accept_encoding_header + + suffix_headers; if (expect_cached || !expect_brotli) { // Order of headers is different if the headers were modified by data // reduction proxy network delegate. mock_write = prefix_headers + accept_encoding_header + - accept_language_header + suffix_headers; + accept_language_header + ect_header + suffix_headers; } net::MockWrite writes[] = {net::MockWrite(mock_write.c_str())}; net::StaticSocketDataProvider socket(reads, arraysize(reads), writes, arraysize(writes)); - mock_socket_factory_.AddSocketDataProvider(&socket); + mock_socket_factory_->AddSocketDataProvider(&socket); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(url, net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = context_->CreateRequest( + url, net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); if (request_headers) request->SetExtraRequestHeaders(*request_headers); @@ -493,7 +688,7 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { std::string response_body = std::string(base::checked_cast<size_t>(response_body_size), ' '); - mock_socket_factory_.AddSSLSocketDataProvider(&ssl_socket_data_provider_); + mock_socket_factory_->AddSSLSocketDataProvider(&ssl_socket_data_provider_); net::MockRead redirect_reads[] = { net::MockRead("HTTP/1.1 302 Redirect\r\n"), @@ -516,6 +711,7 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { "www.google.com\r\nProxy-Connection: " "keep-alive\r\nUser-Agent:\r\nAccept-Encoding: gzip, " "deflate\r\nAccept-Language: en-us,fr\r\n" + "chrome-proxy-ect: 4G\r\n" "Chrome-Proxy: " + io_data()->test_request_options()->GetHeaderValueForTesting() + (page_id_value.empty() ? "" : (", " + page_id_value)) + "\r\n\r\n"; @@ -535,11 +731,11 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { arraysize(redirect_writes)); } - mock_socket_factory_.AddSocketDataProvider(socket.get()); + mock_socket_factory_->AddSocketDataProvider(socket.get()); net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> request = - context_.CreateRequest(url, net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> request = context_->CreateRequest( + url, net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); if (!page_id_value.empty()) { request->SetLoadFlags(request->load_flags() | net::LOAD_MAIN_FRAME_DEPRECATED); @@ -549,6 +745,51 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { base::RunLoop().RunUntilIdle(); } + // Fetches a request while the effective connection type is set to + // |effective_connection_type|. Verifies that the request headers include the + // chrome-proxy-ect header only if |expect_ect_header| is true. The response + // must be served from the cache if |expect_cached| is true. + void FetchURLRequestAndVerifyECTHeader( + net::EffectiveConnectionType effective_connection_type, + bool expect_ect_header, + bool expect_cached) { + test_network_quality_estimator()->set_effective_connection_type( + effective_connection_type); + + net::TestDelegate delegate; + std::unique_ptr<net::URLRequest> request = context_->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); + + request->Start(); + base::RunLoop().RunUntilIdle(); + + EXPECT_EQ(140, request->received_response_content_length()); + EXPECT_EQ(expect_cached, request->was_cached()); + EXPECT_EQ(expect_cached, request->GetTotalSentBytes() == 0); + EXPECT_EQ(expect_cached, request->GetTotalReceivedBytes() == 0); + + net::HttpRequestHeaders sent_request_headers; + EXPECT_NE(expect_cached, + request->GetFullRequestHeaders(&sent_request_headers)); + + if (expect_cached) { + // Request headers are missing. Return since there is nothing left to + // check. + return; + } + + // Verify that chrome-proxy-ect header is present in the request headers + // only if |expect_ect_header| is true. + std::string ect_value; + EXPECT_EQ(expect_ect_header, sent_request_headers.GetHeader( + chrome_proxy_ect_header(), &ect_value)); + + if (!expect_ect_header) + return; + EXPECT_EQ(net::GetNameForEffectiveConnectionType(effective_connection_type), + ect_value); + } + void DelegateStageDone(int result) {} void NotifyNetworkDelegate(net::URLRequest* request, @@ -570,13 +811,13 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { } net::MockClientSocketFactory* mock_socket_factory() { - return &mock_socket_factory_; + return mock_socket_factory_.get(); } - net::TestURLRequestContext* context() { return &context_; } + net::TestURLRequestContext* context() { return context_.get(); } net::NetworkDelegate* network_delegate() const { - return context_.network_delegate(); + return context_->network_delegate(); } TestDataReductionProxyParams* params() const { @@ -603,10 +844,10 @@ class DataReductionProxyNetworkDelegateTest : public testing::Test { private: base::MessageLoopForIO message_loop_; - net::MockClientSocketFactory mock_socket_factory_; + std::unique_ptr<net::MockClientSocketFactory> mock_socket_factory_; std::unique_ptr<net::ProxyService> proxy_service_; - net::TestURLRequestContext context_; - net::URLRequestContextStorage context_storage_; + std::unique_ptr<net::TestURLRequestContext> context_; + std::unique_ptr<net::URLRequestContextStorage> context_storage_; TestLoFiDecider* lofi_decider_; TestLoFiUIService* lofi_ui_service_; @@ -700,24 +941,28 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { base::TrimString(kOtherProxy, "/", &proxy); data_reduction_proxy_info.UseNamedProxy(proxy); + // Needed as a parameter, but functionality is not tested. + TestPreviewsDecider test_previews_decider; + { // Main frame loaded. Lo-Fi should be used. net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> fake_request = - context()->CreateRequest(GURL(kTestURL), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> fake_request = context()->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); fake_request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); - lofi_decider()->SetIsUsingLoFi( - config()->ShouldEnableLoFi(*fake_request.get())); + lofi_decider()->SetIsUsingLoFi(config()->ShouldEnableLoFi( + *fake_request.get(), test_previews_decider)); NotifyNetworkDelegate(fake_request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); VerifyHeaders(tests[i].is_data_reduction_proxy, true, headers); VerifyDataReductionProxyData( *fake_request, tests[i].is_data_reduction_proxy, - config()->ShouldEnableLoFi(*fake_request.get())); + config()->ShouldEnableLoFi(*fake_request.get(), + test_previews_decider)); } { @@ -725,8 +970,8 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> fake_request = - context()->CreateRequest(GURL(kTestURL), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> fake_request = context()->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); lofi_decider()->SetIsUsingLoFi(false); NotifyNetworkDelegate(fake_request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); @@ -740,8 +985,8 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> fake_request = - context()->CreateRequest(GURL(kTestURL), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> fake_request = context()->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); lofi_decider()->SetIsUsingLoFi(true); NotifyNetworkDelegate(fake_request.get(), data_reduction_proxy_info, @@ -757,8 +1002,8 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> fake_request = - context()->CreateRequest(GURL(kTestURL), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> fake_request = context()->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); fake_request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); lofi_decider()->SetIsUsingLoFi(false); NotifyNetworkDelegate(fake_request.get(), data_reduction_proxy_info, @@ -773,8 +1018,8 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> fake_request = - context()->CreateRequest(GURL(kTestURL), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> fake_request = context()->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); lofi_decider()->SetIsUsingLoFi(false); NotifyNetworkDelegate(fake_request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); @@ -788,16 +1033,17 @@ TEST_F(DataReductionProxyNetworkDelegateTest, LoFiTransitions) { net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; net::TestDelegate delegate; - std::unique_ptr<net::URLRequest> fake_request = - context()->CreateRequest(GURL(kTestURL), net::IDLE, &delegate); + std::unique_ptr<net::URLRequest> fake_request = context()->CreateRequest( + GURL(kTestURL), net::IDLE, &delegate, TRAFFIC_ANNOTATION_FOR_TESTS); fake_request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); - lofi_decider()->SetIsUsingLoFi( - config()->ShouldEnableLoFi(*fake_request.get())); + lofi_decider()->SetIsUsingLoFi(config()->ShouldEnableLoFi( + *fake_request.get(), test_previews_decider)); NotifyNetworkDelegate(fake_request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); VerifyDataReductionProxyData( *fake_request, tests[i].is_data_reduction_proxy, - config()->ShouldEnableLoFi(*fake_request.get())); + config()->ShouldEnableLoFi(*fake_request.get(), + test_previews_decider)); } } } @@ -844,8 +1090,9 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RequestDataConfigurations) { test_network_quality_estimator()->set_effective_connection_type( net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL(kTestURL), net::RequestPriority::IDLE, nullptr); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL(kTestURL), net::RequestPriority::IDLE, + nullptr, TRAFFIC_ANNOTATION_FOR_TESTS); request->SetLoadFlags(test.main_frame ? net::LOAD_MAIN_FRAME_DEPRECATED : 0); lofi_decider()->SetIsUsingLoFi(test.lofi_on); @@ -907,8 +1154,9 @@ TEST_F(DataReductionProxyNetworkDelegateTest, else data_reduction_proxy_info.UseNamedProxy("some.other.proxy"); config()->UpdateConfigForTesting(test.data_reduction_proxy_enabled, true); - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL(kTestURL), net::RequestPriority::IDLE, nullptr); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL(kTestURL), net::RequestPriority::IDLE, + nullptr, TRAFFIC_ANNOTATION_FOR_TESTS); request->set_method("GET"); net::HttpRequestHeaders headers; net::ProxyRetryInfoMap proxy_retry_info; @@ -939,8 +1187,9 @@ TEST_F(DataReductionProxyNetworkDelegateTest, RedirectRequestDataCleared) { test_network_quality_estimator()->set_effective_connection_type( net::EFFECTIVE_CONNECTION_TYPE_OFFLINE); - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL(kTestURL), net::RequestPriority::IDLE, nullptr); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL(kTestURL), net::RequestPriority::IDLE, + nullptr, TRAFFIC_ANNOTATION_FOR_TESTS); request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); lofi_decider()->SetIsUsingLoFi(true); io_data()->request_options()->SetSecureSession("fake-session"); @@ -1018,16 +1267,20 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { kResponseContentLength, 1); histogram_tester.ExpectUniqueSample(kOriginalValidOCLHistogramName, kOriginalContentLength, 1); + histogram_tester.ExpectUniqueSample(kOriginalInsecureViaDRPHistogramName, + kOriginalContentLength, 1); histogram_tester.ExpectUniqueSample( kDifferenceValidOCLHistogramName, kOriginalContentLength - kResponseContentLength, 1); histogram_tester.ExpectUniqueSample(kReceivedHistogramName, kResponseContentLength, 1); - histogram_tester.ExpectUniqueSample(kReceivedInsecureHistogramName, + histogram_tester.ExpectUniqueSample(kReceivedInsecureViaDRPHistogramName, kResponseContentLength, 1); - histogram_tester.ExpectTotalCount(kReceivedSecureHistogramName, 0); - histogram_tester.ExpectTotalCount(kReceivedVideoHistogramName, 0); - histogram_tester.ExpectUniqueSample(kReceivedInsecureHistogramName, + histogram_tester.ExpectTotalCount(kReceivedInsecureDirectHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedSecureDirectHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedVideoInsecureViaDRPHistogramName, + 0); + histogram_tester.ExpectUniqueSample(kReceivedInsecureViaDRPHistogramName, kResponseContentLength, 1); histogram_tester.ExpectUniqueSample(kOriginalHistogramName, kOriginalContentLength, 1); @@ -1085,8 +1338,10 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetHistograms) { switches::kDataReductionProxyLoFiValueAlwaysOn); } + // Needed as a parameter, but functionality is not tested. + TestPreviewsDecider test_previews_decider; lofi_decider()->SetIsUsingLoFi( - config()->ShouldEnableLoFi(*fake_request.get())); + config()->ShouldEnableLoFi(*fake_request.get(), test_previews_decider)); fake_request = (FetchURLRequest(GURL(kTestURL), nullptr, response_headers, kResponseContentLength, 0)); @@ -1137,35 +1392,216 @@ TEST_F(DataReductionProxyNetworkDelegateTest, NetVideoHistograms) { FetchURLRequest(GURL(kTestURL), nullptr, video_response_headers, kResponseContentLength, 0); - histogram_tester.ExpectUniqueSample(kReceivedInsecureHistogramName, + histogram_tester.ExpectUniqueSample(kReceivedInsecureViaDRPHistogramName, kResponseContentLength, 1); - histogram_tester.ExpectTotalCount(kReceivedSecureHistogramName, 0); - histogram_tester.ExpectUniqueSample(kReceivedVideoHistogramName, + histogram_tester.ExpectTotalCount(kReceivedInsecureDirectHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedInsecureBypassedHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedInsecureOtherHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedSecureViaDRPHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedSecureDirectHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedSecureBypassedHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedSecureOtherHistogramName, 0); + histogram_tester.ExpectUniqueSample(kReceivedVideoInsecureViaDRPHistogramName, kResponseContentLength, 1); + histogram_tester.ExpectTotalCount(kReceivedVideoInsecureDirectHistogramName, + 0); + histogram_tester.ExpectTotalCount(kReceivedVideoInsecureBypassedHistogramName, + 0); + histogram_tester.ExpectTotalCount(kReceivedVideoInsecureOtherHistogramName, + 0); + histogram_tester.ExpectTotalCount(kReceivedVideoSecureViaDRPHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedVideoSecureDirectHistogramName, 0); + histogram_tester.ExpectTotalCount(kReceivedVideoSecureBypassedHistogramName, + 0); + histogram_tester.ExpectTotalCount(kReceivedVideoSecureOtherHistogramName, 0); } -TEST_F(DataReductionProxyNetworkDelegateTest, NetSSLHistograms) { - Init(BYPASS_PROXY, false); +struct ExpectedHistogram { + const std::string name; + int64_t value; +}; - base::HistogramTester histogram_tester; +bool operator<(const ExpectedHistogram& hist1, const ExpectedHistogram& hist2) { + return hist1.name < hist2.name; +} - // Check https - std::string secure_response_headers = - "HTTP/1.1 200 OK\r\n" - "Date: Wed, 28 Nov 2007 09:40:09 GMT\r\n" - "Expires: Mon, 24 Nov 2014 12:45:26 GMT\r\n" - "Via: 1.1 Chrome-Compression-Proxy\r\n" - "x-original-content-length: " + - base::Int64ToString(kOriginalContentLength) + "\r\n\r\n"; +TEST_F(DataReductionProxyNetworkDelegateTest, DetailedNetHistograms) { + // List of all the histograms we're concerned with. + // Each test case can define interesting histograms from this list. + // Any histogram not called out in an invidual test will have an expected + // count of 0 samples. + const std::set<std::string> all_new_net_histograms{ + // HTTP received content length + kReceivedInsecureDirectHistogramName, + kReceivedInsecureViaDRPHistogramName, + kReceivedInsecureBypassedHistogramName, + kReceivedInsecureOtherHistogramName, + // HTTP video received content length + kReceivedVideoInsecureDirectHistogramName, + kReceivedVideoInsecureViaDRPHistogramName, + kReceivedVideoInsecureBypassedHistogramName, + kReceivedVideoInsecureOtherHistogramName, + // HTTPS received content length + kReceivedSecureDirectHistogramName, kReceivedSecureViaDRPHistogramName, + kReceivedSecureBypassedHistogramName, kReceivedSecureOtherHistogramName, + // HTTPS video received content length + kReceivedVideoSecureDirectHistogramName, + kReceivedVideoSecureViaDRPHistogramName, + kReceivedVideoSecureBypassedHistogramName, + kReceivedVideoSecureOtherHistogramName, + // HTTP Original content length + kOriginalInsecureDirectHistogramName, + kOriginalInsecureViaDRPHistogramName, + kOriginalInsecureBypassedHistogramName, + kOriginalInsecureOtherHistogramName, + // HTTP video Original content length + kOriginalVideoInsecureDirectHistogramName, + kOriginalVideoInsecureViaDRPHistogramName, + kOriginalVideoInsecureBypassedHistogramName, + kOriginalVideoInsecureOtherHistogramName, + // HTTPS Original content length + kOriginalSecureDirectHistogramName, kOriginalSecureViaDRPHistogramName, + kOriginalSecureBypassedHistogramName, kOriginalSecureOtherHistogramName, + // HTTPS video Original content length + kOriginalVideoSecureDirectHistogramName, + kOriginalVideoSecureViaDRPHistogramName, + kOriginalVideoSecureBypassedHistogramName, + kOriginalVideoSecureOtherHistogramName, + }; - mock_socket_factory()->AddSSLSocketDataProvider(ssl_socket_data_provider()); - FetchURLRequest(GURL(kSecureTestURL), nullptr, secure_response_headers, - kResponseContentLength, 0); + const struct { + const std::string name; + bool is_video; + bool is_https; + ProxyTestConfig proxy_config; + int64_t original_content_length; + int64_t content_length; + // Any histogram listed in all_new_net_histograms but not here should have + // no samples. + const std::set<ExpectedHistogram> expected_histograms; + } tests[] = { + {"HTTP nonvideo request via DRP", + false, + false, + USE_INSECURE_PROXY, + kOriginalContentLength, + kResponseContentLength, + { + {kReceivedInsecureViaDRPHistogramName, kResponseContentLength}, + {kOriginalInsecureViaDRPHistogramName, kOriginalContentLength}, + }}, + {"HTTP video request via DRP", + true, + false, + USE_INSECURE_PROXY, + kOriginalContentLength, + kResponseContentLength, + { + {kReceivedInsecureViaDRPHistogramName, kResponseContentLength}, + {kOriginalInsecureViaDRPHistogramName, kOriginalContentLength}, + {kReceivedVideoInsecureViaDRPHistogramName, kResponseContentLength}, + {kOriginalVideoInsecureViaDRPHistogramName, kOriginalContentLength}, + }}, + {"DRP not configured for http", + false, + false, + BYPASS_PROXY, + kOriginalContentLength, + kResponseContentLength, + { + {kReceivedInsecureOtherHistogramName, kResponseContentLength}, + {kOriginalInsecureOtherHistogramName, kResponseContentLength}, + }}, + {"DRP not configured for http video", + true, + false, + BYPASS_PROXY, + kOriginalContentLength, + kResponseContentLength, + { + {kReceivedInsecureOtherHistogramName, kResponseContentLength}, + {kOriginalInsecureOtherHistogramName, kResponseContentLength}, + {kReceivedVideoInsecureOtherHistogramName, kResponseContentLength}, + {kOriginalVideoInsecureOtherHistogramName, kResponseContentLength}, + }}, + {"nonvideo over https", + false, + true, + BYPASS_PROXY, + kOriginalContentLength, + kResponseContentLength, + { + {kReceivedSecureDirectHistogramName, kResponseContentLength}, + {kOriginalSecureDirectHistogramName, kResponseContentLength}, + }}, + {"video over https", + true, + true, + BYPASS_PROXY, + kOriginalContentLength, + kResponseContentLength, + { + {kReceivedSecureDirectHistogramName, kResponseContentLength}, + {kOriginalSecureDirectHistogramName, kResponseContentLength}, + {kReceivedVideoSecureDirectHistogramName, kResponseContentLength}, + {kOriginalVideoSecureDirectHistogramName, kResponseContentLength}, + }}, + }; - histogram_tester.ExpectTotalCount(kReceivedInsecureHistogramName, 0); - histogram_tester.ExpectUniqueSample(kReceivedSecureHistogramName, - kResponseContentLength, 1); - histogram_tester.ExpectTotalCount(kReceivedVideoHistogramName, 0); + for (const auto& test : tests) { + LOG(INFO) << "NetHistograms: " << test.name; + Init(test.proxy_config, false); + base::HistogramTester histogram_tester; + + GURL test_url = GURL(kTestURL); + + if (test.is_https) { + test_url = GURL(kSecureTestURL); + mock_socket_factory()->AddSSLSocketDataProvider( + ssl_socket_data_provider()); + } + + std::string via_header = ""; + std::string ocl_header = ""; + + if (test.proxy_config == USE_INSECURE_PROXY) { + via_header = "Via: 1.1 Chrome-Compression-Proxy\r\n"; + ocl_header = "x-original-content-length: " + + base::Int64ToString(kOriginalContentLength) + "\r\n"; + } + if (test.is_video) { + // Check video + std::string video_response_headers = + "HTTP/1.1 200 OK\r\n" + "Date: Wed, 28 Nov 2007 09:40:09 GMT\r\n" + "Expires: Mon, 24 Nov 2014 12:45:26 GMT\r\n" + "Content-Type: video/mp4\r\n" + + via_header + ocl_header + "\r\n"; + + FetchURLRequest(test_url, nullptr, video_response_headers, + kResponseContentLength, 0); + } else { + // Check https + std::string response_headers = + "HTTP/1.1 200 OK\r\n" + "Date: Wed, 28 Nov 2007 09:40:09 GMT\r\n" + "Expires: Mon, 24 Nov 2014 12:45:26 GMT\r\n" + + via_header + ocl_header + "\r\n\r\n"; + + FetchURLRequest(test_url, nullptr, response_headers, + kResponseContentLength, 0); + } + + for (const auto& histogram : all_new_net_histograms) { + auto expected_it = test.expected_histograms.find({histogram, 0}); + if (expected_it == test.expected_histograms.end()) { + histogram_tester.ExpectTotalCount(histogram, 0); + } else { + histogram_tester.ExpectUniqueSample(expected_it->name, + expected_it->value, 1); + } + } + } } TEST_F(DataReductionProxyNetworkDelegateTest, OnCompletedInternalLoFi) { @@ -1173,11 +1609,12 @@ TEST_F(DataReductionProxyNetworkDelegateTest, OnCompletedInternalLoFi) { // Enable Lo-Fi. const struct { bool lofi_response; - } tests[] = { - {false}, {true}, - }; + bool was_server; + } tests[] = {{false, false}, {true, true}, {true, false}}; - for (size_t i = 0; i < arraysize(tests); ++i) { + for (const auto& test : tests) { + lofi_decider()->SetIsUsingClientLoFi(false); + ClearLoFiUIService(); std::string response_headers = "HTTP/1.1 200 OK\r\n" "Date: Wed, 28 Nov 2007 09:40:09 GMT\r\n" @@ -1185,15 +1622,19 @@ TEST_F(DataReductionProxyNetworkDelegateTest, OnCompletedInternalLoFi) { "Via: 1.1 Chrome-Compression-Proxy\r\n" "x-original-content-length: 200\r\n"; - if (tests[i].lofi_response) - response_headers += "Chrome-Proxy-Content-Transform: empty-image\r\n"; + if (test.lofi_response) { + if (test.was_server) + response_headers += "Chrome-Proxy-Content-Transform: empty-image\r\n"; + else + lofi_decider()->SetIsUsingClientLoFi(true); + } response_headers += "\r\n"; auto request = FetchURLRequest(GURL(kTestURL), nullptr, response_headers, 140, 0); - EXPECT_EQ(tests[i].lofi_response, + EXPECT_EQ(test.was_server, DataReductionProxyData::GetData(*request)->lofi_received()); - VerifyDidNotifyLoFiResponse(tests[i].lofi_response); + VerifyDidNotifyLoFiResponse(test.lofi_response); } } @@ -1371,8 +1812,9 @@ TEST_F(DataReductionProxyNetworkDelegateTest, base::TrimString(params()->DefaultOrigin(), "/", &data_reduction_proxy); data_reduction_proxy_info.UseNamedProxy(data_reduction_proxy); - std::unique_ptr<net::URLRequest> request = context()->CreateRequest( - GURL(kTestURL), net::RequestPriority::IDLE, nullptr); + std::unique_ptr<net::URLRequest> request = + context()->CreateRequest(GURL(kTestURL), net::RequestPriority::IDLE, + nullptr, TRAFFIC_ANNOTATION_FOR_TESTS); request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); io_data()->request_options()->SetSecureSession("fake-session"); @@ -1380,6 +1822,11 @@ TEST_F(DataReductionProxyNetworkDelegateTest, net::ProxyRetryInfoMap proxy_retry_info; // Send a request and verify the page ID is 1. + network_delegate()->NotifyBeforeStartTransaction( + request.get(), + base::Bind(&DataReductionProxyNetworkDelegateTest::DelegateStageDone, + base::Unretained(this)), + &headers); network_delegate()->NotifyBeforeSendHeaders( request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); DataReductionProxyData* data = @@ -1389,9 +1836,14 @@ TEST_F(DataReductionProxyNetworkDelegateTest, // Send a second request and verify the page ID incremements. request = context()->CreateRequest(GURL(kTestURL), net::RequestPriority::IDLE, - nullptr); + nullptr, TRAFFIC_ANNOTATION_FOR_TESTS); request->SetLoadFlags(net::LOAD_MAIN_FRAME_DEPRECATED); + network_delegate()->NotifyBeforeStartTransaction( + request.get(), + base::Bind(&DataReductionProxyNetworkDelegateTest::DelegateStageDone, + base::Unretained(this)), + &headers); network_delegate()->NotifyBeforeSendHeaders( request.get(), data_reduction_proxy_info, proxy_retry_info, &headers); data = DataReductionProxyData::GetData(*request.get()); @@ -1413,6 +1865,271 @@ TEST_F(DataReductionProxyNetworkDelegateTest, EXPECT_EQ(1u, data->page_id().value()); } +// Test that effective connection type is correctly added to the request +// headers when it is enabled using field trial. The server is varying on the +// effective connection type (ECT). +TEST_F(DataReductionProxyNetworkDelegateTest, ECTHeaderEnabledWithVary) { + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + + std::string response_headers = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 140\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Cache-Control: max-age=1200\r\n" + "Vary: chrome-proxy-ect\r\n" + "x-original-content-length: 200\r\n\r\n"; + + int response_body_size = 140; + std::string response_body(base::checked_cast<size_t>(response_body_size), + ' '); + + std::vector<net::MockRead> reads_list; + std::vector<std::string> mock_writes; + std::vector<net::MockWrite> writes_list; + + std::vector<net::EffectiveConnectionType> effective_connection_types; + effective_connection_types.push_back(net::EFFECTIVE_CONNECTION_TYPE_SLOW_2G); + effective_connection_types.push_back(net::EFFECTIVE_CONNECTION_TYPE_2G); + + BuildSocket(response_headers, response_body, true, effective_connection_types, + &reads_list, &mock_writes, &writes_list); + + // Add 2 socket providers since 2 requests in this test are fetched from the + // network. + FetchURLRequestAndVerifyECTHeader(effective_connection_types[0], true, false); + + // When the ECT is set to the same value, fetching the same resource should + // result in a cache hit. + FetchURLRequestAndVerifyECTHeader(effective_connection_types[0], true, true); + + // When the ECT is set to a different value, the response should not be + // served from the cache. + FetchURLRequestAndVerifyECTHeader(effective_connection_types[1], true, false); +} + +// Test that effective connection type is correctly added to the request +// headers when it is enabled using field trial. The server is not varying on +// the effective connection type (ECT). +TEST_F(DataReductionProxyNetworkDelegateTest, ECTHeaderEnabledWithoutVary) { + Init(USE_SECURE_PROXY, false /* enable_brotli_globally */); + + std::string response_headers = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 140\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "Cache-Control: max-age=1200\r\n" + "x-original-content-length: 200\r\n\r\n"; + + int response_body_size = 140; + std::string response_body(base::checked_cast<size_t>(response_body_size), + ' '); + + std::vector<net::MockRead> reads_list; + std::vector<std::string> mock_writes; + std::vector<net::MockWrite> writes_list; + + std::vector<net::EffectiveConnectionType> effective_connection_types; + effective_connection_types.push_back(net::EFFECTIVE_CONNECTION_TYPE_SLOW_2G); + effective_connection_types.push_back(net::EFFECTIVE_CONNECTION_TYPE_2G); + + BuildSocket(response_headers, response_body, true, effective_connection_types, + &reads_list, &mock_writes, &writes_list); + + // Add 1 socket provider since 1 request in this test is fetched from the + // network. + FetchURLRequestAndVerifyECTHeader(effective_connection_types[0], true, false); + + // When the ECT is set to the same value, fetching the same resource should + // result in a cache hit. + FetchURLRequestAndVerifyECTHeader(effective_connection_types[0], true, true); + + // When the ECT is set to a different value, the response should still be + // served from the cache. + FetchURLRequestAndVerifyECTHeader(effective_connection_types[1], true, true); +} + +class DataReductionProxyNetworkDelegateClientLoFiTest : public testing::Test { + public: + DataReductionProxyNetworkDelegateClientLoFiTest() : baseline_savings_(0) {} + ~DataReductionProxyNetworkDelegateClientLoFiTest() override; + + void Reset() { + drp_test_context_.reset(); + mock_socket_factory_.reset(); + context_storage_.reset(); + + context_.reset(new net::TestURLRequestContext(true)); + context_storage_.reset(new net::URLRequestContextStorage(context_.get())); + mock_socket_factory_.reset(new net::MockClientSocketFactory()); + context_->set_client_socket_factory(mock_socket_factory_.get()); + + drp_test_context_ = + DataReductionProxyTestContext::Builder() + .WithURLRequestContext(context_.get()) + .WithMockClientSocketFactory(mock_socket_factory_.get()) + .Build(); + + drp_test_context_->AttachToURLRequestContext(context_storage_.get()); + context_->Init(); + base::RunLoop().RunUntilIdle(); + + baseline_savings_ = + drp_test_context()->settings()->GetTotalHttpContentLengthSaved(); + } + + void SetUpLoFiDecider(bool is_client_lofi_image, + bool is_client_lofi_auto_reload) const { + std::unique_ptr<TestLoFiDecider> lofi_decider(new TestLoFiDecider()); + lofi_decider->SetIsUsingClientLoFi(is_client_lofi_image); + lofi_decider->SetIsClientLoFiAutoReload(is_client_lofi_auto_reload); + drp_test_context_->io_data()->set_lofi_decider( + std::unique_ptr<LoFiDecider>(std::move(lofi_decider))); + } + + int64_t GetSavings() const { + return drp_test_context()->settings()->GetTotalHttpContentLengthSaved() - + baseline_savings_; + } + + net::TestURLRequestContext* context() const { return context_.get(); } + net::MockClientSocketFactory* mock_socket_factory() const { + return mock_socket_factory_.get(); + } + DataReductionProxyTestContext* drp_test_context() const { + return drp_test_context_.get(); + } + + private: + base::MessageLoopForIO loop; + std::unique_ptr<net::TestURLRequestContext> context_; + std::unique_ptr<net::URLRequestContextStorage> context_storage_; + std::unique_ptr<net::MockClientSocketFactory> mock_socket_factory_; + std::unique_ptr<DataReductionProxyTestContext> drp_test_context_; + int64_t baseline_savings_; +}; + +DataReductionProxyNetworkDelegateClientLoFiTest:: + ~DataReductionProxyNetworkDelegateClientLoFiTest() {} + +TEST_F(DataReductionProxyNetworkDelegateClientLoFiTest, DataSavingsNonDRP) { + const char kSimple200ResponseHeaders[] = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 140\r\n\r\n"; + + const struct { + const char* headers; + size_t response_length; + bool is_client_lofi_image; + bool is_client_lofi_auto_reload; + int64_t expected_savings; + } tests[] = { + // 200 responses shouldn't see any savings. + {kSimple200ResponseHeaders, 140, false, false, 0}, + {kSimple200ResponseHeaders, 140, true, false, 0}, + + // Client Lo-Fi Auto-reload responses should see negative savings. + {kSimple200ResponseHeaders, 140, false, true, + -(static_cast<int64_t>(sizeof(kSimple200ResponseHeaders) - 1) + 140)}, + {kSimple200ResponseHeaders, 140, true, true, + -(static_cast<int64_t>(sizeof(kSimple200ResponseHeaders) - 1) + 140)}, + + // A range response that doesn't use Client Lo-Fi shouldn't see any + // savings. + {"HTTP/1.1 206 Partial Content\r\n" + "Content-Range: bytes 0-2047/10000\r\n" + "Content-Length: 2048\r\n\r\n", + 2048, false, false, 0}, + + // A Client Lo-Fi range response should see savings based on the + // Content-Range header. + {"HTTP/1.1 206 Partial Content\r\n" + "Content-Range: bytes 0-2047/10000\r\n" + "Content-Length: 2048\r\n\r\n", + 2048, true, false, 10000 - 2048}, + + // A Client Lo-Fi range response should see savings based on the + // Content-Range header, which in this case is 0 savings because the range + // response contained the entire resource. + {"HTTP/1.1 206 Partial Content\r\n" + "Content-Range: bytes 0-999/1000\r\n" + "Content-Length: 1000\r\n\r\n", + 1000, true, false, 0}, + + // Client Lo-Fi range responses that don't have a Content-Range with the + // full resource length shouldn't see any savings. + {"HTTP/1.1 206 Partial Content\r\n" + "Content-Length: 2048\r\n\r\n", + 2048, true, false, 0}, + {"HTTP/1.1 206 Partial Content\r\n" + "Content-Range: bytes 0-2047/*\r\n" + "Content-Length: 2048\r\n\r\n", + 2048, true, false, 0}, + {"HTTP/1.1 206 Partial Content\r\n" + "Content-Range: invalid_content_range\r\n" + "Content-Length: 2048\r\n\r\n", + 2048, true, false, 0}, + }; + + for (const auto& test : tests) { + Reset(); + SetUpLoFiDecider(test.is_client_lofi_image, + test.is_client_lofi_auto_reload); + + std::string response_body(test.response_length, 'a'); + net::MockRead reads[] = {net::MockRead(test.headers), + net::MockRead(response_body.c_str()), + net::MockRead(net::ASYNC, net::OK)}; + net::StaticSocketDataProvider socket(reads, arraysize(reads), nullptr, 0); + mock_socket_factory()->AddSocketDataProvider(&socket); + + net::TestDelegate test_delegate; + std::unique_ptr<net::URLRequest> request = context()->CreateRequest( + GURL("http://example.com"), net::RequestPriority::IDLE, &test_delegate, + TRAFFIC_ANNOTATION_FOR_TESTS); + + request->Start(); + base::RunLoop().RunUntilIdle(); + + EXPECT_EQ(test.expected_savings, GetSavings()) << (&test - tests); + } +} + +TEST_F(DataReductionProxyNetworkDelegateClientLoFiTest, DataSavingsThroughDRP) { + Reset(); + drp_test_context()->EnableDataReductionProxyWithSecureProxyCheckSuccess(); + SetUpLoFiDecider(true, false); + + const char kHeaders[] = + "HTTP/1.1 206 Partial Content\r\n" + "Content-Range: bytes 0-2047/10000\r\n" + "Content-Length: 2048\r\n" + "Via: 1.1 Chrome-Compression-Proxy\r\n" + "X-Original-Content-Length: 3000\r\n\r\n"; + + std::string response_body(2048, 'a'); + net::MockRead reads[] = {net::MockRead(kHeaders), + net::MockRead(response_body.c_str()), + net::MockRead(net::ASYNC, net::OK)}; + net::StaticSocketDataProvider socket(reads, arraysize(reads), nullptr, 0); + mock_socket_factory()->AddSocketDataProvider(&socket); + + net::TestDelegate test_delegate; + std::unique_ptr<net::URLRequest> request = context()->CreateRequest( + GURL("http://example.com"), net::RequestPriority::IDLE, &test_delegate, + TRAFFIC_ANNOTATION_FOR_TESTS); + + request->Start(); + base::RunLoop().RunUntilIdle(); + + // Since the Data Reduction Proxy is enabled, the length of the raw headers + // should be used in the estimated original size. The X-OCL should be ignored. + EXPECT_EQ(static_cast<int64_t>(net::HttpUtil::AssembleRawHeaders( + kHeaders, sizeof(kHeaders) - 1) + .size() + + 10000 - request->GetTotalReceivedBytes()), + GetSavings()); +} + } // namespace } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc index 4d21e6666ed..24aa6a384bc 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.cc @@ -20,6 +20,7 @@ #include "components/data_use_measurement/core/data_use_user_data.h" #include "net/base/load_flags.h" #include "net/nqe/effective_connection_type.h" +#include "net/traffic_annotation/network_traffic_annotation.h" #include "net/url_request/url_fetcher.h" #include "net/url_request/url_request_context_getter.h" #include "net/url_request/url_request_status.h" @@ -111,6 +112,11 @@ void AddDataToPageloadMetrics(const DataReductionProxyData& request_data, request->set_previews_type(PageloadMetrics_PreviewsType_NONE); } + // Only report opt out information if a server preview was shown (otherwise, + // report opt out unknown). Similarly, if app background (Android) caused this + // report to be sent before the page load is terminated, do not report opt out + // information as the user could reload the original preview after this report + // is sent. if (!was_preview_shown || timing.app_background_occurred) { request->set_previews_opt_out(PageloadMetrics_PreviewsOptOut_UNKNOWN); return; @@ -191,12 +197,39 @@ void DataReductionProxyPingbackClient::CreateFetcherForDataAndStart() { std::string serialized_request = AddTimeAndSerializeRequest(&metrics_request_, CurrentTime()); metrics_request_.Clear(); - current_fetcher_ = - net::URLFetcher::Create(pingback_url_, net::URLFetcher::POST, this); + net::NetworkTrafficAnnotationTag traffic_annotation = + net::DefineNetworkTrafficAnnotation("data_reduction_proxy_pingback", R"( + semantics { + sender: "Data Reduction Proxy" + description: + "Sends page performance and data efficiency metrics to the data " + "reduction proxy." + trigger: + "Sent after a page load, if the page was loaded via the data " + "reduction proxy." + data: + "URL, request time, response time, page size, connection type, and " + "performance measures. See the following for details: " + "components/data_reduction_proxy/proto/pageload_metrics.proto" + destination: GOOGLE_OWNED_SERVICE + } + policy { + cookies_allowed: false + setting: + "Users can control Data Saver on Android via 'Data Saver' setting. " + "Data Saver is not available on iOS, and on desktop it is enabled " + "by insalling the Data Saver extension. While Data Saver is " + "enabled, this feature cannot be disabled by settings." + policy_exception_justification: "Not implemented." + })"); + current_fetcher_ = net::URLFetcher::Create( + pingback_url_, net::URLFetcher::POST, this, traffic_annotation); data_use_measurement::DataUseUserData::AttachToFetcher( current_fetcher_.get(), data_use_measurement::DataUseUserData::DATA_REDUCTION_PROXY); - current_fetcher_->SetLoadFlags(net::LOAD_BYPASS_PROXY); + current_fetcher_->SetLoadFlags(net::LOAD_BYPASS_PROXY | + net::LOAD_DO_NOT_SEND_COOKIES | + net::LOAD_DO_NOT_SAVE_COOKIES); current_fetcher_->SetUploadData("application/x-protobuf", serialized_request); current_fetcher_->SetRequestContext(url_request_context_); // |current_fetcher_| should not retry on 5xx errors since the server may diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc index 85671575bb3..8d241701f34 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_prefs_unittest.cc @@ -39,7 +39,7 @@ class DataReductionProxyPrefsTest : public testing::Test { PrefService* pref_service) { ListPrefUpdate list(local_state_prefs(), pref_name); for (int64_t i = 0; i < 10L; ++i) { - list->Set(i, new base::Value(base::Int64ToString(i + starting_value))); + list->AppendString(base::Int64ToString(i + starting_value)); } } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc index 470a67597e5..2dc55ff8b81 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options.cc @@ -17,6 +17,7 @@ #include "base/time/time.h" #include "build/build_config.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_config.h" +#include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_params.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h" @@ -100,12 +101,13 @@ std::string DataReductionProxyRequestOptions::GetHeaderValueForTesting() const { } void DataReductionProxyRequestOptions::UpdateExperiments() { - // TODO(bengr): Simplify this so there's only one way to set experiment via - // flags. See crbug.com/656195. + experiments_.clear(); std::string experiments = base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( data_reduction_proxy::switches::kDataReductionProxyExperiment); + // The command line override takes precedence over field trial "exp" + // directives. if (!experiments.empty()) { base::StringTokenizer experiment_tokenizer(experiments, ", "); experiment_tokenizer.set_quote_chars("\""); @@ -113,7 +115,11 @@ void DataReductionProxyRequestOptions::UpdateExperiments() { if (!experiment_tokenizer.token().empty()) experiments_.push_back(experiment_tokenizer.token()); } + } else if (params::AreLitePagesEnabledViaFlags()) { + experiments_.push_back(chrome_proxy_force_lite_page_experiment()); } else { + // If no other "exp" directive is forced by flags, add the field trial + // value. AddServerExperimentFromFieldTrial(); } diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc index 1e58b8a5452..fa8df6aff62 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_request_options_unittest.cc @@ -11,6 +11,7 @@ #include "base/command_line.h" #include "base/md5.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/strings/string16.h" #include "base/strings/utf_string_conversions.h" @@ -330,6 +331,56 @@ TEST_F(DataReductionProxyRequestOptionsTest, ParseExperimentsFromFieldTrial) { } } +TEST_F(DataReductionProxyRequestOptionsTest, TestExperimentPrecedence) { + // Tests that combinations of configurations that trigger "exp=" directive in + // the Chrome-Proxy header have the right precendence, and only append a value + // for the highest priority value. + + // Field trial has the lowest priority. + std::map<std::string, std::string> server_experiment; + server_experiment["exp"] = "foo"; + ASSERT_TRUE(variations::AssociateVariationParams( + params::GetServerExperimentsFieldTrialName(), "enabled", + server_experiment)); + + base::FieldTrialList field_trial_list(nullptr); + base::FieldTrialList::CreateFieldTrial( + params::GetServerExperimentsFieldTrialName(), "enabled"); + std::vector<std::string> expected_experiments; + expected_experiments.push_back("foo"); + std::string expected_header; + SetHeaderExpectations(kExpectedSession, kExpectedCredentials, std::string(), + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, + expected_experiments, &expected_header); + CreateRequestOptions(kVersion); + VerifyExpectedHeader(expected_header, kPageIdValue); + + // "force_lite_page" has the next lowest priority. + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + switches::kDataReductionProxyLoFi, + switches::kDataReductionProxyLoFiValueAlwaysOn); + base::CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableDataReductionProxyLitePage); + expected_experiments.clear(); + expected_experiments.push_back(chrome_proxy_force_lite_page_experiment()); + SetHeaderExpectations(kExpectedSession, kExpectedCredentials, std::string(), + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, + expected_experiments, &expected_header); + CreateRequestOptions(kVersion); + VerifyExpectedHeader(expected_header, kPageIdValue); + + // Setting the experiment explicitly has the highest priority. + base::CommandLine::ForCurrentProcess()->AppendSwitchASCII( + data_reduction_proxy::switches::kDataReductionProxyExperiment, "bar"); + expected_experiments.clear(); + expected_experiments.push_back("bar"); + SetHeaderExpectations(kExpectedSession, kExpectedCredentials, std::string(), + kClientStr, kExpectedBuild, kExpectedPatch, kPageId, + expected_experiments, &expected_header); + CreateRequestOptions(kVersion); + VerifyExpectedHeader(expected_header, kPageIdValue); +} + TEST_F(DataReductionProxyRequestOptionsTest, GetSessionKeyFromRequestHeaders) { const struct { std::string chrome_proxy_header_key; diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.cc index e0abfe8769e..970be81f2ab 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.cc @@ -13,6 +13,7 @@ #include "base/sequenced_task_runner.h" #include "base/single_thread_task_runner.h" #include "base/task_runner_util.h" +#include "base/time/time.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_compression_stats.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_io_data.h" #include "components/data_reduction_proxy/core/browser/data_reduction_proxy_pingback_client.h" @@ -91,6 +92,16 @@ void DataReductionProxyService::Shutdown() { weak_factory_.InvalidateWeakPtrs(); } +void DataReductionProxyService::UpdateDataUseForHost(int64_t network_bytes, + int64_t original_bytes, + const std::string& host) { + DCHECK(CalledOnValidThread()); + if (compression_stats_) { + compression_stats_->RecordDataUsage(host, original_bytes, network_bytes, + base::Time::Now()); + } +} + void DataReductionProxyService::UpdateContentLengths( int64_t data_used, int64_t original_size, diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h index b1d8fc167ae..077f0d20a1c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_service.h @@ -77,6 +77,11 @@ class DataReductionProxyService // final step in initialization. bool Initialized() const; + // Records data usage per host. + void UpdateDataUseForHost(int64_t network_bytes, + int64_t original_bytes, + const std::string& host); + // Records daily data savings statistics in |compression_stats_|. void UpdateContentLengths(int64_t data_used, int64_t original_size, diff --git a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc index 8af34b7db5c..e111c5a745c 100644 --- a/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/browser/data_reduction_proxy_settings_unittest.cc @@ -12,6 +12,7 @@ #include "base/command_line.h" #include "base/macros.h" #include "base/md5.h" +#include "base/message_loop/message_loop.h" #include "base/metrics/field_trial.h" #include "base/metrics/histogram_samples.h" #include "base/test/histogram_tester.h" diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc index 6e09db8dc84..f2547d74569 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.cc @@ -7,8 +7,7 @@ namespace data_reduction_proxy { namespace features { -// Enables the Data Reduction Proxy menu item in the main menu rather than under -// Settings on Android. +// Enables the Data Reduction Proxy footer in the main menu on Android. const base::Feature kDataReductionMainMenu{"DataReductionProxyMainMenu", base::FEATURE_DISABLED_BY_DEFAULT}; @@ -16,5 +15,12 @@ const base::Feature kDataReductionMainMenu{"DataReductionProxyMainMenu", const base::Feature kDataReductionSiteBreakdown{ "DataReductionProxySiteBreakdown", base::FEATURE_DISABLED_BY_DEFAULT}; +// Enables a new version of the data reduction proxy protocol where the server +// decides if a server-generated preview should be served. The previous +// version required the client to make this decision. The new protocol relies +// on updates primarily to the Chrome-Proxy-Accept-Transform header. +const base::Feature kDataReductionProxyDecidesTransform{ + "DataReductionProxyDecidesTransform", base::FEATURE_DISABLED_BY_DEFAULT}; + } // namespace features } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h index 9913b055a51..af951c894ae 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_features.h @@ -12,6 +12,7 @@ namespace features { extern const base::Feature kDataReductionMainMenu; extern const base::Feature kDataReductionSiteBreakdown; +extern const base::Feature kDataReductionProxyDecidesTransform; } // namespace features } // namespace data_reduction_proxy diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc index 3a697aecaf8..8961f1b7afb 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.cc @@ -29,6 +29,7 @@ using base::TimeDelta; namespace { const char kChromeProxyHeader[] = "chrome-proxy"; +const char kChromeProxyECTHeader[] = "chrome-proxy-ect"; const char kChromeProxyAcceptTransformHeader[] = "chrome-proxy-accept-transform"; const char kChromeProxyContentTransformHeader[] = @@ -45,8 +46,7 @@ const char kIdentityDirective[] = "identity"; // The legacy Chrome-Proxy response header directive for LoFi images. const char kLegacyChromeProxyLoFiResponseDirective[] = "q=low"; -const char kChromeProxyLitePageIngoreBlacklistDirective[] = - "exp=ignore_preview_blacklist"; +const char kChromeProxyForceLitePageExperiment[] = "force_lite_page"; const char kIfHeavyQualifier[] = "if-heavy"; @@ -126,6 +126,10 @@ const char* chrome_proxy_header() { return kChromeProxyHeader; } +const char* chrome_proxy_ect_header() { + return kChromeProxyECTHeader; +} + const char* chrome_proxy_accept_transform_header() { return kChromeProxyAcceptTransformHeader; } @@ -150,8 +154,8 @@ const char* identity_directive() { return kIdentityDirective; } -const char* chrome_proxy_lite_page_ignore_blacklist_directive() { - return kChromeProxyLitePageIngoreBlacklistDirective; +const char* chrome_proxy_force_lite_page_experiment() { + return kChromeProxyForceLitePageExperiment; } const char* if_heavy_qualifier() { diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h index e059d641d74..83202075cf1 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h @@ -68,6 +68,10 @@ struct DataReductionProxyInfo { // Gets the header used for data reduction proxy requests and responses. const char* chrome_proxy_header(); +// Gets the chrome-proxy-ect request header that includes the effective +// connection type. +const char* chrome_proxy_ect_header(); + // Gets the ChromeProxyAcceptTransform header name. const char* chrome_proxy_accept_transform_header(); @@ -94,9 +98,9 @@ const char* identity_directive(); // preview requests and responses. const char* chrome_proxy_lite_page_directive(); -// Gets the Chrome-Proxy directive used by data reduction proxy lite page -// preview experiment to ignore the blacklist. -const char* chrome_proxy_lite_page_ignore_blacklist_directive(); +// Gets the Chrome-Proxy experiment ("exp") value to force a lite page preview +// for requests that accept lite pages. +const char* chrome_proxy_force_lite_page_experiment(); // Requests a transformation only if the server determines that the page is // otherwise heavy (i.e., the associated page load ordinarily requires the diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc index 44bb16c9f56..f0a2d3d0831 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.cc @@ -45,6 +45,9 @@ const char kLitePageFallbackFieldTrial[] = "DataCompressionProxyLitePageFallback"; const char kLoFiFlagFieldTrial[] = "DataCompressionProxyLoFiFlag"; +const char kBlackListTransitionFieldTrial[] = + "DataReductionProxyPreviewsBlackListTransition"; + const char kTrustedSpdyProxyFieldTrialName[] = "DataReductionTrustedSpdyProxy"; // Default URL for retrieving the Data Reduction Proxy configuration. @@ -59,6 +62,9 @@ const char kPingbackURL[] = const char kServerExperimentsFieldTrial[] = "DataReductionProxyServerExperiments"; +// LitePage black list version. +const char kLitePageBlackListVersion[] = "lite-page-blacklist-version"; + bool IsIncludedInFieldTrial(const std::string& name) { return base::StartsWith(FieldTrialList::FindFullName(name), kEnabled, base::CompareCase::SENSITIVE); @@ -249,15 +255,6 @@ const char* GetQuicFieldTrialName() { return kQuicFieldTrial; } -bool IsZeroRttQuicEnabled() { - if (!IsIncludedInQuicFieldTrial()) - return false; - std::map<std::string, std::string> params; - variations::GetVariationParams(GetQuicFieldTrialName(), ¶ms); - return GetStringValueForVariationParamWithDefaultValue( - params, "enable_zero_rtt", "false") == "true"; -} - bool IsBrotliAcceptEncodingEnabled() { // Brotli encoding is enabled by default since the data reduction proxy server // controls when to serve Brotli encoded content. It can be disabled in @@ -276,6 +273,12 @@ bool IsConfigClientEnabled() { kDisabled, base::CompareCase::SENSITIVE); } +bool IsBlackListEnabledForServerPreviews() { + return base::StartsWith( + base::FieldTrialList::FindFullName(kBlackListTransitionFieldTrial), + kEnabled, base::CompareCase::SENSITIVE); +} + GURL GetConfigServiceURL() { base::CommandLine* command_line = base::CommandLine::ForCurrentProcess(); std::string url; @@ -321,6 +324,12 @@ bool ShouldForceEnableDataReductionProxy() { data_reduction_proxy::switches::kEnableDataReductionProxy); } +int LitePageVersion() { + return GetFieldTrialParameterAsInteger( + data_reduction_proxy::params::GetLoFiFieldTrialName(), + kLitePageBlackListVersion, 0, 0); +} + int GetFieldTrialParameterAsInteger(const std::string& group, const std::string& param_name, int default_value, diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h index eab391b026e..badc927146f 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params.h @@ -127,9 +127,6 @@ bool IsQuicEnabledForNonCoreProxies(); const char* GetQuicFieldTrialName(); -// Returns true if zero RTT for QUIC is enabled. -bool IsZeroRttQuicEnabled(); - // Returns true if Brotli should be added to the accept-encoding header. bool IsBrotliAcceptEncodingEnabled(); @@ -148,6 +145,13 @@ GURL GetConfigServiceURL(); // command line. bool ShouldForceEnableDataReductionProxy(); +// Whether the blacklist should be used for server Lo-Fi and server Lite Page +// instead of the prefs-based rules. +bool IsBlackListEnabledForServerPreviews(); + +// The current LitePage experiment blacklist version. +int LitePageVersion(); + // Retrieves the int stored in |param_name| from the field trial group // |group|. If the value is not present, cannot be parsed, or is less than // |min_value|, returns |default_value|. diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc index b045ae0ddd9..0c983328807 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_params_unittest.cc @@ -408,29 +408,23 @@ TEST_F(DataReductionProxyParamsTest, QuicFieldTrial) { const struct { std::string trial_group_name; bool expected_enabled; - std::string zero_rtt_param; - bool expected_zero_rtt; bool enable_warmup_url; bool expect_warmup_url_enabled; std::string warmup_url; } tests[] = { - {"Enabled", true, "true", true, true, true, std::string()}, - {"Enabled", true, "true", true, false, false, std::string()}, - {"Enabled_Control", true, "true", true, true, true, std::string()}, - {"Enabled_Control", true, "false", false, true, true, std::string()}, - {"Enabled_Control", true, std::string(), false, true, true, - std::string()}, - {"Control", false, "true", false, true, true, std::string()}, - {"Disabled", false, "false", false, true, false, std::string()}, - {"enabled", true, "false", false, true, true, std::string()}, - {"Enabled", true, "true", true, true, true, "example.com/test.html"}, - {std::string(), true, "true", false, false, false, std::string()}, + {"Enabled", true, true, true, std::string()}, + {"Enabled", true, false, false, std::string()}, + {"Enabled_Control", true, true, true, std::string()}, + {"Control", false, true, true, std::string()}, + {"Disabled", false, true, false, std::string()}, + {"enabled", true, true, true, std::string()}, + {"Enabled", true, true, true, "example.com/test.html"}, + {std::string(), true, false, false, std::string()}, }; for (const auto& test : tests) { variations::testing::ClearAllVariationParams(); std::map<std::string, std::string> variation_params; - variation_params["enable_zero_rtt"] = test.zero_rtt_param; if (test.enable_warmup_url) variation_params["enable_warmup"] = "true"; @@ -445,7 +439,6 @@ TEST_F(DataReductionProxyParamsTest, QuicFieldTrial) { test.trial_group_name); EXPECT_EQ(test.expected_enabled, params::IsIncludedInQuicFieldTrial()); - EXPECT_EQ(test.expected_zero_rtt, params::IsZeroRttQuicEnabled()); if (!test.warmup_url.empty()) { EXPECT_EQ(GURL(test.warmup_url), params::GetWarmupURL()); } else { diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc index 82dc96a2074..4877facd493 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.cc @@ -18,6 +18,10 @@ const char kDataReductionProxyConfigURL[] = "data-reduction-proxy-config-url"; // Proxy field trials. const char kDataReductionProxyExperiment[] = "data-reduction-proxy-experiment"; +// The Chrome-Proxy "exp" directive value used by data reduction proxy to +// receive an alternative back end implementation. +const char kDataReductionProxyServerAlternative[] = "alternative"; + // The origin of the data reduction proxy fallback. const char kDataReductionProxyFallback[] = "spdy-proxy-auth-fallback"; diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h index cbc063d6836..0773ba8d281 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h @@ -11,6 +11,7 @@ namespace switches { // All switches in alphabetical order. The switches should be documented // alongside the definition of their values in the .cc file. +extern const char kDataReductionPingbackURL[]; extern const char kDataReductionProxy[]; extern const char kDataReductionProxyConfigURL[]; extern const char kDataReductionProxyExperiment[]; @@ -22,9 +23,9 @@ extern const char kDataReductionProxyLoFiValueAlwaysOn[]; extern const char kDataReductionProxyLoFiValueCellularOnly[]; extern const char kDataReductionProxyLoFiValueDisabled[]; extern const char kDataReductionProxyLoFiValueSlowConnectionsOnly[]; -extern const char kDataReductionPingbackURL[]; extern const char kDataReductionProxySecureProxyCheckURL[]; extern const char kDataReductionProxyServerExperimentsDisabled[]; +extern const char kDataReductionProxyServerAlternative[]; extern const char kDataReductionProxyWarmupURL[]; extern const char kEnableDataReductionProxy[]; extern const char kEnableDataReductionProxyBypassWarning[]; diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.cc b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.cc index 9af46dd61df..ed6759db08d 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.cc +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.cc @@ -13,6 +13,7 @@ #include "net/base/net_errors.h" #include "net/base/url_util.h" #include "net/http/http_response_headers.h" +#include "net/http/http_util.h" #include "net/proxy/proxy_config.h" #include "net/proxy/proxy_info.h" #include "net/url_request/url_request.h" @@ -131,11 +132,6 @@ const char* GetStringForClient(Client client) { } } -bool IsMethodIdempotent(const std::string& method) { - return method == "GET" || method == "OPTIONS" || method == "HEAD" || - method == "PUT" || method == "DELETE" || method == "TRACE"; -} - GURL AddApiKeyToUrl(const GURL& url) { GURL new_url = url; #if defined(USE_GOOGLE_API_KEYS) @@ -151,7 +147,7 @@ bool EligibleForDataReductionProxy(const net::ProxyInfo& proxy_info, const GURL& url, const std::string& method) { return proxy_info.is_direct() && proxy_info.proxy_list().size() == 1 && - !url.SchemeIsWSOrWSS() && IsMethodIdempotent(method); + !url.SchemeIsWSOrWSS() && net::HttpUtil::IsMethodIdempotent(method); } bool ApplyProxyConfigToProxyInfo(const net::ProxyConfig& proxy_config, diff --git a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.h b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.h index be0a16dabff..811faa9025f 100644 --- a/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.h +++ b/chromium/components/data_reduction_proxy/core/common/data_reduction_proxy_util.h @@ -66,9 +66,6 @@ void GetChromiumBuildAndPatchAsInts(const std::string& version_string, // Get the human-readable version of |client|. const char* GetStringForClient(Client client); -// Returns true if the request method is idempotent. -bool IsMethodIdempotent(const std::string& method); - GURL AddApiKeyToUrl(const GURL& url); // Returns whether this is valid for data reduction proxy use. |proxy_info| diff --git a/chromium/components/data_reduction_proxy/core/common/lofi_decider.h b/chromium/components/data_reduction_proxy/core/common/lofi_decider.h index 2eca7590fdb..61ce4668d6d 100644 --- a/chromium/components/data_reduction_proxy/core/common/lofi_decider.h +++ b/chromium/components/data_reduction_proxy/core/common/lofi_decider.h @@ -54,15 +54,19 @@ class LoFiDecider { virtual void RemoveAcceptTransformHeader( net::HttpRequestHeaders* headers) const = 0; - // Adds a directive to tell the server to ignore blacklists when a Lite Page - // preview is being requested due to command line flags being set. - virtual void MaybeSetIgnorePreviewsBlacklistDirective( - net::HttpRequestHeaders* headers) const = 0; - // Returns true if the Lo-Fi specific UMA should be recorded. It is set to // true if Lo-Fi is enabled for |request|, Chrome session is in Lo-Fi // Enabled or Control field trial, and the network quality was slow. virtual bool ShouldRecordLoFiUMA(const net::URLRequest& request) const = 0; + + // Returns whether the request was a client-side Lo-Fi image request. + virtual bool IsClientLoFiImageRequest( + const net::URLRequest& request) const = 0; + + // Returns true if the request is for a client-side Lo-Fi image that is being + // automatically reloaded because of a decoding error. + virtual bool IsClientLoFiAutoReloadRequest( + const net::URLRequest& request) const = 0; }; } // namespace data_reduction_proxy |