// Copyright (c) 2019 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "base/feature_list.h" #include "base/logging.h" #include "base/strings/string_split.h" #include "base/strings/string_util.h" #include "base/test/scoped_feature_list.h" #include "build/build_config.h" #include "chrome/browser/profiles/profile.h" #include "chrome/browser/ui/browser.h" #include "chrome/browser/ui/tabs/tab_strip_model.h" #include "chrome/test/base/in_process_browser_test.h" #include "chrome/test/base/ui_test_utils.h" #include "components/language/core/browser/pref_names.h" #include "components/prefs/pref_service.h" #include "components/user_prefs/user_prefs.h" #include "content/public/browser/browser_context.h" #include "content/public/common/content_features.h" #include "content/public/common/service_manager_connection.h" #include "content/public/test/browser_test_utils.h" #include "content/public/test/content_browser_test_utils.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver_set.h" #include "net/test/embedded_test_server/embedded_test_server.h" #include "net/test/embedded_test_server/request_handler_util.h" #include "services/image_annotation/public/cpp/image_processor.h" #include "services/image_annotation/public/mojom/constants.mojom.h" #include "services/image_annotation/public/mojom/image_annotation.mojom.h" #include "services/service_manager/public/cpp/binder_registry.h" #include "services/service_manager/public/cpp/connector.h" #include "services/service_manager/public/cpp/service.h" #include "services/service_manager/public/cpp/service_binding.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "ui/accessibility/ax_enum_util.h" #include "ui/accessibility/ax_enums.mojom.h" #include "ui/accessibility/ax_tree.h" #include "url/gurl.h" constexpr base::FilePath::CharType kDocRoot[] = FILE_PATH_LITERAL("chrome/test/data/accessibility"); namespace { void DescribeNodesWithAnnotations(const ui::AXNode& node, std::vector* descriptions) { std::string annotation = node.GetStringAttribute(ax::mojom::StringAttribute::kImageAnnotation); if (!annotation.empty()) { descriptions->push_back(ui::ToString(node.data().role) + std::string(" ") + annotation); } for (const auto* child : node.children()) DescribeNodesWithAnnotations(*child, descriptions); } std::vector DescribeNodesWithAnnotations( const ui::AXTreeUpdate& tree_update) { ui::AXTree tree(tree_update); std::vector descriptions; DCHECK(tree.root()); DescribeNodesWithAnnotations(*tree.root(), &descriptions); return descriptions; } bool HasNodeWithAnnotationStatus(const ui::AXTreeUpdate& tree_update, ax::mojom::ImageAnnotationStatus status) { for (const auto& node_data : tree_update.nodes) { if (node_data.GetImageAnnotationStatus() == status) return true; } return false; } // A fake implementation of the Annotator mojo interface that // returns predictable results based on the filename of the image // it's asked to annotate. Enables us to test the rest of the // system without using the real annotator that queries a back-end // API. class FakeAnnotator : public image_annotation::mojom::Annotator { public: static void SetReturnOcrResults(bool ocr) { return_ocr_results_ = ocr; } static void SetReturnLabelResults(bool label) { return_label_results_ = label; } static void SetReturnErrorCode( image_annotation::mojom::AnnotateImageError error_code) { return_error_code_ = error_code; } FakeAnnotator() = default; ~FakeAnnotator() override = default; void BindReceiver( mojo::PendingReceiver receiver) { receivers_.Add(this, std::move(receiver)); } void AnnotateImage( const std::string& image_id, const std::string& description_language_tag, mojo::PendingRemote image_processor, AnnotateImageCallback callback) override { if (return_error_code_) { image_annotation::mojom::AnnotateImageResultPtr result = image_annotation::mojom::AnnotateImageResult::NewErrorCode( *return_error_code_); std::move(callback).Run(std::move(result)); return; } // Use the filename to create an annotation string. // Adds some trailing whitespace and punctuation to check that clean-up // happens correctly when combining annotation strings. std::string image_filename = GURL(image_id).ExtractFileName(); image_annotation::mojom::AnnotationPtr ocr_annotation = image_annotation::mojom::Annotation::New( image_annotation::mojom::AnnotationType::kOcr, 1.0, image_filename + " Annotation . "); image_annotation::mojom::AnnotationPtr label_annotation = image_annotation::mojom::Annotation::New( image_annotation::mojom::AnnotationType::kLabel, 1.0, image_filename + " '" + description_language_tag + "' Label"); // Return enabled results as an annotation. std::vector annotations; if (return_ocr_results_) annotations.push_back(std::move(ocr_annotation)); if (return_label_results_) annotations.push_back(std::move(label_annotation)); image_annotation::mojom::AnnotateImageResultPtr result = image_annotation::mojom::AnnotateImageResult::NewAnnotations( std::move(annotations)); std::move(callback).Run(std::move(result)); } private: mojo::ReceiverSet receivers_; static bool return_ocr_results_; static bool return_label_results_; static base::Optional return_error_code_; DISALLOW_COPY_AND_ASSIGN(FakeAnnotator); }; // static bool FakeAnnotator::return_ocr_results_ = false; // static bool FakeAnnotator::return_label_results_ = false; // static base::Optional FakeAnnotator::return_error_code_; // The fake ImageAnnotationService, which handles mojo calls from the renderer // process and passes them to FakeAnnotator. class FakeImageAnnotationService : public service_manager::Service { public: explicit FakeImageAnnotationService( service_manager::mojom::ServiceRequest request) : service_binding_(this, std::move(request)) {} ~FakeImageAnnotationService() override = default; private: // service_manager::Service: void OnBindInterface(const service_manager::BindSourceInfo& source_info, const std::string& interface_name, mojo::ScopedMessagePipeHandle interface_pipe) override { registry_.BindInterface(interface_name, std::move(interface_pipe)); } void OnStart() override { registry_.AddInterface( base::BindRepeating(&FakeAnnotator::BindReceiver, base::Unretained(&annotator_))); } service_manager::BinderRegistry registry_; service_manager::ServiceBinding service_binding_; FakeAnnotator annotator_; DISALLOW_COPY_AND_ASSIGN(FakeImageAnnotationService); }; void HandleImageAnnotatorServiceRequest( service_manager::mojom::ServiceRequest request) { new FakeImageAnnotationService(std::move(request)); } } // namespace class ImageAnnotationBrowserTest : public InProcessBrowserTest { public: ImageAnnotationBrowserTest() : https_server_(net::EmbeddedTestServer::TYPE_HTTPS) { https_server_.AddDefaultHandlers(base::FilePath(kDocRoot)); } protected: void SetUp() override { scoped_feature_list_.InitAndEnableFeature( features::kExperimentalAccessibilityLabels); InProcessBrowserTest::SetUp(); } void SetUpOnMainThread() override { InProcessBrowserTest::SetUpOnMainThread(); ASSERT_TRUE(https_server_.Start()); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); content::ServiceManagerConnection* service_manager_connection = content::BrowserContext::GetServiceManagerConnectionFor( web_contents->GetBrowserContext()); service_manager_connection->AddServiceRequestHandler( image_annotation::mojom::kServiceName, base::BindRepeating(&HandleImageAnnotatorServiceRequest)); ui::AXMode mode = ui::kAXModeComplete; mode.set_mode(ui::AXMode::kLabelImages, true); web_contents->SetAccessibilityMode(mode); SetAcceptLanguages("en,fr"); } void SetAcceptLanguages(const std::string& accept_languages) { content::BrowserContext* context = static_cast(browser()->profile()); DCHECK(context); PrefService* prefs = user_prefs::UserPrefs::Get(context); DCHECK(prefs); prefs->Set(language::prefs::kAcceptLanguages, base::Value(accept_languages)); } protected: net::EmbeddedTestServer https_server_; private: base::test::ScopedFeatureList scoped_feature_list_; DISALLOW_COPY_AND_ASSIGN(ImageAnnotationBrowserTest); }; IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, AnnotateImageInAccessibilityTree) { FakeAnnotator::SetReturnOcrResults(true); FakeAnnotator::SetReturnLabelResults(true); ui_test_utils::NavigateToURL(browser(), https_server_.GetURL("/image_annotation.html")); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); content::WaitForAccessibilityTreeToContainNodeWithName( web_contents, "Appears to say: red.png Annotation. Appears to be: red.png 'en' Label"); } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, ImagesInLinks) { FakeAnnotator::SetReturnOcrResults(true); ui_test_utils::NavigateToURL( browser(), https_server_.GetURL("/image_annotation_link.html")); // Block until the accessibility tree has at least 8 annotations. If // that never happens, the test will time out. content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); while (10 > DescribeNodesWithAnnotations( content::GetAccessibilityTreeSnapshot(web_contents)) .size()) { content::WaitForAccessibilityTreeToChange(web_contents); } // All images should be annotated. Only links that contain exactly one image // should be annotated. ui::AXTreeUpdate ax_tree_update = content::GetAccessibilityTreeSnapshot(web_contents); EXPECT_THAT( DescribeNodesWithAnnotations(ax_tree_update), testing::ElementsAre("image Appears to say: red.png Annotation", "link Appears to say: green.png Annotation", "image Appears to say: green.png Annotation", "image Appears to say: red.png Annotation", "image Appears to say: printer.png Annotation", "image Appears to say: red.png Annotation", "link Appears to say: printer.png Annotation", "image Appears to say: printer.png Annotation", "link Appears to say: green.png Annotation", "image Appears to say: green.png Annotation")); } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, ImageDoc) { FakeAnnotator::SetReturnOcrResults(true); ui_test_utils::NavigateToURL( browser(), https_server_.GetURL("/image_annotation_doc.html")); // Block until the accessibility tree has at least 2 annotations. If // that never happens, the test will time out. content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); while (2 > DescribeNodesWithAnnotations( content::GetAccessibilityTreeSnapshot(web_contents)) .size()) { content::WaitForAccessibilityTreeToChange(web_contents); } // When a document contains exactly one image, the document should be // annotated with the image's annotation, too. ui::AXTreeUpdate ax_tree_update = content::GetAccessibilityTreeSnapshot(web_contents); EXPECT_THAT( DescribeNodesWithAnnotations(ax_tree_update), testing::ElementsAre("rootWebArea Appears to say: red.png Annotation", "image Appears to say: red.png Annotation")); } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, ImageUrl) { FakeAnnotator::SetReturnOcrResults(true); ui_test_utils::NavigateToURL(browser(), https_server_.GetURL("/red.png")); // Block until the accessibility tree has at least 2 annotations. If // that never happens, the test will time out. content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); while (2 > DescribeNodesWithAnnotations( content::GetAccessibilityTreeSnapshot(web_contents)) .size()) { content::WaitForAccessibilityTreeToChange(web_contents); } // When a document contains exactly one image, the document should be // annotated with the image's annotation, too. ui::AXTreeUpdate ax_tree_update = content::GetAccessibilityTreeSnapshot(web_contents); EXPECT_THAT( DescribeNodesWithAnnotations(ax_tree_update), testing::ElementsAre("rootWebArea Appears to say: red.png Annotation", "image Appears to say: red.png Annotation")); } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, NoAnnotationsAvailable) { // Don't return any results. FakeAnnotator::SetReturnOcrResults(false); FakeAnnotator::SetReturnLabelResults(false); ui_test_utils::NavigateToURL( browser(), https_server_.GetURL("/image_annotation_doc.html")); // Block until the annotation status for the root is empty. If that // never occurs then the test will time out. content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); ui::AXTreeUpdate snapshot = content::GetAccessibilityTreeSnapshot(web_contents); while (snapshot.nodes[0].GetImageAnnotationStatus() != ax::mojom::ImageAnnotationStatus::kAnnotationEmpty) { content::WaitForAccessibilityTreeToChange(web_contents); snapshot = content::GetAccessibilityTreeSnapshot(web_contents); } } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, AnnotationError) { // Return an error code. FakeAnnotator::SetReturnErrorCode( image_annotation::mojom::AnnotateImageError::kFailure); ui_test_utils::NavigateToURL( browser(), https_server_.GetURL("/image_annotation_doc.html")); // Block until the annotation status for the root contains an error code. If // that never occurs then the test will time out. content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); ui::AXTreeUpdate snapshot = content::GetAccessibilityTreeSnapshot(web_contents); while (snapshot.nodes[0].GetImageAnnotationStatus() != ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed) { content::WaitForAccessibilityTreeToChange(web_contents); snapshot = content::GetAccessibilityTreeSnapshot(web_contents); } } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, ImageWithSrcSet) { FakeAnnotator::SetReturnOcrResults(true); FakeAnnotator::SetReturnLabelResults(true); ui_test_utils::NavigateToURL(browser(), https_server_.GetURL("/image_srcset.html")); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); content::WaitForAccessibilityTreeToContainNodeWithName( web_contents, "Appears to say: red.png Annotation. Appears to be: red.png 'en' Label"); } // Disabled due to flakiness. http://crbug.com/983404 IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, DISABLED_AnnotationLanguages) { FakeAnnotator::SetReturnOcrResults(true); FakeAnnotator::SetReturnLabelResults(true); ui_test_utils::NavigateToURL(browser(), https_server_.GetURL("/image_annotation.html")); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); content::WaitForAccessibilityTreeToContainNodeWithName( web_contents, "Appears to say: red.png Annotation. Appears to be: red.png 'en' Label"); SetAcceptLanguages("fr,en"); ui_test_utils::NavigateToURL(browser(), https_server_.GetURL("/image_annotation.html")); web_contents = browser()->tab_strip_model()->GetActiveWebContents(); content::WaitForAccessibilityTreeToContainNodeWithName( web_contents, "Appears to say: red.png Annotation. Appears to be: red.png 'fr' Label"); } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, DoesntAnnotateInternalPages) { FakeAnnotator::SetReturnLabelResults(true); ui_test_utils::NavigateToURL(browser(), GURL("chrome://version")); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); ui::AXMode mode = ui::kAXModeComplete; mode.set_mode(ui::AXMode::kLabelImages, true); web_contents->SetAccessibilityMode(mode); std::string svg_image = "data:image/svg+xml;utf8,"; const std::string javascript = "var image = document.createElement('img');" "image.src = \"" + svg_image + "\";" "var outer = document.getElementById('outer');" "outer.insertBefore(image, outer.childNodes[0]);"; EXPECT_TRUE(content::ExecuteScript(web_contents, javascript)); ui::AXTreeUpdate snapshot = content::GetAccessibilityTreeSnapshot(web_contents); // Wait for the accessibility tree to contain an error that the image cannot // be annotated due to the page url's scheme. while (!HasNodeWithAnnotationStatus( snapshot, ax::mojom::ImageAnnotationStatus::kWillNotAnnotateDueToScheme)) { content::WaitForAccessibilityTreeToChange(web_contents); snapshot = content::GetAccessibilityTreeSnapshot(web_contents); } } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, TutorMessageOnlyOnFirstImage) { // We should not promote the image annotation service on more than one image // in the same renderer. FakeAnnotator::SetReturnOcrResults(false); FakeAnnotator::SetReturnLabelResults(false); // The following test page should have at least two images on it. ui_test_utils::NavigateToURL(browser(), https_server_.GetURL("/image_annotation.html")); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); ui::AXMode mode = ui::kAXModeComplete; mode.set_mode(ui::AXMode::kLabelImages, false); web_contents->SetAccessibilityMode(mode); // Block until there are at least two images that have been processed. One of // them should get the tutor message and the other shouldn't. The annotation // status for the image that didn't get the tutor message should be // kSilentlyEligibleForAnnotation whilst the status for the image that did // should be kEligibleForAnnotation. If that never occurs then the test will // time out. ui::AXTreeUpdate snapshot = content::GetAccessibilityTreeSnapshot(web_contents); while ( !HasNodeWithAnnotationStatus( snapshot, ax::mojom::ImageAnnotationStatus::kSilentlyEligibleForAnnotation) || !HasNodeWithAnnotationStatus( snapshot, ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation)) { content::WaitForAccessibilityTreeToChange(web_contents); snapshot = content::GetAccessibilityTreeSnapshot(web_contents); } } IN_PROC_BROWSER_TEST_F(ImageAnnotationBrowserTest, TutorMessageOnlyOnFirstImageInLinks) { // We should not promote the image annotation service on more than one image // in the same renderer. FakeAnnotator::SetReturnOcrResults(false); FakeAnnotator::SetReturnLabelResults(false); // The following test page should have at least two images on it. ui_test_utils::NavigateToURL( browser(), https_server_.GetURL("/image_annotation_link.html")); content::WebContents* web_contents = browser()->tab_strip_model()->GetActiveWebContents(); ui::AXMode mode = ui::kAXModeComplete; mode.set_mode(ui::AXMode::kLabelImages, false); web_contents->SetAccessibilityMode(mode); // Block until there are at least two images that have been processed. One of // them should get the tutor message and the other shouldn't. The annotation // status for the image that didn't get the tutor message should be // kSilentlyEligibleForAnnotation whilst the status for the image that did // should be kEligibleForAnnotation. If that never occurs then the test will // time out. ui::AXTreeUpdate snapshot = content::GetAccessibilityTreeSnapshot(web_contents); while ( !HasNodeWithAnnotationStatus( snapshot, ax::mojom::ImageAnnotationStatus::kSilentlyEligibleForAnnotation) || !HasNodeWithAnnotationStatus( snapshot, ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation)) { content::WaitForAccessibilityTreeToChange(web_contents); snapshot = content::GetAccessibilityTreeSnapshot(web_contents); } }