summaryrefslogtreecommitdiff
path: root/chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc
diff options
context:
space:
mode:
Diffstat (limited to 'chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc')
-rw-r--r--chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc387
1 files changed, 101 insertions, 286 deletions
diff --git a/chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc b/chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc
index 0bce668ea90..aed1001cd56 100644
--- a/chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc
+++ b/chromium/components/autofill_assistant/browser/web/web_controller_browsertest.cc
@@ -13,7 +13,6 @@
#include "base/bind.h"
#include "base/callback.h"
-#include "base/callback_forward.h"
#include "base/callback_helpers.h"
#include "base/containers/checked_range.h"
#include "base/location.h"
@@ -57,11 +56,10 @@
#include "components/autofill_assistant/browser/user_model.h"
#include "components/autofill_assistant/browser/web/element.h"
#include "components/autofill_assistant/browser/web/element_action_util.h"
-#include "components/autofill_assistant/browser/web/element_finder.h"
+#include "components/autofill_assistant/browser/web/element_finder_result.h"
+#include "components/autofill_assistant/browser/web/element_finder_result_type.h"
#include "components/autofill_assistant/browser/web/element_store.h"
-#include "components/autofill_assistant/content/common/autofill_assistant_agent.mojom.h"
-#include "components/autofill_assistant/content/common/autofill_assistant_types.mojom.h"
-#include "components/autofill_assistant/content/common/node_data.h"
+#include "components/autofill_assistant/browser/web/mock_autofill_assistant_agent.h"
#include "content/public/browser/render_frame_host.h"
#include "content/public/browser/web_contents.h"
#include "content/public/browser/web_contents_observer.h"
@@ -69,12 +67,8 @@
#include "content/public/test/browser_test_utils.h"
#include "content/public/test/content_browser_test_utils.h"
#include "content/shell/browser/shell.h"
-#include "mojo/public/cpp/bindings/associated_receiver_set.h"
-#include "net/test/embedded_test_server/embedded_test_server.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
-#include "third_party/blink/public/common/associated_interfaces/associated_interface_provider.h"
-#include "third_party/blink/public/common/switches.h"
#include "url/gurl.h"
#include "url/url_constants.h"
@@ -86,34 +80,8 @@ using ::testing::_;
using ::testing::AnyOf;
using ::testing::IsEmpty;
using ::testing::Return;
-using ::testing::SaveArg;
using ::testing::WithArgs;
-class MockAutofillAssistantAgent : public mojom::AutofillAssistantAgent {
- public:
- MockAutofillAssistantAgent() = default;
- ~MockAutofillAssistantAgent() override = default;
-
- void BindPendingReceiver(mojo::ScopedInterfaceEndpointHandle handle) {
- receivers_.Add(
- this, mojo::PendingAssociatedReceiver<mojom::AutofillAssistantAgent>(
- std::move(handle)));
- }
-
- MOCK_METHOD(void,
- GetSemanticNodes,
- (int32_t role,
- int32_t objective,
- bool ignore_objective,
- base::TimeDelta model_timeout,
- base::OnceCallback<void(mojom::NodeDataStatus,
- const std::vector<NodeData>&)> callback),
- (override));
-
- private:
- mojo::AssociatedReceiverSet<mojom::AutofillAssistantAgent> receivers_;
-};
-
} // namespace
class WebControllerBrowserTest : public autofill_assistant::BaseBrowserTest,
@@ -129,22 +97,13 @@ class WebControllerBrowserTest : public autofill_assistant::BaseBrowserTest,
void SetUpOnMainThread() override {
BaseBrowserTest::SetUpOnMainThread();
- // Register the same agent on all frames, such that the callback can be
- // mocked.
- shell()->web_contents()->GetMainFrame()->ForEachRenderFrameHost(
- base::BindLambdaForTesting([this](content::RenderFrameHost* host) {
- host->GetRemoteAssociatedInterfaces()->OverrideBinderForTesting(
- mojom::AutofillAssistantAgent::Name_,
- base::BindRepeating(
- &MockAutofillAssistantAgent::BindPendingReceiver,
- base::Unretained(&autofill_assistant_agent_)));
- }));
+ MockAutofillAssistantAgent::RegisterForAllFrames(
+ shell()->web_contents(), &autofill_assistant_agent_);
- annotate_dom_model_service_ = std::make_unique<AnnotateDomModelService>(
- /* opt_guide= */ nullptr, /* background_task_runner= */ nullptr);
web_controller_ = WebController::CreateForWebContents(
shell()->web_contents(), &user_data_, &log_info_,
- annotate_dom_model_service_.get(), /*enable_full_stack_traces= */ true);
+ /* annotate_dom_model_service= */ nullptr,
+ /*enable_full_stack_traces= */ true);
Observe(shell()->web_contents());
}
@@ -601,11 +560,11 @@ class WebControllerBrowserTest : public autofill_assistant::BaseBrowserTest,
void CheckFindElementResult(const ElementFinderResult& result,
bool is_main_frame) {
if (is_main_frame) {
- EXPECT_EQ(shell()->web_contents()->GetMainFrame(),
+ EXPECT_EQ(shell()->web_contents()->GetPrimaryMainFrame(),
result.render_frame_host());
EXPECT_EQ(result.frame_stack().size(), 0u);
} else {
- EXPECT_NE(shell()->web_contents()->GetMainFrame(),
+ EXPECT_NE(shell()->web_contents()->GetPrimaryMainFrame(),
result.render_frame_host());
EXPECT_GE(result.frame_stack().size(), 1u);
}
@@ -983,38 +942,21 @@ document.getElementById("overlay_in_frame").style.visibility='hidden';
return ClientStatus(captured_processed_actions[0].status());
}
- int GetBackendNodeId(Selector selector, ClientStatus* status_out) {
- std::unique_ptr<ElementFinderResult> element_result;
- int backend_node_id = -1;
-
- base::RunLoop run_loop_1;
- web_controller_->FindElement(
- selector, true,
- base::BindLambdaForTesting(
- [&](const ClientStatus& status,
- std::unique_ptr<ElementFinderResult> result) {
- element_result = std::move(result);
- *status_out = status;
- run_loop_1.Quit();
- }));
- run_loop_1.Run();
- if (!status_out->ok()) {
- return backend_node_id;
- }
+ ClientStatus GetBackendNodeId(const ElementFinderResult& element,
+ int* backend_node_id) {
+ ClientStatus result_status;
- // Second part in sequence, lookup backend node id.
- base::RunLoop run_loop_2;
+ base::RunLoop run_loop;
web_controller_->GetBackendNodeId(
- *element_result,
+ element,
base::BindLambdaForTesting([&](const ClientStatus& status, int id) {
- *status_out = status;
- backend_node_id = id;
- run_loop_2.Quit();
+ result_status = status;
+ *backend_node_id = id;
+ run_loop.Quit();
}));
- run_loop_2.Run();
+ run_loop.Run();
- log_info_.Clear();
- return backend_node_id;
+ return result_status;
}
protected:
@@ -1023,7 +965,6 @@ document.getElementById("overlay_in_frame").style.visibility='hidden';
UserModel user_model_;
ProcessedActionStatusDetailsProto log_info_;
MockAutofillAssistantAgent autofill_assistant_agent_;
- std::unique_ptr<AnnotateDomModelService> annotate_dom_model_service_;
};
IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, ElementExistenceCheck) {
@@ -2761,7 +2702,7 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
// This makes the devtools action fail.
ElementFinderResult element;
element.SetNodeFrameId("doesnotexist");
- element.SetRenderFrameHost(web_contents()->GetMainFrame());
+ element.SetRenderFrameHost(web_contents()->GetPrimaryMainFrame());
EXPECT_EQ(ELEMENT_POSITION_NOT_FOUND,
WaitUntilElementIsStable(element, 10, base::Milliseconds(100))
@@ -3151,46 +3092,6 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
EXPECT_EQ(status.proto_status(), ACTION_APPLIED);
}
-IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, WaitForDomForSemanticElement) {
- // This element is unique.
- SelectorProto baseline_selector = ToSelectorProto("#select");
-
- ClientStatus element_status;
- int backend_node_id =
- GetBackendNodeId(Selector(baseline_selector), &element_status);
- EXPECT_TRUE(element_status.ok());
-
- NodeData node_data;
- node_data.backend_node_id = backend_node_id;
- EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, false, base::Milliseconds(5000), _))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data}))
- // Capture any other frames.
- .WillRepeatedly(RunOnceCallback<4>(
- mojom::NodeDataStatus::kUnexpectedError, std::vector<NodeData>()));
-
- ActionProto action_proto;
- auto* wait_for_dom = action_proto.mutable_wait_for_dom();
- auto* condition = wait_for_dom->mutable_wait_condition();
- condition->mutable_client_id()->set_identifier("e");
- condition->set_require_unique_element(true);
- auto* semantic_information =
- condition->mutable_match()->mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
-
- base::MockCallback<base::OnceCallback<void(ScriptExecutor*)>>
- run_expectations;
- EXPECT_CALL(run_expectations, Run(_))
- .WillOnce([](ScriptExecutor* script_executor) {
- EXPECT_TRUE(script_executor->GetElementStore()->HasElement("e"));
- });
- ClientStatus status = RunWaitForDom(action_proto, /* use_observers= */ false,
- run_expectations.Get());
- EXPECT_EQ(status.proto_status(), ACTION_APPLIED);
-}
-
IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, FindElementError) {
ClientStatus element_status;
ElementFinderResult element;
@@ -3224,7 +3125,7 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
base::RunLoop button_run_loop;
web_controller_->RunElementFinder(
frame_element, Selector({"#shadowsection", "#shadowbutton"}),
- ElementFinder::ResultType::kExactlyOneMatch,
+ ElementFinderResultType::kExactlyOneMatch,
base::BindOnce(&WebControllerBrowserTest::OnFindElement,
base::Unretained(this), button_run_loop.QuitClosure(),
&button_status, &button_element));
@@ -3262,7 +3163,7 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, RunElementFinderFromOOPIF) {
base::RunLoop button_run_loop;
web_controller_->RunElementFinder(
fake_frame_element, Selector({"#button"}),
- ElementFinder::ResultType::kExactlyOneMatch,
+ ElementFinderResultType::kExactlyOneMatch,
base::BindOnce(&WebControllerBrowserTest::OnFindElement,
base::Unretained(this), button_run_loop.QuitClosure(),
&button_status, &button_element));
@@ -3447,186 +3348,100 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, ExecuteJSWithException) {
testing::ElementsAre(18, 12, 10));
}
-IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
- ElementExistenceCheckWithSemanticModel) {
- ClientStatus status;
- int backend_node_id = GetBackendNodeId(Selector({"#button"}), &status);
- EXPECT_TRUE(status.ok());
-
- NodeData node_data;
- node_data.backend_node_id = backend_node_id;
- EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, false, base::Milliseconds(5000), _))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data}))
- // Capture any other frames.
- .WillRepeatedly(RunOnceCallback<4>(
- mojom::NodeDataStatus::kUnexpectedError, std::vector<NodeData>()));
-
- // We pretend that the button is the correct element.
+IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, ParentFilter) {
SelectorProto proto;
- auto* semantic_information = proto.mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
- RunStrictElementCheck(Selector(proto), true);
-
- ASSERT_EQ(log_info_.element_finder_info().size(), 1);
- const auto& result =
- log_info_.element_finder_info(0).semantic_inference_result();
- ASSERT_EQ(1, result.predicted_elements().size());
- EXPECT_EQ(backend_node_id, result.predicted_elements(0).backend_node_id());
- EXPECT_THAT(
- 1, result.predicted_elements(0).semantic_information().semantic_role());
- EXPECT_THAT(2,
- result.predicted_elements(0).semantic_information().objective());
-}
+ proto.add_filters()->set_css_selector("#select option:checked");
+ proto.add_filters()->mutable_parent();
-IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
- ElementExistenceCheckWithSemanticModelOOPIF) {
- ClientStatus status;
- int backend_node_id =
- GetBackendNodeId(Selector({"#iframeExternal", "#button"}), &status);
- EXPECT_TRUE(status.ok());
+ std::string element_tag;
+ EXPECT_EQ(ACTION_APPLIED,
+ GetElementTag(Selector(proto), &element_tag).proto_status());
+ EXPECT_EQ("SELECT", element_tag);
- NodeData node_data;
- node_data.backend_node_id = backend_node_id;
- EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, false, base::Milliseconds(5000), _))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data}))
- // Capture any other frames.
- .WillRepeatedly(RunOnceCallback<4>(
- mojom::NodeDataStatus::kUnexpectedError, std::vector<NodeData>()));
-
- // We pretend that the button is the correct element.
- SelectorProto proto;
- auto* semantic_information = proto.mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
- RunStrictElementCheck(Selector(proto), true);
+ SelectorProto failing_proto;
+ failing_proto.add_filters()->set_css_selector("body");
+ failing_proto.add_filters()->mutable_parent(); // document
+ failing_proto.add_filters()->mutable_parent(); // Nothing
- ASSERT_EQ(log_info_.element_finder_info().size(), 1);
- const auto& result =
- log_info_.element_finder_info(0).semantic_inference_result();
- ASSERT_EQ(1, result.predicted_elements().size());
- EXPECT_EQ(backend_node_id, result.predicted_elements(0).backend_node_id());
- EXPECT_THAT(
- 1, result.predicted_elements(0).semantic_information().semantic_role());
- EXPECT_THAT(2,
- result.predicted_elements(0).semantic_information().objective());
+ ClientStatus failing_status;
+ ElementFinderResult ignored_element;
+ FindElement(Selector(failing_proto), &failing_status, &ignored_element);
+ EXPECT_EQ(ELEMENT_RESOLUTION_FAILED, failing_status.proto_status());
}
-IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
- ElementExistenceCheckWithSemanticModelNotFound) {
- // All frames return an empty list as a result.
- EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, false, base::Milliseconds(5000), _))
- .WillRepeatedly(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{}));
+IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, WebpageZoom) {
+ double initial_width =
+ content::EvalJs(
+ shell(),
+ R"(document.querySelector("#select").getBoundingClientRect().width)")
+ .ExtractDouble();
+ EXPECT_GT(initial_width, 0);
- SelectorProto proto;
- auto* semantic_information = proto.mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
- FindElementExpectEmptyResult(Selector(proto));
-}
+ ClientStatus body_status;
+ ElementFinderResult body;
+ FindElement(Selector({"body"}), &body_status, &body);
+ EXPECT_EQ(ACTION_APPLIED, body_status.proto_status());
-IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
- ElementExistenceCheckWithSemanticMultipleFound) {
- SelectorProto proto;
- auto* semantic_information = proto.mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
-
- NodeData node_data;
- node_data.backend_node_id = 5;
- NodeData node_data_other;
- node_data_other.backend_node_id = 13;
- EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, false, base::Milliseconds(5000), _))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data}))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data_other}))
- // Capture any other frames.
- .WillRepeatedly(RunOnceCallback<4>(
- mojom::NodeDataStatus::kUnexpectedError, std::vector<NodeData>()));
-
- // Two elements are found in different frames.
- ClientStatus status;
- FindElement(Selector(proto), &status, nullptr);
- EXPECT_EQ(TOO_MANY_ELEMENTS, status.proto_status());
-}
+ ClientStatus zoom_status;
+ base::RunLoop zoom_run_loop;
+ web_controller_->ExecuteJS(
+ "this.style.webkitTransform = 'scale(2)'", body,
+ base::BindOnce(&WebControllerBrowserTest::OnClientStatus,
+ base::Unretained(this), zoom_run_loop.QuitClosure(),
+ &zoom_status));
+ zoom_run_loop.Run();
+ EXPECT_EQ(ACTION_APPLIED, zoom_status.proto_status());
-IN_PROC_BROWSER_TEST_F(
- WebControllerBrowserTest,
- ElementExistenceCheckWithSemanticModelUsesIgnoreObjective) {
- NodeData node_data;
- node_data.backend_node_id = 5;
- EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, true, base::Milliseconds(5000), _))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data}))
- .WillRepeatedly(RunOnceCallback<4>(
- mojom::NodeDataStatus::kUnexpectedError, std::vector<NodeData>()));
+ double after_zoom_width =
+ content::EvalJs(
+ shell(),
+ R"(document.querySelector("#select").getBoundingClientRect().width)")
+ .ExtractDouble();
+ EXPECT_NEAR(after_zoom_width, initial_width * 2, 1);
- SelectorProto proto;
- auto* semantic_information = proto.mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
- // All we want is this to be propagated to the GetSemanticNodes call as
- // configured in the previous expectation.
- semantic_information->set_ignore_objective(true);
-
- ClientStatus ignore_status;
- FindElement(Selector(proto), &ignore_status, nullptr);
-
- // TODO(b/217160707): For now we expect the originally passed in semantic info
- // to be logged instead of the objective inferred by the model.
- ASSERT_EQ(log_info_.element_finder_info().size(), 1);
- const auto& result =
- log_info_.element_finder_info(0).semantic_inference_result();
- ASSERT_EQ(1, result.predicted_elements().size());
- EXPECT_EQ(5, result.predicted_elements(0).backend_node_id());
- EXPECT_THAT(
- 1, result.predicted_elements(0).semantic_information().semantic_role());
- EXPECT_THAT(2,
- result.predicted_elements(0).semantic_information().objective());
+ ClientStatus reset_status;
+ base::RunLoop reset_run_loop;
+ web_controller_->ExecuteJS(
+ "this.style.webkitTransform = 'scale(1)'", body,
+ base::BindOnce(&WebControllerBrowserTest::OnClientStatus,
+ base::Unretained(this), reset_run_loop.QuitClosure(),
+ &reset_status));
+ reset_run_loop.Run();
+ EXPECT_EQ(ACTION_APPLIED, reset_status.proto_status());
+
+ double after_reset_width =
+ content::EvalJs(
+ shell(),
+ R"(document.querySelector("#select").getBoundingClientRect().width)")
+ .ExtractDouble();
+ EXPECT_NEAR(after_reset_width, initial_width, 1);
}
-IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, SemanticAndCssComparison) {
- ClientStatus status;
- int backend_node_id = GetBackendNodeId(Selector({"#button"}), &status);
- EXPECT_TRUE(status.ok());
+IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, SetFieldValueThroughNative) {
+ ClientStatus element_status;
+ ElementFinderResult input;
+ FindElement(Selector({"#input1"}), &element_status, &input);
+ ASSERT_EQ(ACTION_APPLIED, element_status.proto_status());
- NodeData node_data;
- node_data.backend_node_id = backend_node_id;
+ int backend_node_id;
+ ASSERT_EQ(ACTION_APPLIED,
+ GetBackendNodeId(input, &backend_node_id).proto_status());
+ std::u16string expected_value = u"native";
EXPECT_CALL(autofill_assistant_agent_,
- GetSemanticNodes(1, 2, false, base::Milliseconds(5000), _))
- .WillOnce(RunOnceCallback<4>(mojom::NodeDataStatus::kSuccess,
- std::vector<NodeData>{node_data}))
- // Capture any other frames.
- .WillRepeatedly(RunOnceCallback<4>(
- mojom::NodeDataStatus::kUnexpectedError, std::vector<NodeData>()));
-
- // We pretend that the button is the correct element.
- SelectorProto proto = ToSelectorProto("#button");
- auto* semantic_information = proto.mutable_semantic_information();
- semantic_information->set_semantic_role(1);
- semantic_information->set_objective(2);
- semantic_information->set_check_matches_css_element(true);
- RunStrictElementCheck(Selector(proto), true);
+ SetElementValue(backend_node_id, expected_value,
+ /* send_events= */ true, _))
+ .WillOnce(RunOnceCallback<3>(true));
- ASSERT_EQ(log_info_.element_finder_info().size(), 1);
- const auto& result =
- log_info_.element_finder_info(0).semantic_inference_result();
- ASSERT_EQ(1, result.predicted_elements().size());
- EXPECT_EQ(backend_node_id, result.predicted_elements(0).backend_node_id());
- EXPECT_THAT(
- 1, result.predicted_elements(0).semantic_information().semantic_role());
- EXPECT_THAT(2,
- result.predicted_elements(0).semantic_information().objective());
- EXPECT_TRUE(result.predicted_elements(0).matches_css_element());
+ ClientStatus fill_status;
+ base::RunLoop run_loop;
+ web_controller_->SetNativeValue(
+ "native", input,
+ base::BindOnce(&WebControllerBrowserTest::OnClientStatus,
+ base::Unretained(this), run_loop.QuitClosure(),
+ &fill_status));
+ run_loop.Run();
+
+ EXPECT_EQ(ACTION_APPLIED, fill_status.proto_status());
}
} // namespace autofill_assistant
View Lorry Controller Status