// Copyright (c) 2016 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "content/common/input/actions_parser.h" #include #include "base/format_macros.h" #include "base/strings/string_split.h" #include "base/strings/stringprintf.h" #include "base/values.h" #include "components/viz/common/frame_sinks/begin_frame_args.h" #include "content/common/input/input_injector.mojom.h" #include "content/common/input/synthetic_smooth_scroll_gesture_params.h" #include "ui/events/types/scroll_types.h" using Button = content::SyntheticPointerActionParams::Button; using PointerActionType = content::SyntheticPointerActionParams::PointerActionType; namespace content { namespace { PointerActionType ToSyntheticPointerActionType(const std::string& action_type) { if (action_type == "pointerDown") return PointerActionType::PRESS; if (action_type == "pointerMove") return PointerActionType::MOVE; if (action_type == "pointerUp") return PointerActionType::RELEASE; if (action_type == "pointerLeave") return PointerActionType::LEAVE; if (action_type == "pause") return PointerActionType::IDLE; return PointerActionType::NOT_INITIALIZED; } content::mojom::GestureSourceType ToSyntheticGestureSourceType( const std::string& pointer_type) { if (pointer_type == "touch") return content::mojom::GestureSourceType::kTouchInput; else if (pointer_type == "mouse") return content::mojom::GestureSourceType::kMouseInput; else if (pointer_type == "pen") return content::mojom::GestureSourceType::kPenInput; return content::mojom::GestureSourceType::kDefaultInput; } Button ToSyntheticMouseButton(int button) { if (button == 0) return Button::LEFT; if (button == 1) return Button::MIDDLE; if (button == 2) return Button::RIGHT; if (button == 3) return Button::BACK; if (button == 4) return Button::FORWARD; NOTREACHED() << "Unexpected button"; return Button(); } int ToKeyModifiers(const std::string& key) { if (key == "Alt") return blink::WebInputEvent::kAltKey; if (key == "Control") return blink::WebInputEvent::kControlKey; if (key == "Meta") return blink::WebInputEvent::kMetaKey; if (key == "Shift") return blink::WebInputEvent::kShiftKey; if (key == "CapsLock") return blink::WebInputEvent::kCapsLockOn; if (key == "NumLock") return blink::WebInputEvent::kNumLockOn; if (key == "AltGraph") return blink::WebInputEvent::kAltGrKey; return 0; } } // namespace ActionsParser::ActionsParser(base::Value action_sequence_list) : action_sequence_list_(std::move(action_sequence_list)) { // We have two different JSON formats from testdriver Action API or // gpuBenchmarking.pointerActionSequence. Below we are deciding where the // action sequence list comes from. if (action_sequence_list_.is_list() && action_sequence_list_.GetListDeprecated().size() > 0) { use_testdriver_api_ = ActionsDictionaryUsesTestDriverApi( action_sequence_list_.GetListDeprecated()[0]); } } ActionsParser::~ActionsParser() {} bool ActionsParser::Parse() { if (!action_sequence_list_.is_list() || action_sequence_list_.GetListDeprecated().size() == 0) { error_message_ = std::string("provided action sequence list is not a list or is empty"); return false; } for (const auto& action_sequence : action_sequence_list_.GetListDeprecated()) { if (!action_sequence.is_dict()) { error_message_ = std::string("Expected ActionSequence is not a dictionary"); return false; } if (use_testdriver_api_) { if (!ParseTestDriverActionSequence(action_sequence)) return false; } else { if (!ParseGpuBenchmarkingActionSequence(action_sequence)) return false; } } if (source_type_ == "wheel") return true; gesture_params_ = std::make_unique(); SyntheticPointerActionListParams* pointer_actions = static_cast(gesture_params_.get()); pointer_actions->gesture_source_type = ToSyntheticGestureSourceType(pointer_type_); // Group a list of actions from all pointers into a // SyntheticPointerActionListParams object, which is a list of actions, which // will be dispatched together. for (size_t index = 0; index < longest_action_sequence_; ++index) { SyntheticPointerActionListParams::ParamList param_list; size_t longest_pause_frame = 0; for (const auto& pointer_action_list : pointer_actions_lists_) { if (index < pointer_action_list.size()) { param_list.push_back(pointer_action_list[index]); if (pointer_action_list[index].pointer_action_type() == PointerActionType::IDLE) { size_t num_pause_frame = static_cast(std::ceil( pointer_action_list[index].duration().InMilliseconds() / viz::BeginFrameArgs::DefaultInterval().InMilliseconds())); longest_pause_frame = std::max(longest_pause_frame, num_pause_frame); } } } pointer_actions->PushPointerActionParamsList(param_list); for (size_t pause_index = 1; pause_index < longest_pause_frame; ++pause_index) { SyntheticPointerActionListParams::ParamList pause_param_list; SyntheticPointerActionParams pause_action_param(PointerActionType::IDLE); for (size_t i = 0; i < param_list.size(); ++i) { pause_param_list.push_back(pause_action_param); } pointer_actions->PushPointerActionParamsList(pause_param_list); } } return true; } bool ActionsParser::ActionsDictionaryUsesTestDriverApi( const base::Value& action_sequence) { // If the JSON format of each action_sequence has "type" element, it is from // the new Action API, otherwise it is from // gpuBenchmarking.pointerActionSequence API. We have to keep both formats // for now, but later on once we switch to the new Action API in all tests, // we will remove the old format. if (action_sequence.FindKey("type")) return true; return false; } bool ActionsParser::ParseGpuBenchmarkingActionSequence( const base::Value& action_sequence) { // The GpuBenchmarking format is implicitly for pointers only and for // historic reasons, the "source" key refers to what TestDriver calls the // pointer_type_. source_type_ = "pointer"; if (use_testdriver_api_ != ActionsDictionaryUsesTestDriverApi(action_sequence)) { error_message_ = std::string( "all action sequences must be of the same gpuBenchmarking format"); return false; } const std::string* pointer_type = action_sequence.FindStringKey("source"); if (!pointer_type) { error_message_ = std::string("source type is not defined or not a string"); return false; } if (*pointer_type != "touch" && *pointer_type != "mouse" && *pointer_type != "pen") { error_message_ = base::StringPrintf( "source type %s is an unsupported input type", (*pointer_type).c_str()); return false; } if (pointer_type_.empty()) { pointer_type_ = *pointer_type; } else if (pointer_type_ != *pointer_type) { error_message_ = std::string("currently multiple input types are not not supported"); return false; } if (*pointer_type != "touch" && input_source_count_ > 0) { error_message_ = std::string( "for source type of mouse and pen, we only support one device " "in one sequence"); return false; } const base::Value* actions = action_sequence.FindKeyOfType("actions", base::Value::Type::LIST); if (!actions) { error_message_ = base::StringPrintf( "action_sequence[%zu].actions is not defined or not a list", action_index_); return false; } else if (actions->GetListDeprecated().size() == 0) { error_message_ = base::StringPrintf( "action_sequence[%zu].actions is an empty list", action_index_); return false; } if (!ParseActionItemList(*actions, "pointer")) return false; input_source_count_++; return true; } bool ActionsParser::ParseTestDriverActionSequence( const base::Value& action_sequence) { if (use_testdriver_api_ != ActionsDictionaryUsesTestDriverApi(action_sequence)) { error_message_ = std::string( "all action sequences must be of the same TestDriver format"); return false; } const std::string* source_type = action_sequence.FindStringKey("type"); if (!source_type) { error_message_ = std::string("input source type is not defined or not a string"); return false; } if (*source_type != "none") { if (source_type_.empty()) { source_type_ = *source_type; } else if (source_type_ != *source_type) { error_message_ = std::string( "currently multiple input source types are not supported"); return false; } } if (*source_type == "pointer") { if (!ParsePointerParameters(action_sequence)) return false; } else if (*source_type == "key") { error_message_ = std::string("we do not support action sequence type of key"); return false; } else if (*source_type == "wheel") { // do nothing } else if (*source_type == "none") { // do nothing } else { error_message_ = base::StringPrintf("the input source type %s is invalid", (*source_type).c_str()); return false; } const base::Value* actions = action_sequence.FindKeyOfType("actions", base::Value::Type::LIST); if (!actions) { error_message_ = base::StringPrintf( "action_sequence[%zu].actions is not defined or not a list", action_index_); return false; } else if (actions->GetListDeprecated().size() == 0) { error_message_ = base::StringPrintf( "action_sequence[%zu].actions is an empty list", action_index_); return false; } else if (*source_type == "wheel" && actions->GetListDeprecated().size() > 1) { error_message_ = base::StringPrintf( "action_sequence[%zu].actions should only have one action for the " "wheel input source", action_index_); return false; } if (!ParseActionItemList(*actions, *source_type)) return false; if (*source_type != "none") input_source_count_++; return true; } bool ActionsParser::ParsePointerParameters(const base::Value& action_sequence) { const base::Value* parameters = action_sequence.FindKey("parameters"); // The default pointer type is mouse. std::string pointer_type = "mouse"; if (parameters) { if (!parameters->is_dict()) { error_message_ = std::string("action sequence parameters is not a dictionary"); return false; } const std::string* pointer_type_value = parameters->FindStringKey("pointerType"); if (!pointer_type_value) { error_message_ = std::string( "action sequence pointer type is not defined or not a string"); return false; } if (*pointer_type_value != "touch" && *pointer_type_value != "mouse" && *pointer_type_value != "pen") { error_message_ = base::StringPrintf( "action sequence pointer type %s is an unsupported input type", (*pointer_type_value).c_str()); return false; } pointer_type = *pointer_type_value; } if (pointer_type_.empty()) { pointer_type_ = pointer_type; } else if (pointer_type_ != pointer_type) { error_message_ = std::string( "currently multiple action sequence pointer type are not " "supported"); return false; } if (pointer_type != "touch" && input_source_count_ > 0) { error_message_ = std::string( "for input type of mouse and pen, we only support one device"); return false; } // TODO(lanwei): according to the Webdriver spec, "Let id be the result of // getting the property id from action sequence.", we should move "id" from // parameters dictionary to action sequence. const std::string* pointer_name = action_sequence.FindStringKey("id"); if (!pointer_name) { error_message_ = std::string("pointer name is not defined or not a string"); return false; } if (pointer_name_set_.find(*pointer_name) != pointer_name_set_.end()) { error_message_ = std::string("pointer name already exists"); return false; } pointer_name_set_.insert(*pointer_name); return true; } bool ActionsParser::ParseActionItemList(const base::Value& actions, std::string source_type) { DCHECK(source_type == "none" || source_type == source_type_); SyntheticPointerActionListParams::ParamList param_list; for (const auto& action : actions.GetListDeprecated()) { if (!action.is_dict()) { error_message_ = base::StringPrintf( "actions[%zu].actions is not defined or not a dictionary", action_index_); return false; } else if (!ParseAction(action, param_list, source_type)) { return false; } } if (param_list.size() > longest_action_sequence_) longest_action_sequence_ = param_list.size(); pointer_actions_lists_.push_back(param_list); return true; } bool ActionsParser::ParseAction( const base::Value& action, SyntheticPointerActionListParams::ParamList& param_list, std::string source_type) { std::string subtype; if (use_testdriver_api_) { const std::string* type_value = action.FindStringKey("type"); if (!type_value) { error_message_ = base::StringPrintf( "actions[%zu].actions.type is not defined or not a string", action_index_); return false; } subtype = *type_value; } else { const std::string* name_value = action.FindStringKey("name"); if (!name_value) { error_message_ = base::StringPrintf( "actions[%zu].actions.name is not defined or not a string", action_index_); return false; } subtype = *name_value; } if (source_type == "wheel") { return ParseWheelAction(action, subtype); } else if (source_type == "pointer") { return ParsePointerAction(action, subtype, param_list); } else if (source_type == "none") { return ParseNullAction(action, subtype, param_list); } else { NOTREACHED(); } return false; } bool ActionsParser::ParseWheelAction(const base::Value& action, std::string subtype) { if (subtype == "pause") { error_message_ = base::StringPrintf( "actions[%zu].actions.type of pause is not supported now", action_index_); return false; } else if (subtype != "scroll") { error_message_ = base::StringPrintf( "actions[%zu].actions.type is not scroll or pause when source type is " "wheel", action_index_); return false; } double position_x = 0; double position_y = 0; if (!GetPosition(action, position_x, position_y)) return false; int delta_x = 0; int delta_y = 0; if (!GetScrollDelta(action, delta_x, delta_y)) return false; gesture_params_ = std::make_unique(); SyntheticSmoothScrollGestureParams* scroll = static_cast(gesture_params_.get()); scroll->gesture_source_type = content::mojom::GestureSourceType::kMouseInput; scroll->speed_in_pixels_s = 8000; scroll->prevent_fling = true; scroll->granularity = ui::ScrollGranularity::kScrollByPrecisePixel; scroll->anchor.SetPoint(position_x, position_y); scroll->fling_velocity_x = 0; scroll->fling_velocity_y = 0; scroll->distances.push_back(-gfx::Vector2dF(delta_x, delta_y)); scroll->modifiers = 0; return true; } bool ActionsParser::ParsePointerAction( const base::Value& action, std::string subtype, SyntheticPointerActionListParams::ParamList& param_list) { double position_x = 0; double position_y = 0; if ((subtype == "pointerDown" || subtype == "pointerMove") && !GetPosition(action, position_x, position_y)) { return false; } PointerActionType pointer_action_type = PointerActionType::NOT_INITIALIZED; pointer_action_type = ToSyntheticPointerActionType(subtype); if (pointer_action_type == PointerActionType::NOT_INITIALIZED) { error_message_ = base::StringPrintf( "actions[%zu].actions.name is an unsupported action name", action_index_); return false; } Button button = pointer_action_type == PointerActionType::MOVE ? Button::NO_BUTTON : Button::LEFT; const base::Value* button_id_value = action.FindKey("button"); if (button_id_value) { if (!button_id_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.button is not an integer", action_index_); return false; } int button_id = button_id_value->GetInt(); if (button_id < 0 || button_id > 4) { error_message_ = base::StringPrintf( "actions[%zu].actions.button is an unsupported button", action_index_); return false; } button = ToSyntheticMouseButton(button_id); } std::string keys; const base::Value* keys_value = action.FindKey("keys"); if (keys_value) { if (!keys_value->is_string()) { error_message_ = base::StringPrintf( "actions[%zu].actions.key is not a string", action_index_); return false; } keys = keys_value->GetString(); } int key_modifiers = 0; std::vector key_list = base::SplitString(keys, ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); for (std::string& key : key_list) { int key_modifier = ToKeyModifiers(key); if (key_modifier == 0) { error_message_ = base::StringPrintf( "actions[%zu].actions.key is not a valid key", action_index_); return false; } key_modifiers |= key_modifier; } double width = 40; const base::Value* width_value = action.FindKey("width"); if (width_value) { width = width_value->GetDouble(); if (width < 0) { error_message_ = base::StringPrintf( "actions[%zu].actions.width should not be negative", action_index_); return false; } } double height = 40; const base::Value* height_value = action.FindKey("height"); if (height_value) { height = height_value->GetDouble(); if (height < 0) { error_message_ = base::StringPrintf( "actions[%zu].actions.height should not be negative", action_index_); return false; } } double pressure = 0.5; const base::Value* pressure_value = action.FindKey("pressure"); if (pressure_value) { pressure = pressure_value->GetDouble(); if (pressure < 0 || pressure > 1) { error_message_ = base::StringPrintf( "actions[%zu].actions.pressure must be a non-negative number in the " "range of [0,1]", action_index_); return false; } } double tangential_pressure = 0; const base::Value* tangential_pressure_value = action.FindKey("tangentialPressure"); if (tangential_pressure_value) { tangential_pressure = tangential_pressure_value->GetDouble(); if (tangential_pressure < -1 || tangential_pressure > 1) { error_message_ = base::StringPrintf( "actions[%zu].actions.tangentialPressure must be a non-negative " "number in the range of [-1,1]", action_index_); return false; } } int tilt_x = 0; const base::Value* tilt_x_value = action.FindKey("tiltX"); if (tilt_x_value) { if (!tilt_x_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.tiltX is not an integer", action_index_); return false; } tilt_x = tilt_x_value->GetInt(); if (tilt_x < -90 || tilt_x > 90) { error_message_ = base::StringPrintf( "actions[%zu].actions.tiltX must be an integer in the range of " "[-90,90]", action_index_); return false; } } int tilt_y = 0; const base::Value* tilt_y_value = action.FindKey("tiltY"); if (tilt_y_value) { if (!tilt_y_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.tiltY is not an integer", action_index_); return false; } tilt_y = tilt_y_value->GetInt(); if (tilt_y < -90 || tilt_y > 90) { error_message_ = base::StringPrintf( "actions[%zu].actions.tiltY must be an integer in the range of " "[-90,90]", action_index_); return false; } } int twist = 0; const base::Value* twist_value = action.FindKey("twist"); if (twist_value) { if (!twist_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.twist is not an integer", action_index_); return false; } twist = twist_value->GetInt(); if (twist < 0 || twist > 359) { error_message_ = base::StringPrintf( "actions[%zu].actions.twist must be an integer in the range of " "[0,359]", action_index_); return false; } } int duration = viz::BeginFrameArgs::DefaultInterval().InMilliseconds(); if (pointer_action_type == PointerActionType::IDLE && !GetPauseDuration(action, duration)) { return false; } SyntheticPointerActionParams action_param(pointer_action_type); action_param.set_pointer_id(input_source_count_); switch (pointer_action_type) { case PointerActionType::PRESS: action_param.set_position(gfx::PointF(position_x, position_y)); action_param.set_button(button); action_param.set_key_modifiers(key_modifiers); action_param.set_width(width); action_param.set_height(height); action_param.set_force(pressure); action_param.set_tangential_pressure(tangential_pressure); action_param.set_tilt_x(tilt_x); action_param.set_tilt_y(tilt_y); action_param.set_rotation_angle(twist); break; case PointerActionType::MOVE: action_param.set_position(gfx::PointF(position_x, position_y)); action_param.set_key_modifiers(key_modifiers); action_param.set_width(width); action_param.set_height(height); action_param.set_force(pressure); action_param.set_tangential_pressure(tangential_pressure); action_param.set_tilt_x(tilt_x); action_param.set_tilt_y(tilt_y); action_param.set_rotation_angle(twist); action_param.set_button(button); break; case PointerActionType::RELEASE: action_param.set_button(button); action_param.set_key_modifiers(key_modifiers); break; case PointerActionType::IDLE: action_param.set_duration(base::Milliseconds(duration)); break; case PointerActionType::CANCEL: case PointerActionType::LEAVE: case PointerActionType::NOT_INITIALIZED: break; } param_list.push_back(action_param); return true; } bool ActionsParser::ParseNullAction( const base::Value& action, std::string subtype, SyntheticPointerActionListParams::ParamList& param_list) { PointerActionType pointer_action_type = PointerActionType::NOT_INITIALIZED; pointer_action_type = ToSyntheticPointerActionType(subtype); if (pointer_action_type != PointerActionType::IDLE) { error_message_ = base::StringPrintf( "actions[%zu].actions.name should only be pause", action_index_); return false; } int duration = viz::BeginFrameArgs::DefaultInterval().InMilliseconds(); if (!GetPauseDuration(action, duration)) return false; SyntheticPointerActionParams action_param(pointer_action_type); action_param.set_pointer_id(0); action_param.set_duration(base::Milliseconds(duration)); param_list.push_back(action_param); return true; } bool ActionsParser::GetPosition(const base::Value& action, double& position_x, double& position_y) { const base::Value* position_x_value = action.FindKey("x"); const base::Value* position_y_value = action.FindKey("y"); // TODO(lanwei): we should clarify the case when x or y is undefined in the // WebDriver spec. // https://www.w3.org/TR/webdriver/#dfn-process-a-pointer-move-action. if (!position_x_value || (!position_x_value->is_int() && !position_x_value->is_double())) { error_message_ = base::StringPrintf( "actions[%zu].actions.x is not defined or not a number", action_index_); return false; } position_x = position_x_value->GetDouble(); if (!position_y_value || (!position_y_value->is_int() && !position_y_value->is_double())) { error_message_ = base::StringPrintf( "actions[%zu].actions.y is not defined or not a number", action_index_); return false; } position_y = position_y_value->GetDouble(); return true; } bool ActionsParser::GetScrollDelta(const base::Value& action, int& delta_x, int& delta_y) { const base::Value* delta_x_value = action.FindKey("deltaX"); const base::Value* delta_y_value = action.FindKey("deltaY"); if (!delta_x_value || !delta_x_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.delta_x is not defined or not an integer", action_index_); return false; } delta_x = delta_x_value->GetInt(); if (!delta_y_value || !delta_y_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.delta_y is not defined or not an integer", action_index_); return false; } delta_y = delta_y_value->GetInt(); return true; } bool ActionsParser::GetPauseDuration(const base::Value& action, int& duration) { const base::Value* duration_value = action.FindKey("duration"); // TODO(lanwei): we should always have a duration value for pause action. if (duration_value) { if (!duration_value->is_double() && !duration_value->is_int()) { error_message_ = base::StringPrintf( "actions[%zu].actions.duration is not a number", action_index_); return false; } duration = duration_value->GetDouble(); } if (duration < 0) { error_message_ = base::StringPrintf( "actions[%zu].actions.duration should not be negative", action_index_); return false; } return true; } } // namespace content