1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
// Copyright 2021 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
module handwriting.mojom;
// https://github.com/WICG/handwriting-recognition/blob/main/explainer.md
// CrOS's handwriting service's API, defined in
// //chromeos/services/machine_leaning/public/mojom and mirrored into platform2,
// is largely derived from the types and interfaces defined below.
//
// The feature query APIs are not plumbed through, but otherwise, new
// fields/methods likely need to be plumbed through the corresponding CrOS APIs
// as well.
//
// Any changes in overlapping part may need to be kept synced in all these
// three places,
// 1. "chromeos/services/machine_learning/public/mojom/web_platform_handwriting.mojom"
// in chrome repo.
// 2. "platform2/ml/mojom/web_platform_handwriting.mojom" in CrOS repo.
// 3. This file.
import "mojo/public/mojom/base/time.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";
// Represents a single point in a handwriting stroke.
// Corresponds to handwriting_point.idl.
struct HandwritingPoint {
// Represent the horizontal (location.x) and vertical (location.y) location
// of the point.
// The top-left corner coordinate is (location.x=0, location.y=0).
gfx.mojom.PointF location;
// The time elapsed since the starting time (e.g. when the first ink point
// of the drawing is captured).
mojo_base.mojom.TimeDelta? t;
};
// Represents a stroke which is just a series of points.
// Corresponds to handwriting_stroke.idl.
struct HandwritingStroke {
array<HandwritingPoint> points;
};
// Represents a segment of a handwriting stroke in the grapheme detected.
// One `HandwritingDrawingSegment` can only refer one stroke, denoted by
// `stroke_index` which is the index of the stroke in the input stroke arrays
// (i.e., the first parameter of the `HandwritingRecognizer::GetPrediction`
// function).
// The reason we need this struct is that different parts of one single stroke
// can belong to different grapheme detected.
// Corresponds to handwriting_drawing_segment.idl.
struct HandwritingDrawingSegment {
// The index of the corresponding stroke in the input stroke array.
uint32 stroke_index;
// The index of the first point in the stroke that belongs to this drawing
// segment.
uint32 begin_point_index;
// The index of the last point in the stroke that belongs to this drawing
// segment.
uint32 end_point_index;
};
// Represents a segment detected.
// Corresponds to handwriting_segment.idl.
struct HandwritingSegment {
// The string representation of this grapheme.
string grapheme;
// HandwritingPrediction.text.slice(begin_index, end_index) === grapheme
// If the grapheme spans multiple Unicode code points,
// `end_index - begin_index` is greater than 1.
uint32 begin_index;
uint32 end_index;
array<HandwritingDrawingSegment> drawing_segments;
};
// Represents one single prediction result.
// The final prediction output is an array of these.
// Corresponds to handwriting_prediction.idl.
struct HandwritingPrediction {
string text;
array<HandwritingSegment> segmentation_result;
};
// Represents the hints provided to the recognizer for better performance.
// Corresponds to handwriting_hints.idl.
struct HandwritingHints {
// The type of content to be recognized. The recognizer may use these to
// better rank the recognition results. (e.g. "text", "email", "number",
// "per-character").
string recognition_type;
// Identifies how the strokes are captured. (e.g. "touch", "mouse", "pen")
string input_type;
// The text that comes before the handwriting. This can be texts that were
// previously recognized, or were given as the writing context (e.g.
// "Write your name here:"). This is the linguistic context to help
// disambiguate the handwriting (e.g. “Hello world” vs. “Hello word”).
string text_context;
// The maximum number of alternative predictions to generate.
uint32 alternatives;
};
// Represents the struct for query features supported by the backend handwriting
// recognizer.
// Corresponds to handwriting_feature_query.idl.
struct HandwritingFeatureQuery {
// If non-empty, queries whether the languages are supported.
// Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
array<string> languages;
// Whether the "alternative" feature is queried.
bool alternatives;
// Whether the "segmentation" feature is queried.
bool segmentation_result;
};
// Because "bool" is not nullable, we need to define an enum to present the
// status. And this will also be helpful when we need to extend it, for
// example, adding `kNeedDownload`.
// Add `kNotQueried` to make it possible to use `mojo::Converter` to convert
// `HandwritingFeatureQueryResult` types (otherwise, there is no way to know
// whether a feature is queried or not in the converter function.
enum HandwritingFeatureStatus {
kNotQueried,
kSupported,
kNotSupported,
};
// The feature query result.
// Corresponds to handwriting_feature_query_result.idl.
struct HandwritingFeatureQueryResult {
HandwritingFeatureStatus languages = kNotQueried;
HandwritingFeatureStatus alternatives = kNotQueried;
HandwritingFeatureStatus segmentation_result = kNotQueried;
};
// Used in creating recognizer.
// Corresponds to handwriting_model_constraint.idl.
struct HandwritingModelConstraint {
// Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
array<string> languages;
};
// Interface for a renderer to use a specific handwriting recognition backend.
// The browser handles the requests and forwards them to the appropriate
// backend.
interface HandwritingRecognizer {
// Does the recognition and outputs the prediction result.
// This is used by IDL API `blink::HandwritingDrawing::getPrediction`.
// The input `strokes` and `hints` should both come from
// `blink::HandwritingDrawing`.
// If the returned `Optional` has no value, it means there is some error in
// recognition. If the returned `Optional` has value but the array is empty,
// it means the recognizer can not recognize anything from the input.
GetPrediction(array<HandwritingStroke> strokes, HandwritingHints hints)
=> (array<HandwritingPrediction>? prediction);
};
// Per-document interface that allows the renderer to ask the browser for
// a specific handwriting recognizer backend, query capabilities, et cetera.
// Corresponds to `navigator_handwriting_recognition_service.idl`.
interface HandwritingRecognitionService {
// Creates a recognizer.
// This is used by IDL API
// `blink::HandwritingRecognitionService::createHandwritingRecognizer`.
CreateHandwritingRecognizer(HandwritingModelConstraint constraint)
=> (CreateHandwritingRecognizerResult result,
pending_remote<HandwritingRecognizer>? handwriting_recognizer);
// Queries the features supported by the platform.
// This is used by IDL API
// `blink::HandwritingRecognitionService::queryHandwritingRecognizerSupport`.
QueryHandwritingRecognizerSupport(HandwritingFeatureQuery query)
=> (HandwritingFeatureQueryResult? result);
};
enum CreateHandwritingRecognizerResult {
kOk,
kError,
// The provided model constraint can't be satisfied (e.g. the language is
// not supported).
kNotSupported,
};
|
