Add API to the engine to support attributed text (flutter#25373)

ci/licenses_golden/licenses_flutter

@@ -411,6 +411,9 @@ FILE: ../../../flutter/lib/ui/semantics/semantics_update.cc
 FILE: ../../../flutter/lib/ui/semantics/semantics_update.h
 FILE: ../../../flutter/lib/ui/semantics/semantics_update_builder.cc
 FILE: ../../../flutter/lib/ui/semantics/semantics_update_builder.h
+FILE: ../../../flutter/lib/ui/semantics/semantics_update_builder_unittests.cc
+FILE: ../../../flutter/lib/ui/semantics/string_attribute.cc
+FILE: ../../../flutter/lib/ui/semantics/string_attribute.h
 FILE: ../../../flutter/lib/ui/snapshot_delegate.h
 FILE: ../../../flutter/lib/ui/text.dart
 FILE: ../../../flutter/lib/ui/text/asset_manager_font_provider.cc

fml/platform/android/jni_util.cc

@@ -132,14 +132,33 @@ ScopedJavaLocalRef<jobjectArray> VectorToStringArray(
   ScopedJavaLocalRef<jclass> string_clazz(env,
                                           env->FindClass("java/lang/String"));
   FML_DCHECK(!string_clazz.is_null());
-  jobjectArray joa =
+  jobjectArray java_array =
       env->NewObjectArray(vector.size(), string_clazz.obj(), NULL);
   ASSERT_NO_EXCEPTION();
   for (size_t i = 0; i < vector.size(); ++i) {
     ScopedJavaLocalRef<jstring> item = StringToJavaString(env, vector[i]);
-    env->SetObjectArrayElement(joa, i, item.obj());
+    env->SetObjectArrayElement(java_array, i, item.obj());
   }
-  return ScopedJavaLocalRef<jobjectArray>(env, joa);
+  return ScopedJavaLocalRef<jobjectArray>(env, java_array);
+}
+
+ScopedJavaLocalRef<jobjectArray> VectorToBufferArray(
+    JNIEnv* env,
+    const std::vector<std::vector<uint8_t>>& vector) {
+  FML_DCHECK(env);
+  ScopedJavaLocalRef<jclass> byte_buffer_clazz(
+      env, env->FindClass("java/nio/ByteBuffer"));
+  FML_DCHECK(!byte_buffer_clazz.is_null());
+  jobjectArray java_array =
+      env->NewObjectArray(vector.size(), byte_buffer_clazz.obj(), NULL);
+  ASSERT_NO_EXCEPTION();
+  for (size_t i = 0; i < vector.size(); ++i) {
+    ScopedJavaLocalRef<jobject> item(
+        env,
+        env->NewDirectByteBuffer((void*)(vector[i].data()), vector[i].size()));
+    env->SetObjectArrayElement(java_array, i, item.obj());
+  }
+  return ScopedJavaLocalRef<jobjectArray>(env, java_array);
 }
 
 bool HasException(JNIEnv* env) {

fml/platform/android/jni_util.h

@@ -34,6 +34,10 @@ ScopedJavaLocalRef<jobjectArray> VectorToStringArray(
     JNIEnv* env,
     const std::vector<std::string>& vector);
 
+ScopedJavaLocalRef<jobjectArray> VectorToBufferArray(
+    JNIEnv* env,
+    const std::vector<std::vector<uint8_t>>& vector);
+
 bool HasException(JNIEnv* env);
 
 bool ClearException(JNIEnv* env);

lib/ui/BUILD.gn

@@ -82,6 +82,8 @@ source_set("ui") {
     "semantics/semantics_update.h",
     "semantics/semantics_update_builder.cc",
     "semantics/semantics_update_builder.h",
+    "semantics/string_attribute.cc",
+    "semantics/string_attribute.h",
     "snapshot_delegate.h",
     "text/asset_manager_font_provider.cc",
     "text/asset_manager_font_provider.h",
@@ -205,6 +207,7 @@ if (enable_unittests) {
       "painting/path_unittests.cc",
       "painting/single_frame_codec_unittests.cc",
       "painting/vertices_unittests.cc",
+      "semantics/semantics_update_builder_unittests.cc",
       "window/platform_configuration_unittests.cc",
       "window/pointer_data_packet_converter_unittests.cc",
     ]

lib/ui/dart_ui.cc

@@ -26,6 +26,7 @@
 #include "flutter/lib/ui/painting/vertices.h"
 #include "flutter/lib/ui/semantics/semantics_update.h"
 #include "flutter/lib/ui/semantics/semantics_update_builder.h"
+#include "flutter/lib/ui/semantics/string_attribute.h"
 #include "flutter/lib/ui/text/font_collection.h"
 #include "flutter/lib/ui/text/paragraph.h"
 #include "flutter/lib/ui/text/paragraph_builder.h"
@@ -74,6 +75,7 @@ void DartUI::InitForGlobal() {
     ImageShader::RegisterNatives(g_natives);
     ImmutableBuffer::RegisterNatives(g_natives);
     IsolateNameServerNatives::RegisterNatives(g_natives);
+    NativeStringAttribute::RegisterNatives(g_natives);
     Paragraph::RegisterNatives(g_natives);
     ParagraphBuilder::RegisterNatives(g_natives);
     Picture::RegisterNatives(g_natives);

lib/ui/fixtures/ui_test.dart

@@ -58,6 +58,97 @@ void createVertices() {
 }
 void _validateVertices(Vertices vertices) native 'ValidateVertices';
 
+@pragma('vm:entry-point')
+void sendSemanticsUpdate() {
+  final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder();
+  final String label = "label";
+  final List<StringAttribute> labelAttributes = <StringAttribute> [
+    SpellOutStringAttribute(range: TextRange(start: 1, end: 2)),
+  ];
+
+  final String value = "value";
+  final List<StringAttribute> valueAttributes = <StringAttribute> [
+    SpellOutStringAttribute(range: TextRange(start: 2, end: 3)),
+  ];
+
+  final String increasedValue = "increasedValue";
+  final List<StringAttribute> increasedValueAttributes = <StringAttribute> [
+    SpellOutStringAttribute(range: TextRange(start: 4, end: 5)),
+  ];
+
+  final String decreasedValue = "decreasedValue";
+  final List<StringAttribute> decreasedValueAttributes = <StringAttribute> [
+    SpellOutStringAttribute(range: TextRange(start: 5, end: 6)),
+  ];
+
+  final String hint = "hint";
+  final List<StringAttribute> hintAttributes = <StringAttribute> [
+    LocaleStringAttribute(
+      locale: Locale('en', 'MX'), range: TextRange(start: 0, end: 1),
+    ),
+  ];
+
+  final Float64List transform = Float64List(16);
+  final Int32List childrenInTraversalOrder = Int32List(0);
+  final Int32List childrenInHitTestOrder = Int32List(0);
+  final Int32List additionalActions = Int32List(0);
+  transform[0] = 1;
+  transform[1] = 0;
+  transform[2] = 0;
+  transform[3] = 0;
+
+  transform[4] = 0;
+  transform[5] = 1;
+  transform[6] = 0;
+  transform[7] = 0;
+
+  transform[8] = 0;
+  transform[9] = 0;
+  transform[10] = 1;
+  transform[11] = 0;
+
+  transform[12] = 0;
+  transform[13] = 0;
+  transform[14] = 0;
+  transform[15] = 0;
+  builder.updateNode(
+    id: 0,
+    flags: 0,
+    actions: 0,
+    maxValueLength: 0,
+    currentValueLength: 0,
+    textSelectionBase: -1,
+    textSelectionExtent: -1,
+    platformViewId: -1,
+    scrollChildren: 0,
+    scrollIndex: 0,
+    scrollPosition: 0,
+    scrollExtentMax: 0,
+    scrollExtentMin: 0,
+    rect: Rect.fromLTRB(0, 0, 10, 10),
+    elevation: 0,
+    thickness: 0,
+    label: label,
+    labelAttributes: labelAttributes,
+    value: value,
+    valueAttributes: valueAttributes,
+    increasedValue: increasedValue,
+    increasedValueAttributes: increasedValueAttributes,
+    decreasedValue: decreasedValue,
+    decreasedValueAttributes: decreasedValueAttributes,
+    hint: hint,
+    hintAttributes: hintAttributes,
+    textDirection: TextDirection.ltr,
+    transform: transform,
+    childrenInTraversalOrder: childrenInTraversalOrder,
+    childrenInHitTestOrder: childrenInHitTestOrder,
+    additionalActions: additionalActions
+  );
+  _semanticsUpdate(builder.build());
+}
+
+void _semanticsUpdate(SemanticsUpdate update) native 'SemanticsUpdate';
+
 @pragma('vm:entry-point')
 void createPath() {
   final Path path = Path()..lineTo(10, 10);

lib/ui/semantics.dart

@@ -641,6 +641,108 @@ class SemanticsFlag {
   }
 }
 
+// When adding a new StringAttribute, the classes in these files must be
+// updated as well.
+//  * engine/src/flutter/lib/web_ui/lib/src/ui/semantics.dart
+//  * engine/src/flutter/lib/ui/semantics/string_attribute.h
+//  * engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java
+
+/// An abstract interface for string attributes that affects how assistive
+/// technologies, e.g. VoiceOver or TalkBack, treat the text.
+///
+/// See also:
+///
+///  * [AttributedString], where the string attributes are used.
+///  * [SpellOutStringAttribute], which causes the assistive technologies to
+///    spell out the string character by character when announcing the string.
+///  * [LocaleStringAttribute], which causes the assistive technologies to
+///    treat the string in the specific language.
+abstract class StringAttribute extends NativeFieldWrapperClass2 {
+  StringAttribute._({
+    required this.range,
+  });
+
+  // The range of the text to which this attribute applies.
+  final TextRange range;
+
+  // Returns a copy of this atttribute with the given range.
+  StringAttribute copy({required TextRange range});
+}
+
+/// A string attribute that causes the assistive technologies, e.g. VoiceOver,
+/// to spell out the string character by character.
+///
+/// See also:
+///
+///  * [AttributedString], where the string attributes are used.
+///  * [LocaleStringAttribute], which causes the assistive technologies to
+///    treat the string in the specific language.
+class SpellOutStringAttribute extends StringAttribute {
+  /// Creates a string attribute that denotes the text in [range] must be
+  /// spell out when the assistive technologies announce the string.
+  SpellOutStringAttribute({
+    required TextRange range,
+  }) : super._(range: range) {
+    _initSpellOutStringAttribute(this, range.start, range.end);
+  }
+
+  void _initSpellOutStringAttribute(
+    SpellOutStringAttribute instance,
+    int start,
+    int end,
+  ) native 'NativeStringAttribute_initSpellOutStringAttribute';
+
+  @override
+  StringAttribute copy({required TextRange range}) {
+    return SpellOutStringAttribute(range: range);
+  }
+
+  @override
+  String toString() {
+    return 'SpellOutStringAttribute($range)';
+  }
+}
+
+/// A string attribute that causes the assistive technologies, e.g. VoiceOver,
+/// to treat string as a certain language.
+///
+/// See also:
+///
+///  * [AttributedString], where the string attributes are used.
+///  * [SpellOutStringAttribute], which causes the assistive technologies to
+///    spell out the string character by character when announcing the string.
+class LocaleStringAttribute extends StringAttribute {
+  /// Creates a string attribute that denotes the text in [range] must be
+  /// treated as the language specified by the [locale] when the assistive
+  /// technologies announce the string.
+  LocaleStringAttribute({
+    required TextRange range,
+    required this.locale,
+  }) : super._(range: range) {
+    _initLocaleStringAttribute(this, range.start, range.end, locale.toLanguageTag());
+  }
+
+  /// The lanuage of this attribute.
+  final Locale locale;
+
+  void _initLocaleStringAttribute(
+    LocaleStringAttribute instance,
+    int start,
+    int end,
+    String locale,
+  ) native 'NativeStringAttribute_initLocaleStringAttribute';
+
+  @override
+  StringAttribute copy({required TextRange range}) {
+    return LocaleStringAttribute(range: range, locale: locale);
+  }
+
+  @override
+  String toString() {
+    return 'LocaleStringAttribute($range, ${locale.toLanguageTag()})';
+  }
+}
+
 /// An object that creates [SemanticsUpdate] objects.
 ///
 /// Once created, the [SemanticsUpdate] objects can be passed to
@@ -686,6 +788,12 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
   /// string describes what result an action performed on this node has. The
   /// reading direction of all these strings is given by `textDirection`.
   ///
+  /// The `labelAttirbutes`, `valueAttirbutes`, `hintAttributes`,
+  /// `increasedValueAttirbutes`, and `decreasedValueAttributes` are the lists of
+  /// [StringAttribute] carried by the `label`, `value`, `hint`, `increasedValue`,
+  /// and `decreasedValue` respectively. Their contents must not be changed during
+  /// the semantics update.
+  ///
   /// The fields `textSelectionBase` and `textSelectionExtent` describe the
   /// currently selected text within `value`. A value of -1 indicates no
   /// current text selection base or extent.
@@ -743,10 +851,15 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     required double thickness,
     required Rect rect,
     required String label,
-    required String hint,
+    List<StringAttribute>? labelAttributes,
     required String value,
+    List<StringAttribute>? valueAttributes,
     required String increasedValue,
+    List<StringAttribute>? increasedValueAttributes,
     required String decreasedValue,
+    List<StringAttribute>? decreasedValueAttributes,
+    required String hint,
+    List<StringAttribute>? hintAttributes,
     TextDirection? textDirection,
     required Float64List transform,
     required Int32List childrenInTraversalOrder,
@@ -779,10 +892,15 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
       elevation,
       thickness,
       label,
-      hint,
+      labelAttributes,
       value,
+      valueAttributes,
       increasedValue,
+      increasedValueAttributes,
       decreasedValue,
+      decreasedValueAttributes,
+      hint,
+      hintAttributes,
       textDirection != null ? textDirection.index + 1 : 0,
       transform,
       childrenInTraversalOrder,
@@ -811,10 +929,15 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     double elevation,
     double thickness,
     String label,
-    String hint,
+    List<StringAttribute>? labelAttributes,
     String value,
+    List<StringAttribute>? valueAttributes,
     String increasedValue,
+    List<StringAttribute>? increasedValueAttributes,
     String decreasedValue,
+    List<StringAttribute>? decreasedValueAttributes,
+    String hint,
+    List<StringAttribute>? hintAttributes,
     int textDirection,
     Float64List transform,
     Int32List childrenInTraversalOrder,

lib/ui/semantics/semantics_node.h

@@ -13,6 +13,8 @@
 #include "third_party/skia/include/core/SkM44.h"
 #include "third_party/skia/include/core/SkRect.h"
 
+#include "flutter/lib/ui/semantics/string_attribute.h"
+
 namespace flutter {
 
 // Must match the SemanticsAction enum in semantics.dart and in each of the
@@ -115,10 +117,15 @@ struct SemanticsNode {
   double elevation = 0.0;
   double thickness = 0.0;
   std::string label;
+  StringAttributes labelAttributes;
   std::string hint;
+  StringAttributes hintAttributes;
   std::string value;
+  StringAttributes valueAttributes;
   std::string increasedValue;
+  StringAttributes increasedValueAttributes;
   std::string decreasedValue;
+  StringAttributes decreasedValueAttributes;
   int32_t textDirection = 0;  // 0=unknown, 1=rtl, 2=ltr
 
   SkRect rect = SkRect::MakeEmpty();  // Local space, relative to parent.