Roll forward: Parameters for SemanticActions; a11y text selection (fl…

…utter#4452) Reverts the revert in flutter#4448 with fixes to pass on the bot. This change will require framework changes in flutter/flutter#13490.
pandavickey · Dec 12, 2017 · e07eafa · e07eafa
1 parent 2bdb219
commit e07eafa
Show file tree

Hide file tree

Showing 17 changed files with 92 additions and 36 deletions.
diff --git a/lib/ui/hooks.dart b/lib/ui/hooks.dart
@@ -85,12 +85,13 @@ void _dispatchPointerDataPacket(ByteData packet) {
     _invoke1<PointerDataPacket>(window.onPointerDataPacket, window._onPointerDataPacketZone, _unpackPointerDataPacket(packet));
 }
 
-void _dispatchSemanticsAction(int id, int action) {
-  _invoke2<int, SemanticsAction>(
+void _dispatchSemanticsAction(int id, int action, ByteData args) {
+  _invoke3<int, SemanticsAction, ByteData>(
     window.onSemanticsAction,
     window._onSemanticsActionZone,
     id,
     SemanticsAction.values[action],
+    args,
   );
 }
 

diff --git a/lib/ui/window.dart b/lib/ui/window.dart
@@ -14,7 +14,7 @@ typedef void FrameCallback(Duration duration);
 typedef void PointerDataPacketCallback(PointerDataPacket packet);
 
 /// Signature for [Window.onSemanticsAction].
-typedef void SemanticsActionCallback(int id, SemanticsAction action);
+typedef void SemanticsActionCallback(int id, SemanticsAction action, ByteData args);
 
 /// Signature for responses to platform messages.
 ///
@@ -439,7 +439,7 @@ class Window {
 
   /// The setting indicating whether time should always be shown in the 24-hour
   /// format.
-  /// 
+  ///
   /// This option is used by [showTimePicker].
   bool get alwaysUse24HourFormat => _alwaysUse24HourFormat;
   bool _alwaysUse24HourFormat = false;

diff --git a/lib/ui/window/window.cc b/lib/ui/window/window.cc
@@ -223,14 +223,22 @@ void Window::DispatchPointerDataPacket(const PointerDataPacket& packet) {
                   {data_handle});
 }
 
-void Window::DispatchSemanticsAction(int32_t id, SemanticsAction action) {
+void Window::DispatchSemanticsAction(int32_t id,
+                                     SemanticsAction action,
+                                     std::vector<uint8_t> args) {
   tonic::DartState* dart_state = library_.dart_state().get();
   if (!dart_state)
     return;
   tonic::DartState::Scope scope(dart_state);
 
-  DartInvokeField(library_.value(), "_dispatchSemanticsAction",
-                  {ToDart(id), ToDart(static_cast<int32_t>(action))});
+  Dart_Handle args_handle = (args.empty()) ? Dart_Null() : ToByteData(args);
+
+  if (Dart_IsError(args_handle))
+    return;
+
+  DartInvokeField(
+      library_.value(), "_dispatchSemanticsAction",
+      {ToDart(id), ToDart(static_cast<int32_t>(action)), args_handle});
 }
 
 void Window::BeginFrame(fxl::TimePoint frameTime) {

diff --git a/lib/ui/window/window.h b/lib/ui/window/window.h
@@ -49,7 +49,9 @@ class Window {
   void UpdateSemanticsEnabled(bool enabled);
   void DispatchPlatformMessage(fxl::RefPtr<PlatformMessage> message);
   void DispatchPointerDataPacket(const PointerDataPacket& packet);
-  void DispatchSemanticsAction(int32_t id, SemanticsAction action);
+  void DispatchSemanticsAction(int32_t id,
+                               SemanticsAction action,
+                               std::vector<uint8_t> args);
   void BeginFrame(fxl::TimePoint frameTime);
 
   void CompletePlatformMessageResponse(int response_id,

diff --git a/runtime/runtime_controller.cc b/runtime/runtime_controller.cc
@@ -106,10 +106,11 @@ void RuntimeController::DispatchPointerDataPacket(
 }
 
 void RuntimeController::DispatchSemanticsAction(int32_t id,
-                                                SemanticsAction action) {
+                                                SemanticsAction action,
+                                                std::vector<uint8_t> args) {
   TRACE_EVENT1("flutter", "RuntimeController::DispatchSemanticsAction", "mode",
                "basic");
-  GetWindow()->DispatchSemanticsAction(id, action);
+  GetWindow()->DispatchSemanticsAction(id, action, std::move(args));
 }
 
 Window* RuntimeController::GetWindow() {

diff --git a/runtime/runtime_controller.h b/runtime/runtime_controller.h
@@ -42,7 +42,9 @@ class RuntimeController : public WindowClient, public IsolateClient {
 
   void DispatchPlatformMessage(fxl::RefPtr<PlatformMessage> message);
   void DispatchPointerDataPacket(const PointerDataPacket& packet);
-  void DispatchSemanticsAction(int32_t id, SemanticsAction action);
+  void DispatchSemanticsAction(int32_t id,
+                               SemanticsAction action,
+                               std::vector<uint8_t> args);
 
   Dart_Port GetMainPort();
   std::string GetIsolateName();

diff --git a/shell/common/engine.cc b/shell/common/engine.cc
@@ -554,9 +554,11 @@ void Engine::DispatchPointerDataPacket(const PointerDataPacket& packet) {
     runtime_->DispatchPointerDataPacket(packet);
 }
 
-void Engine::DispatchSemanticsAction(int id, blink::SemanticsAction action) {
+void Engine::DispatchSemanticsAction(int id,
+                                     blink::SemanticsAction action,
+                                     std::vector<uint8_t> args) {
   if (runtime_)
-    runtime_->DispatchSemanticsAction(id, action);
+    runtime_->DispatchSemanticsAction(id, action, std::move(args));
 }
 
 void Engine::SetSemanticsEnabled(bool enabled) {

diff --git a/shell/common/engine.h b/shell/common/engine.h
@@ -72,7 +72,9 @@ class Engine : public blink::RuntimeDelegate {
   void SetViewportMetrics(const blink::ViewportMetrics& metrics);
   void DispatchPlatformMessage(fxl::RefPtr<blink::PlatformMessage> message);
   void DispatchPointerDataPacket(const PointerDataPacket& packet);
-  void DispatchSemanticsAction(int id, blink::SemanticsAction action);
+  void DispatchSemanticsAction(int id,
+                               blink::SemanticsAction action,
+                               std::vector<uint8_t> args);
   void SetSemanticsEnabled(bool enabled);
   void ScheduleFrame(bool regenerate_layer_tree = true) override;
 

diff --git a/shell/common/platform_view.cc b/shell/common/platform_view.cc
@@ -55,12 +55,13 @@ void PlatformView::DispatchPlatformMessage(
 }
 
 void PlatformView::DispatchSemanticsAction(int32_t id,
-                                           blink::SemanticsAction action) {
+                                           blink::SemanticsAction action,
+                                           std::vector<uint8_t> args) {
   blink::Threads::UI()->PostTask(
-      [ engine = engine_->GetWeakPtr(), id, action ] {
+      [ engine = engine_->GetWeakPtr(), id, action, args = std::move(args) ] {
         if (engine) {
           engine->DispatchSemanticsAction(
-              id, static_cast<blink::SemanticsAction>(action));
+              id, static_cast<blink::SemanticsAction>(action), std::move(args));
         }
       });
 }

diff --git a/shell/common/platform_view.h b/shell/common/platform_view.h
@@ -41,7 +41,9 @@ class PlatformView : public std::enable_shared_from_this<PlatformView> {
   virtual void Attach() = 0;
 
   void DispatchPlatformMessage(fxl::RefPtr<blink::PlatformMessage> message);
-  void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action);
+  void DispatchSemanticsAction(int32_t id,
+                               blink::SemanticsAction action,
+                               std::vector<uint8_t> args);
   void SetSemanticsEnabled(bool enabled);
 
   void NotifyCreated(std::unique_ptr<Surface> surface);

diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java
@@ -312,17 +312,16 @@ public boolean performAction(int virtualViewId, int action, Bundle arguments) {
     boolean performCursorMoveAction(SemanticsObject object, int virtualViewId, Bundle arguments, boolean forward) {
         final int granularity = arguments.getInt(
             AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT);
-        // TODO(goderbauer): support extending selections.
-        // final boolean extendSelection = arguments.getBoolean(
-        //     AccessibilityNodeInfo.ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN);
+        final boolean extendSelection = arguments.getBoolean(
+            AccessibilityNodeInfo.ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN);
         switch (granularity) {
             case AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER: {
                 if (forward && object.hasAction(Action.MOVE_CURSOR_FORWARD_BY_CHARACTER)) {
-                    mOwner.dispatchSemanticsAction(virtualViewId, Action.MOVE_CURSOR_FORWARD_BY_CHARACTER);
+                    mOwner.dispatchSemanticsAction(virtualViewId, Action.MOVE_CURSOR_FORWARD_BY_CHARACTER, extendSelection);
                     return true;
                 }
                 if (!forward && object.hasAction(Action.MOVE_CURSOR_BACKWARD_BY_CHARACTER)) {
-                    mOwner.dispatchSemanticsAction(virtualViewId, Action.MOVE_CURSOR_BACKWARD_BY_CHARACTER);
+                    mOwner.dispatchSemanticsAction(virtualViewId, Action.MOVE_CURSOR_BACKWARD_BY_CHARACTER, extendSelection);
                     return true;
                 }
             }

diff --git a/shell/platform/android/io/flutter/view/FlutterView.java b/shell/platform/android/io/flutter/view/FlutterView.java
@@ -686,7 +686,7 @@ private static native void nativeDispatchPointerDataPacket(long nativePlatformVi
         ByteBuffer buffer, int position);
 
     private static native void nativeDispatchSemanticsAction(long nativePlatformViewAndroid, int id,
-        int action);
+        int action, ByteBuffer args, int argsPosition);
 
     private static native void nativeSetSemanticsEnabled(long nativePlatformViewAndroid,
         boolean enabled);
@@ -747,9 +747,19 @@ public void onFirstFrame() {
     private TouchExplorationListener mTouchExplorationListener;
 
     protected void dispatchSemanticsAction(int id, AccessibilityBridge.Action action) {
+        dispatchSemanticsAction(id, action, null);
+    }
+
+    protected void dispatchSemanticsAction(int id, AccessibilityBridge.Action action, Object args) {
         if (!isAttached())
             return;
-        nativeDispatchSemanticsAction(mNativeView.get(), id, action.value);
+        ByteBuffer encodedArgs = null;
+        int position = 0;
+        if (args != null) {
+            encodedArgs = StandardMessageCodec.INSTANCE.encodeMessage(args);
+            position = encodedArgs.position();
+        }
+        nativeDispatchSemanticsAction(mNativeView.get(), id, action.value, encodedArgs, position);
     }
 
     @Override

diff --git a/shell/platform/android/platform_view_android.cc b/shell/platform/android/platform_view_android.cc
@@ -410,9 +410,24 @@ void PlatformViewAndroid::HandlePlatformMessageEmptyResponse(int response_id) {
                                            nullptr);
 }
 
-void PlatformViewAndroid::DispatchSemanticsAction(jint id, jint action) {
+void PlatformViewAndroid::DispatchSemanticsAction(JNIEnv* env,
+                                                  jint id,
+                                                  jint action,
+                                                  jobject args,
+                                                  jint args_position) {
+  if (env->IsSameObject(args, NULL)) {
+    std::vector<uint8_t> args_vector;
+    PlatformView::DispatchSemanticsAction(
+        id, static_cast<blink::SemanticsAction>(action), args_vector);
+    return;
+  }
+
+  uint8_t* args_data = static_cast<uint8_t*>(env->GetDirectBufferAddress(args));
+  std::vector<uint8_t> args_vector =
+      std::vector<uint8_t>(args_data, args_data + args_position);
+
   PlatformView::DispatchSemanticsAction(
-      id, static_cast<blink::SemanticsAction>(action));
+      id, static_cast<blink::SemanticsAction>(action), std::move(args_vector));
 }
 
 void PlatformViewAndroid::SetSemanticsEnabled(jboolean enabled) {

diff --git a/shell/platform/android/platform_view_android.h b/shell/platform/android/platform_view_android.h
@@ -79,7 +79,11 @@ class PlatformViewAndroid : public PlatformView {
   void InvokePlatformMessageEmptyResponseCallback(JNIEnv* env,
                                                   jint response_id);
 
-  void DispatchSemanticsAction(jint id, jint action);
+  void DispatchSemanticsAction(JNIEnv* env,
+                               jint id,
+                               jint action,
+                               jobject args,
+                               jint args_position);
 
   void SetSemanticsEnabled(jboolean enabled);
 

diff --git a/shell/platform/android/platform_view_android_jni.cc b/shell/platform/android/platform_view_android_jni.cc
@@ -236,8 +236,11 @@ static void DispatchSemanticsAction(JNIEnv* env,
                                     jobject jcaller,
                                     jlong platform_view,
                                     jint id,
-                                    jint action) {
-  return PLATFORM_VIEW->DispatchSemanticsAction(id, action);
+                                    jint action,
+                                    jobject args,
+                                    jint args_position) {
+  return PLATFORM_VIEW->DispatchSemanticsAction(env, id, action, args,
+                                                args_position);
 }
 
 static void SetSemanticsEnabled(JNIEnv* env,
@@ -413,7 +416,7 @@ bool PlatformViewAndroid::Register(JNIEnv* env) {
       },
       {
           .name = "nativeDispatchSemanticsAction",
-          .signature = "(JII)V",
+          .signature = "(JIILjava/nio/ByteBuffer;I)V",
           .fnPtr = reinterpret_cast<void*>(&shell::DispatchSemanticsAction),
       },
       {

diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm
@@ -423,7 +423,8 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
 }
 
 void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, blink::SemanticsAction action) {
-  platform_view_->DispatchSemanticsAction(uid, action);
+  std::vector<uint8_t> args;
+  platform_view_->DispatchSemanticsAction(uid, action, args);
 }
 
 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid) {

diff --git a/testing/dart/window_hooks_integration_test.dart b/testing/dart/window_hooks_integration_test.dart
@@ -177,20 +177,23 @@ void main() {
     test('onSemanticsAction preserves callback zone', () {
       Zone innerZone;
       Zone runZone;
+      int id;
       int action;
 
       runZoned(() {
         innerZone = Zone.current;
-        window.onSemanticsAction = (int value, _) {
+        window.onSemanticsAction = (int i, SemanticsAction a, ByteData _) {
           runZone = Zone.current;
-          action = value;
+          action = a.index;
+          id = i;
         };
       });
 
-      _dispatchSemanticsAction(1234, 0);
+      _dispatchSemanticsAction(1234, 4, null);
       expect(runZone, isNotNull);
       expect(runZone, same(innerZone));
-      expect(action, equals(1234));
+      expect(id, equals(1234));
+      expect(action, equals(4));
     });
 
     test('onPlatformMessage preserves callback zone', () {