Support Scribble Handwriting (flutter#24224)

AUTHORS

@@ -20,4 +20,5 @@ Hidenori Matsubayashi <[email protected]>
 Sarbagya Dhaubanjar <[email protected]>
 Callum Moffat <[email protected]>
 Koutaro Mori <[email protected]>
-TheOneWithTheBraid <[email protected]>
+TheOneWithTheBraid <[email protected]>
+Twin Sun, LLC <[email protected]>

ci/licenses_golden/licenses_flutter

@@ -1093,6 +1093,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterFakeKeyEvents.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterFakeKeyEvents.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterHeadlessDartRunner.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterIndirectScribbleDelegate.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterKeyPrimaryResponder.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterKeySecondaryResponder.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterKeyboardManager.h
@@ -1132,6 +1133,7 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterView.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h
+FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewResponder.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/FlutterViewTest.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/IOKit.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/KeyCodeMap.mm

shell/platform/darwin/ios/framework/Source/FlutterEngine.mm

@@ -20,6 +20,7 @@
 #import "flutter/shell/platform/darwin/common/command_line.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterBinaryMessengerRelay.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject_Internal.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterIndirectScribbleDelegate.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterObservatoryPublisher.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformPlugin.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputDelegate.h"
@@ -42,7 +43,9 @@ @interface FlutterEngineRegistrar : NSObject <FlutterPluginRegistrar>
 - (instancetype)initWithPlugin:(NSString*)pluginKey flutterEngine:(FlutterEngine*)flutterEngine;
 @end
 
-@interface FlutterEngine () <FlutterTextInputDelegate, FlutterBinaryMessenger>
+@interface FlutterEngine () <FlutterIndirectScribbleDelegate,
+                             FlutterTextInputDelegate,
+                             FlutterBinaryMessenger>
 // Maintains a dictionary of plugin names that have registered with the engine.  Used by
 // FlutterEngineRegistrar to implement a FlutterPluginRegistrar.
 @property(nonatomic, readonly) NSMutableDictionary* pluginPublications;
@@ -331,6 +334,7 @@ - (void)setViewController:(FlutterViewController*)viewController {
 
 - (void)attachView {
   self.iosPlatformView->attachView();
+  [_textInputPlugin.get() setupIndirectScribbleInteraction:self.viewController];
 }
 
 - (void)setFlutterViewControllerWillDeallocObserver:(id<NSObject>)observer {
@@ -355,6 +359,7 @@ - (void)notifyViewControllerDeallocated {
       platform_view->SetOwnerViewController({});
     }
   }
+  [_textInputPlugin.get() resetViewResponder];
   _viewController.reset();
 }
 
@@ -514,6 +519,8 @@ - (void)setupChannels {
 
   _textInputPlugin.reset([[FlutterTextInputPlugin alloc] init]);
   _textInputPlugin.get().textInputDelegate = self;
+  _textInputPlugin.get().indirectScribbleDelegate = self;
+  [_textInputPlugin.get() setupIndirectScribbleInteraction:self.viewController];
 
   _platformPlugin.reset([[FlutterPlatformPlugin alloc] initWithEngine:[self getWeakPtr]]);
 
@@ -720,22 +727,30 @@ - (void)notifyLowMemory {
 
 #pragma mark - Text input delegate
 
-- (void)updateEditingClient:(int)client withState:(NSDictionary*)state {
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+         updateEditingClient:(int)client
+                   withState:(NSDictionary*)state {
   [_textInputChannel.get() invokeMethod:@"TextInputClient.updateEditingState"
                               arguments:@[ @(client), state ]];
 }
 
-- (void)updateEditingClient:(int)client withState:(NSDictionary*)state withTag:(NSString*)tag {
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+         updateEditingClient:(int)client
+                   withState:(NSDictionary*)state
+                     withTag:(NSString*)tag {
   [_textInputChannel.get() invokeMethod:@"TextInputClient.updateEditingStateWithTag"
                               arguments:@[ @(client), @{tag : state} ]];
 }
 
-- (void)updateEditingClient:(int)client withDelta:(NSDictionary*)delta {
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+         updateEditingClient:(int)client
+                   withDelta:(NSDictionary*)delta {
   [_textInputChannel.get() invokeMethod:@"TextInputClient.updateEditingStateWithDeltas"
                               arguments:@[ @(client), delta ]];
 }
 
-- (void)updateFloatingCursor:(FlutterFloatingCursorDragState)state
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+        updateFloatingCursor:(FlutterFloatingCursorDragState)state
                   withClient:(int)client
                 withPosition:(NSDictionary*)position {
   NSString* stateString;
@@ -754,7 +769,9 @@ - (void)updateFloatingCursor:(FlutterFloatingCursorDragState)state
                               arguments:@[ @(client), stateString, position ]];
 }
 
-- (void)performAction:(FlutterTextInputAction)action withClient:(int)client {
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+               performAction:(FlutterTextInputAction)action
+                  withClient:(int)client {
   NSString* actionString;
   switch (action) {
     case FlutterTextInputActionUnspecified:
@@ -799,15 +816,63 @@ - (void)performAction:(FlutterTextInputAction)action withClient:(int)client {
                               arguments:@[ @(client), actionString ]];
 }
 
-- (void)showAutocorrectionPromptRectForStart:(NSUInteger)start
-                                         end:(NSUInteger)end
-                                  withClient:(int)client {
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+    showAutocorrectionPromptRectForStart:(NSUInteger)start
+                                     end:(NSUInteger)end
+                              withClient:(int)client {
   [_textInputChannel.get() invokeMethod:@"TextInputClient.showAutocorrectionPromptRect"
                               arguments:@[ @(client), @(start), @(end) ]];
 }
 
 #pragma mark - FlutterViewEngineDelegate
 
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView showToolbar:(int)client {
+  [_textInputChannel.get() invokeMethod:@"TextInputClient.showToolbar" arguments:@[ @(client) ]];
+}
+
+- (void)flutterTextInputPlugin:(FlutterTextInputPlugin*)textInputPlugin
+                  focusElement:(UIScribbleElementIdentifier)elementIdentifier
+                       atPoint:(CGPoint)referencePoint
+                        result:(FlutterResult)callback {
+  [_textInputChannel.get()
+      invokeMethod:@"TextInputClient.focusElement"
+         arguments:@[ elementIdentifier, @(referencePoint.x), @(referencePoint.y) ]
+            result:callback];
+}
+
+- (void)flutterTextInputPlugin:(FlutterTextInputPlugin*)textInputPlugin
+         requestElementsInRect:(CGRect)rect
+                        result:(FlutterResult)callback {
+  [_textInputChannel.get()
+      invokeMethod:@"TextInputClient.requestElementsInRect"
+         arguments:@[ @(rect.origin.x), @(rect.origin.y), @(rect.size.width), @(rect.size.height) ]
+            result:callback];
+}
+
+- (void)flutterTextInputViewScribbleInteractionBegan:(FlutterTextInputView*)textInputView {
+  [_textInputChannel.get() invokeMethod:@"TextInputClient.scribbleInteractionBegan" arguments:nil];
+}
+
+- (void)flutterTextInputViewScribbleInteractionFinished:(FlutterTextInputView*)textInputView {
+  [_textInputChannel.get() invokeMethod:@"TextInputClient.scribbleInteractionFinished"
+                              arguments:nil];
+}
+
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+    insertTextPlaceholderWithSize:(CGSize)size
+                       withClient:(int)client {
+  [_textInputChannel.get() invokeMethod:@"TextInputClient.insertTextPlaceholder"
+                              arguments:@[ @(client), @(size.width), @(size.height) ]];
+}
+
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+       removeTextPlaceholder:(int)client {
+  [_textInputChannel.get() invokeMethod:@"TextInputClient.removeTextPlaceholder"
+                              arguments:@[ @(client) ]];
+}
+
+#pragma mark - Screenshot Delegate
+
 - (flutter::Rasterizer::Screenshot)takeScreenshot:(flutter::Rasterizer::ScreenshotType)type
                                   asBase64Encoded:(BOOL)base64Encode {
   FML_DCHECK(_shell) << "Cannot takeScreenshot without a shell";

shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h

@@ -21,6 +21,7 @@
 
 #import "flutter/shell/platform/darwin/ios/framework/Headers/FlutterEngine.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterDartProject_Internal.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterIndirectScribbleDelegate.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformPlugin.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterRestorationPlugin.h"

shell/platform/darwin/ios/framework/Source/FlutterIndirectScribbleDelegate.h

@@ -0,0 +1,24 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERINDIRECTSCRIBBLEDELEGATE_H_
+#define SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERINDIRECTSCRIBBLEDELEGATE_H_
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+@class FlutterTextInputPlugin;
+
+@protocol FlutterIndirectScribbleDelegate <NSObject>
+- (void)flutterTextInputPlugin:(FlutterTextInputPlugin*)textInputPlugin
+                  focusElement:(UIScribbleElementIdentifier)elementIdentifier
+                       atPoint:(CGPoint)referencePoint
+                        result:(FlutterResult)callback;
+- (void)flutterTextInputPlugin:(FlutterTextInputPlugin*)textInputPlugin
+         requestElementsInRect:(CGRect)rect
+                        result:(FlutterResult)callback;
+@end
+NS_ASSUME_NONNULL_END
+
+#endif  // SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERINDIRECTSCRIBBLEDELEGATE_H_

shell/platform/darwin/ios/framework/Source/FlutterTextInputDelegate.h

@@ -7,6 +7,9 @@
 
 #import <Foundation/Foundation.h>
 
+@class FlutterTextInputPlugin;
+@class FlutterTextInputView;
+
 typedef NS_ENUM(NSInteger, FlutterTextInputAction) {
   FlutterTextInputActionUnspecified,
   FlutterTextInputActionDone,
@@ -28,16 +31,35 @@ typedef NS_ENUM(NSInteger, FlutterFloatingCursorDragState) {
 };
 
 @protocol FlutterTextInputDelegate <NSObject>
-- (void)updateEditingClient:(int)client withState:(NSDictionary*)state;
-- (void)updateEditingClient:(int)client withState:(NSDictionary*)state withTag:(NSString*)tag;
-- (void)updateEditingClient:(int)client withDelta:(NSDictionary*)state;
-- (void)performAction:(FlutterTextInputAction)action withClient:(int)client;
-- (void)updateFloatingCursor:(FlutterFloatingCursorDragState)state
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+         updateEditingClient:(int)client
+                   withState:(NSDictionary*)state;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+         updateEditingClient:(int)client
+                   withState:(NSDictionary*)state
+                     withTag:(NSString*)tag;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+         updateEditingClient:(int)client
+                   withDelta:(NSDictionary*)state;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+               performAction:(FlutterTextInputAction)action
+                  withClient:(int)client;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+        updateFloatingCursor:(FlutterFloatingCursorDragState)state
                   withClient:(int)client
                 withPosition:(NSDictionary*)point;
-- (void)showAutocorrectionPromptRectForStart:(NSUInteger)start
-                                         end:(NSUInteger)end
-                                  withClient:(int)client;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+    showAutocorrectionPromptRectForStart:(NSUInteger)start
+                                     end:(NSUInteger)end
+                              withClient:(int)client;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView showToolbar:(int)client;
+- (void)flutterTextInputViewScribbleInteractionBegan:(FlutterTextInputView*)textInputView;
+- (void)flutterTextInputViewScribbleInteractionFinished:(FlutterTextInputView*)textInputView;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView
+    insertTextPlaceholderWithSize:(CGSize)size
+                       withClient:(int)client;
+- (void)flutterTextInputView:(FlutterTextInputView*)textInputView removeTextPlaceholder:(int)client;
+
 @end
 
 #endif  // SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERTEXTINPUTDELEGATE_H_

shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h

@@ -8,14 +8,32 @@
 #import <UIKit/UIKit.h>
 
 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterChannels.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterIndirectScribbleDelegate.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterKeySecondaryResponder.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextEditingDelta.h"
 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputDelegate.h"
+#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewResponder.h"
 
-@interface FlutterTextInputPlugin : NSObject <FlutterKeySecondaryResponder>
+typedef NS_ENUM(NSInteger, FlutterScribbleFocusStatus) {
+  FlutterScribbleFocusStatusUnfocused,
+  FlutterScribbleFocusStatusFocusing,
+  FlutterScribbleFocusStatusFocused,
+};
+
+typedef NS_ENUM(NSInteger, FlutterScribbleInteractionStatus) {
+  FlutterScribbleInteractionStatusNone,
+  FlutterScribbleInteractionStatusStarted,
+  FlutterScribbleInteractionStatusEnding,
+};
+
+@interface FlutterTextInputPlugin
+    : NSObject <FlutterKeySecondaryResponder, UIIndirectScribbleInteractionDelegate>
 
 @property(nonatomic, assign) id<FlutterTextInputDelegate> textInputDelegate;
 @property(nonatomic, assign) UIViewController* viewController;
+@property(nonatomic, assign) id<FlutterIndirectScribbleDelegate> indirectScribbleDelegate;
+@property(nonatomic, strong)
+    NSMutableDictionary<UIScribbleElementIdentifier, NSValue*>* scribbleElements;
 - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result;
 
 /**
@@ -26,6 +44,13 @@
  */
 - (UIView<UITextInput>*)textInputView;
 
+/**
+ * These are used by the UIIndirectScribbleInteractionDelegate methods to handle focusing on the
+ * correct element.
+ */
+- (void)setupIndirectScribbleInteraction:(id<FlutterViewResponder>)viewResponder;
+- (void)resetViewResponder;
+
 @end
 
 /** An indexed position in the buffer of a Flutter text editing widget. */
@@ -51,10 +76,41 @@
 @interface FlutterTokenizer : UITextInputStringTokenizer
 @end
 
+@interface FlutterTextSelectionRect : UITextSelectionRect
+
+@property(nonatomic, assign) CGRect rect;
+@property(nonatomic) NSUInteger position;
+@property(nonatomic, assign) NSWritingDirection writingDirection;
+@property(nonatomic) BOOL containsStart;
+@property(nonatomic) BOOL containsEnd;
+@property(nonatomic) BOOL isVertical;
+
++ (instancetype)selectionRectWithRectAndInfo:(CGRect)rect
+                                    position:(NSUInteger)position
+                            writingDirection:(NSWritingDirection)writingDirection
+                               containsStart:(BOOL)containsStart
+                                 containsEnd:(BOOL)containsEnd
+                                  isVertical:(BOOL)isVertical;
+
++ (instancetype)selectionRectWithRect:(CGRect)rect position:(NSUInteger)position;
+
+- (instancetype)initWithRectAndInfo:(CGRect)rect
+                           position:(NSUInteger)position
+                   writingDirection:(NSWritingDirection)writingDirection
+                      containsStart:(BOOL)containsStart
+                        containsEnd:(BOOL)containsEnd
+                         isVertical:(BOOL)isVertical;
+
+- (instancetype)init NS_UNAVAILABLE;
+@end
+
+API_AVAILABLE(ios(13.0)) @interface FlutterTextPlaceholder : UITextPlaceholder
+@end
+
 #if FLUTTER_RUNTIME_MODE == FLUTTER_RUNTIME_MODE_DEBUG
 FLUTTER_DARWIN_EXPORT
 #endif
-@interface FlutterTextInputView : UIView <UITextInput>
+@interface FlutterTextInputView : UIView <UITextInput, UIScribbleInteractionDelegate>
 
 // UITextInput
 @property(nonatomic, readonly) NSMutableString* text;
@@ -81,5 +137,11 @@ FLUTTER_DARWIN_EXPORT
 @property(nonatomic, assign) id<FlutterTextInputDelegate> textInputDelegate;
 @property(nonatomic, assign) UIAccessibilityElement* backingTextInputAccessibilityObject;
 
+// Scribble Support
+@property(nonatomic, assign) id<FlutterViewResponder> viewResponder;
+@property(nonatomic) FlutterScribbleFocusStatus scribbleFocusStatus;
+@property(nonatomic, strong) NSArray<FlutterTextSelectionRect*>* selectionRects;
+- (void)resetScribbleInteractionStatusIfEnding;
+
 @end
 #endif  // SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERTEXTINPUTPLUGIN_H_