Skip to content

Commit

Permalink
Reland ios accessibility scrolling support (flutter#26860)
Browse files Browse the repository at this point in the history
  • Loading branch information
chunhtai authored Jun 23, 2021
1 parent 99b752e commit 7aa61d6
Show file tree
Hide file tree
Showing 5 changed files with 455 additions and 28 deletions.
11 changes: 8 additions & 3 deletions lib/ui/semantics/semantics_node.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,17 @@ enum class SemanticsAction : int32_t {
kSetText = 1 << 21,
};

const int kScrollableSemanticsActions =
static_cast<int32_t>(SemanticsAction::kScrollLeft) |
static_cast<int32_t>(SemanticsAction::kScrollRight) |
const int kVerticalScrollSemanticsActions =
static_cast<int32_t>(SemanticsAction::kScrollUp) |
static_cast<int32_t>(SemanticsAction::kScrollDown);

const int kHorizontalScrollSemanticsActions =
static_cast<int32_t>(SemanticsAction::kScrollLeft) |
static_cast<int32_t>(SemanticsAction::kScrollRight);

const int kScrollableSemanticsActions =
kVerticalScrollSemanticsActions | kHorizontalScrollSemanticsActions;

/// C/C++ representation of `SemanticsFlags` defined in
/// `lib/ui/semantics.dart`.
///\warning This must match the `SemanticsFlags` enum in
Expand Down
24 changes: 23 additions & 1 deletion shell/platform/darwin/ios/framework/Source/SemanticsObject.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
#import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge_ios.h"

constexpr int32_t kRootNodeId = 0;
// This can be arbitrary number as long as it is bigger than 0.
constexpr float kScrollExtentMaxForInf = 1000;

@class FlutterCustomAccessibilityAction;
@class FlutterPlatformViewSemanticsContainer;
Expand All @@ -31,7 +33,7 @@ constexpr int32_t kRootNodeId = 0;
* The parent of this node in the node tree. Will be nil for the root node and
* during transient state changes.
*/
@property(nonatomic, readonly) SemanticsObject* parent;
@property(nonatomic, assign) SemanticsObject* parent;

/**
* The accessibility bridge that this semantics object is attached to. This
Expand Down Expand Up @@ -94,6 +96,14 @@ constexpr int32_t kRootNodeId = 0;

- (BOOL)onCustomAccessibilityAction:(FlutterCustomAccessibilityAction*)action;

/**
* Called after accessibility bridge finishes a semantics update.
*
* Subclasses can override this method if they contain states that can only be
* updated once every node in the accessibility tree has finished updating.
*/
- (void)accessibilityBridgeDidFinishUpdate;

#pragma mark - Designated initializers

- (instancetype)init __attribute__((unavailable("Use initWithBridge instead")));
Expand Down Expand Up @@ -159,6 +169,18 @@ constexpr int32_t kRootNodeId = 0;

@end

/// The semantics object for scrollable. This class creates an UIScrollView to interact with the
/// iOS.
@interface FlutterScrollableSemanticsObject : UIScrollView

- (instancetype)init NS_UNAVAILABLE;
- (instancetype)initWithFrame:(CGRect)frame NS_UNAVAILABLE;
- (instancetype)initWithCoder:(NSCoder*)coder NS_UNAVAILABLE;
- (instancetype)initWithSemanticsObject:(SemanticsObject*)semanticsObject NS_DESIGNATED_INITIALIZER;
- (void)accessibilityBridgeDidFinishUpdate;

@end

/**
* Represents a semantics object that has children and hence has to be presented to the OS as a
* UIAccessibilityContainer.
Expand Down
248 changes: 227 additions & 21 deletions shell/platform/darwin/ios/framework/Source/SemanticsObject.mm
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,58 @@
return flutter::SemanticsAction::kScrollUp;
}

SkM44 GetGlobalTransform(SemanticsObject* reference) {
SkM44 globalTransform = [reference node].transform;
for (SemanticsObject* parent = [reference parent]; parent; parent = parent.parent) {
globalTransform = parent.node.transform * globalTransform;
}
return globalTransform;
}

SkPoint ApplyTransform(SkPoint& point, const SkM44& transform) {
SkV4 vector = transform.map(point.x(), point.y(), 0, 1);
return SkPoint::Make(vector.x / vector.w, vector.y / vector.w);
}

CGPoint ConvertPointToGlobal(SemanticsObject* reference, CGPoint local_point) {
SkM44 globalTransform = GetGlobalTransform(reference);
SkPoint point = SkPoint::Make(local_point.x, local_point.y);
point = ApplyTransform(point, globalTransform);
// `rect` is in the physical pixel coordinate system. iOS expects the accessibility frame in
// the logical pixel coordinate system. Therefore, we divide by the `scale` (pixel ratio) to
// convert.
CGFloat scale = [[[reference bridge]->view() window] screen].scale;
auto result = CGPointMake(point.x() / scale, point.y() / scale);
return [[reference bridge]->view() convertPoint:result toView:nil];
}

CGRect ConvertRectToGlobal(SemanticsObject* reference, CGRect local_rect) {
SkM44 globalTransform = GetGlobalTransform(reference);

SkPoint quad[4] = {
SkPoint::Make(local_rect.origin.x, local_rect.origin.y), // top left
SkPoint::Make(local_rect.origin.x + local_rect.size.width, local_rect.origin.y), // top right
SkPoint::Make(local_rect.origin.x + local_rect.size.width,
local_rect.origin.y + local_rect.size.height), // bottom right
SkPoint::Make(local_rect.origin.x,
local_rect.origin.y + local_rect.size.height) // bottom left
};
for (auto& point : quad) {
point = ApplyTransform(point, globalTransform);
}
SkRect rect;
NSCAssert(rect.setBoundsCheck(quad, 4), @"Transformed points can't form a rect");
rect.setBounds(quad, 4);

// `rect` is in the physical pixel coordinate system. iOS expects the accessibility frame in
// the logical pixel coordinate system. Therefore, we divide by the `scale` (pixel ratio) to
// convert.
CGFloat scale = [[[reference bridge]->view() window] screen].scale;
auto result =
CGRectMake(rect.x() / scale, rect.y() / scale, rect.width() / scale, rect.height() / scale);
return UIAccessibilityConvertFrameToScreenCoordinates(result, [reference bridge]->view());
}

} // namespace

@implementation FlutterSwitchSemanticsObject {
Expand Down Expand Up @@ -88,6 +140,175 @@ - (UIAccessibilityTraits)accessibilityTraits {

@end // FlutterSwitchSemanticsObject

@interface FlutterScrollableSemanticsObject ()
@property(nonatomic, strong) SemanticsObject* semanticsObject;
@end

@implementation FlutterScrollableSemanticsObject {
fml::scoped_nsobject<SemanticsObjectContainer> _container;
}

- (instancetype)initWithSemanticsObject:(SemanticsObject*)semanticsObject {
self = [super initWithFrame:CGRectZero];
if (self) {
_semanticsObject = [semanticsObject retain];
[semanticsObject.bridge->view() addSubview:self];
}
return self;
}

- (void)dealloc {
_container.get().semanticsObject = nil;
[_semanticsObject release];
[self removeFromSuperview];
[super dealloc];
}

- (UIView*)hitTest:(CGPoint)point withEvent:(UIEvent*)event {
return nil;
}

- (NSMethodSignature*)methodSignatureForSelector:(SEL)sel {
NSMethodSignature* result = [super methodSignatureForSelector:sel];
if (!result) {
result = [_semanticsObject methodSignatureForSelector:sel];
}
return result;
}

- (void)forwardInvocation:(NSInvocation*)anInvocation {
[anInvocation setTarget:_semanticsObject];
[anInvocation invoke];
}

- (void)accessibilityBridgeDidFinishUpdate {
// In order to make iOS think this UIScrollView is scrollable, the following
// requirements must be true.
// 1. contentSize must be bigger than the frame size.
// 2. The scrollable isAccessibilityElement must return YES
//
// Once the requirements are met, the iOS uses contentOffset to determine
// what scroll actions are available. e.g. If the view scrolls vertically and
// contentOffset is 0.0, only the scroll down action is available.
[self setFrame:[_semanticsObject accessibilityFrame]];
[self setContentSize:[self contentSizeInternal]];
[self setContentOffset:[self contentOffsetInternal] animated:NO];
if (self.contentSize.width > self.frame.size.width ||
self.contentSize.height > self.frame.size.height) {
self.isAccessibilityElement = YES;
} else {
self.isAccessibilityElement = NO;
}
}

- (void)setChildren:(NSArray<SemanticsObject*>*)children {
[_semanticsObject setChildren:children];
// The children's parent is pointing to _semanticsObject, need to manually
// set it this object.
for (SemanticsObject* child in _semanticsObject.children) {
child.parent = (SemanticsObject*)self;
}
}

- (id)accessibilityContainer {
if (_container == nil) {
_container.reset([[SemanticsObjectContainer alloc]
initWithSemanticsObject:(SemanticsObject*)self
bridge:[_semanticsObject bridge]]);
}
return _container.get();
}

// private methods

- (float)scrollExtentMax {
if (![_semanticsObject isAccessibilityBridgeAlive]) {
return 0.0f;
}
float scrollExtentMax = _semanticsObject.node.scrollExtentMax;
if (isnan(scrollExtentMax)) {
scrollExtentMax = 0.0f;
} else if (!isfinite(scrollExtentMax)) {
scrollExtentMax = kScrollExtentMaxForInf + [self scrollPosition];
}
return scrollExtentMax;
}

- (float)scrollPosition {
if (![_semanticsObject isAccessibilityBridgeAlive]) {
return 0.0f;
}
float scrollPosition = _semanticsObject.node.scrollPosition;
if (isnan(scrollPosition)) {
scrollPosition = 0.0f;
}
NSCAssert(isfinite(scrollPosition), @"The scrollPosition must not be infinity");
return scrollPosition;
}

- (CGSize)contentSizeInternal {
CGRect result;
const SkRect& rect = _semanticsObject.node.rect;

if (_semanticsObject.node.actions & flutter::kVerticalScrollSemanticsActions) {
result = CGRectMake(rect.x(), rect.y(), rect.width(), rect.height() + [self scrollExtentMax]);
} else if (_semanticsObject.node.actions & flutter::kHorizontalScrollSemanticsActions) {
result = CGRectMake(rect.x(), rect.y(), rect.width() + [self scrollExtentMax], rect.height());
} else {
result = CGRectMake(rect.x(), rect.y(), rect.width(), rect.height());
}
return ConvertRectToGlobal(_semanticsObject, result).size;
}

- (CGPoint)contentOffsetInternal {
CGPoint result;
CGPoint origin = self.frame.origin;
const SkRect& rect = _semanticsObject.node.rect;
if (_semanticsObject.node.actions & flutter::kVerticalScrollSemanticsActions) {
result = ConvertPointToGlobal(_semanticsObject,
CGPointMake(rect.x(), rect.y() + [self scrollPosition]));
} else if (_semanticsObject.node.actions & flutter::kHorizontalScrollSemanticsActions) {
result = ConvertPointToGlobal(_semanticsObject,
CGPointMake(rect.x() + [self scrollPosition], rect.y()));
} else {
result = origin;
}
return CGPointMake(result.x - origin.x, result.y - origin.y);
}

// The following methods are explicitly forwarded to the wrapped SemanticsObject because the
// forwarding logic above doesn't apply to them since they are also implemented in the
// UIScrollView class, the base class.

- (BOOL)accessibilityActivate {
return [_semanticsObject accessibilityActivate];
}

- (void)accessibilityIncrement {
[_semanticsObject accessibilityIncrement];
}

- (void)accessibilityDecrement {
[_semanticsObject accessibilityDecrement];
}

- (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
return [_semanticsObject accessibilityScroll:direction];
}

- (BOOL)accessibilityPerformEscape {
return [_semanticsObject accessibilityPerformEscape];
}

- (void)accessibilityElementDidBecomeFocused {
[_semanticsObject accessibilityElementDidBecomeFocused];
}

- (void)accessibilityElementDidLoseFocus {
[_semanticsObject accessibilityElementDidLoseFocus];
}
@end // FlutterScrollableSemanticsObject

@implementation FlutterCustomAccessibilityAction {
}
@end
Expand Down Expand Up @@ -174,6 +395,9 @@ - (void)setSemanticsNode:(const flutter::SemanticsNode*)node {
_node = *node;
}

- (void)accessibilityBridgeDidFinishUpdate { /* Do nothing by default */
}

/**
* Whether calling `setSemanticsNode:` with `node` would cause a layout change.
*/
Expand Down Expand Up @@ -398,27 +622,9 @@ - (CGRect)accessibilityFrame {
}

- (CGRect)globalRect {
SkM44 globalTransform = [self node].transform;
for (SemanticsObject* parent = [self parent]; parent; parent = parent.parent) {
globalTransform = parent.node.transform * globalTransform;
}

SkPoint quad[4];
[self node].rect.toQuad(quad);
for (auto& point : quad) {
SkV4 vector = globalTransform.map(point.x(), point.y(), 0, 1);
point.set(vector.x / vector.w, vector.y / vector.w);
}
SkRect rect;
rect.setBounds(quad, 4);

// `rect` is in the physical pixel coordinate system. iOS expects the accessibility frame in
// the logical pixel coordinate system. Therefore, we divide by the `scale` (pixel ratio) to
// convert.
CGFloat scale = [[[self bridge]->view() window] screen].scale;
auto result =
CGRectMake(rect.x() / scale, rect.y() / scale, rect.width() / scale, rect.height() / scale);
return UIAccessibilityConvertFrameToScreenCoordinates(result, [self bridge]->view());
const SkRect& rect = [self node].rect;
CGRect localRect = CGRectMake(rect.x(), rect.y(), rect.width(), rect.height());
return ConvertRectToGlobal(self, localRect);
}

#pragma mark - UIAccessibilityElement protocol
Expand Down
Loading

0 comments on commit 7aa61d6

Please sign in to comment.