Skip to content

Commit

Permalink
Encode scrolling status into tree (flutter#4647)
Browse files Browse the repository at this point in the history
  • Loading branch information
@goderbauer
goderbauer authored Feb 9, 2018
1 parent a031239 commit 8ac6f6e
Show file tree
Hide file tree
Showing 7 changed files with 115 additions and 30 deletions.
16 changes: 16 additions & 0 deletions lib/ui/semantics.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,13 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
/// The fields 'textSelectionBase' and 'textSelectionExtent' describe the
/// currently selected text within `value`.
///
/// For scrollable nodes `scrollPosition` describes the current scroll
/// position in logical pixel. `scrollExtentMax` and `scrollExtentMin`
/// describe the maximum and minimum in-rage values that `scrollPosition` can
/// be. Both or either may be infinity to indicate unbound scrolling. The
/// value for `scrollPosition` can (temporarily) be outside this range, for
/// example during an overscroll.
///
/// The `rect` is the region occupied by this node in its own coordinate
/// system.
///
Expand All @@ -348,6 +355,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
int actions,
int textSelectionBase,
int textSelectionExtent,
double scrollPosition,
double scrollExtentMax,
double scrollExtentMin,
Rect rect,
String label,
String hint,
Expand All @@ -366,6 +376,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
actions,
textSelectionBase,
textSelectionExtent,
scrollPosition,
scrollExtentMax,
scrollExtentMin,
rect.left,
rect.top,
rect.right,
Expand All @@ -386,6 +399,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
int actions,
int textSelectionBase,
int textSelectionExtent,
double scrollPosition,
double scrollExtentMax,
double scrollExtentMin,
double left,
double top,
double right,
Expand Down
3 changes: 3 additions & 0 deletions lib/ui/semantics/semantics_node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ struct SemanticsNode {
int32_t actions = 0;
int32_t textSelectionBase = -1;
int32_t textSelectionExtent = -1;
double scrollPosition = std::nan("");
double scrollExtentMax = std::nan("");
double scrollExtentMin = std::nan("");
std::string label;
std::string hint;
std::string value;
Expand Down
6 changes: 6 additions & 0 deletions lib/ui/semantics/semantics_update_builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ void SemanticsUpdateBuilder::updateNode(int id,
int actions,
int textSelectionBase,
int textSelectionExtent,
double scrollPosition,
double scrollExtentMax,
double scrollExtentMin,
double left,
double top,
double right,
Expand All @@ -58,6 +61,9 @@ void SemanticsUpdateBuilder::updateNode(int id,
node.actions = actions;
node.textSelectionBase = textSelectionBase;
node.textSelectionExtent = textSelectionExtent;
node.scrollPosition = scrollPosition;
node.scrollExtentMax = scrollExtentMax;
node.scrollExtentMin = scrollExtentMin;
node.rect = SkRect::MakeLTRB(left, top, right, bottom);
node.label = label;
node.hint = hint;
Expand Down
3 changes: 3 additions & 0 deletions lib/ui/semantics/semantics_update_builder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ class SemanticsUpdateBuilder
int actions,
int textSelectionBase,
int textSelectionExtent,
double scrollPosition,
double scrollExtentMax,
double scrollExtentMin,
double left,
double top,
double right,
Expand Down
93 changes: 68 additions & 25 deletions shell/platform/android/io/flutter/view/AccessibilityBridge.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
// Constants from higher API levels.
// TODO(goderbauer): Get these from Android Support Library when
// https://github.com/flutter/flutter/issues/11099 is resolved.
public static final int ACTION_SHOW_ON_SCREEN = 16908342; // API level 23
private static final int ACTION_SHOW_ON_SCREEN = 16908342; // API level 23

private static final float SCROLL_EXTENT_FOR_INFINITY = 100000.0f;
private static final float SCROLL_POSITION_CAP_FOR_INFINITY = 70000.0f;

private Map<Integer, SemanticsObject> mObjects;
private final FlutterView mOwner;
Expand Down Expand Up @@ -460,7 +463,9 @@ void updateSemantics(ByteBuffer buffer, String[] strings) {
if (object.hasFlag(Flag.IS_FOCUSED)) {
mInputFocusedObject = object;
}
updated.add(object);
if (object.hadPreviousConfig) {
updated.add(object);
}
}

Set<SemanticsObject> visitedObjects = new HashSet<SemanticsObject>();
Expand All @@ -481,13 +486,46 @@ void updateSemantics(ByteBuffer buffer, String[] strings) {
}
}

// Send accessibility events for updated nodes
// TODO(goderbauer): Send this event only once (!) for changed subtrees,
// see https://github.com/flutter/flutter/issues/14534
sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);

for (SemanticsObject object : updated) {
sendAccessibilityEvent(object.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
if (!object.hadPreviousConfig) {
continue;
}
if (object.didScroll()) {
AccessibilityEvent event =
obtainAccessibilityEvent(object.id, AccessibilityEvent.TYPE_VIEW_SCROLLED);

// Android doesn't support unbound scrolling. So we pretend there is a large
// bound (SCROLL_EXTENT_FOR_INFINITY), which you can never reach.
float position = object.scrollPosition;
float max = object.scrollExtentMax;
if (Float.isInfinite(object.scrollExtentMax)) {
max = SCROLL_EXTENT_FOR_INFINITY;
if (position > SCROLL_POSITION_CAP_FOR_INFINITY) {
position = SCROLL_POSITION_CAP_FOR_INFINITY;
}
}
if (Float.isInfinite(object.scrollExtentMin)) {
max += SCROLL_EXTENT_FOR_INFINITY;
if (position < -SCROLL_POSITION_CAP_FOR_INFINITY) {
position = -SCROLL_POSITION_CAP_FOR_INFINITY;
}
position += SCROLL_EXTENT_FOR_INFINITY;
} else {
max -= object.scrollExtentMin;
position -= object.scrollExtentMin;
}

if (object.hadAction(Action.SCROLL_UP) || object.hadAction(Action.SCROLL_DOWN)) {
event.setScrollY((int) position);
event.setMaxScrollY((int) max);
} else if (object.hadAction(Action.SCROLL_LEFT)
|| object.hadAction(Action.SCROLL_RIGHT)) {
event.setScrollX((int) position);
event.setMaxScrollX((int) max);
}
sendAccessibilityEvent(event);
}
if (mA11yFocusedObject != null && mA11yFocusedObject.id == object.id
&& object.hadFlag(Flag.HAS_CHECKED_STATE)
&& object.hasFlag(Flag.HAS_CHECKED_STATE)
Expand Down Expand Up @@ -586,24 +624,6 @@ public void onMessage(Object message, BasicMessageChannel.Reply<Object> reply) {
final HashMap<String, Object> data = (HashMap<String, Object>)annotatedEvent.get("data");

switch (type) {
case "scroll":
final int nodeId = (int)annotatedEvent.get("nodeId");
AccessibilityEvent event =
obtainAccessibilityEvent(nodeId, AccessibilityEvent.TYPE_VIEW_SCROLLED);
char axis = ((String)data.get("axis")).charAt(0);
double minPosition = (double)data.get("minScrollExtent");
double maxPosition = (double)data.get("maxScrollExtent") - minPosition;
double position = (double)data.get("pixels") - minPosition;
if (axis == 'v') {
event.setScrollY((int)position);
event.setMaxScrollY((int)maxPosition);
} else {
assert axis == 'h';
event.setScrollX((int)position);
event.setMaxScrollX((int)maxPosition);
}
sendAccessibilityEvent(event);
break;
case "announce":
mOwner.announceForAccessibility((String) data.get("message"));
break;
Expand Down Expand Up @@ -660,6 +680,9 @@ private class SemanticsObject {
int actions;
int textSelectionBase;
int textSelectionExtent;
float scrollPosition;
float scrollExtentMax;
float scrollExtentMin;
String label;
String value;
String increasedValue;
Expand All @@ -670,8 +693,12 @@ private class SemanticsObject {

boolean hadPreviousConfig = false;
int previousFlags;
int previousActions;
int previousTextSelectionBase;
int previousTextSelectionExtent;
float previousScrollPosition;
float previousScrollExtentMax;
float previousScrollExtentMin;
String previousValue;

private float left;
Expand All @@ -694,6 +721,10 @@ boolean hasAction(Action action) {
return (actions & action.value) != 0;
}

boolean hadAction(Action action) {
return (previousActions & action.value) != 0;
}

boolean hasFlag(Flag flag) {
return (flags & flag.value) != 0;
}
Expand All @@ -703,6 +734,11 @@ boolean hadFlag(Flag flag) {
return (previousFlags & flag.value) != 0;
}

boolean didScroll() {
return !Float.isNaN(scrollPosition) && !Float.isNaN(previousScrollPosition)
&& previousScrollPosition != scrollPosition;
}

void log(String indent, boolean recursive) {
Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" + actions + " flags=" + flags + "\n" +
indent + " +-- textDirection=" + textDirection + "\n"+
Expand All @@ -721,13 +757,20 @@ void updateWith(ByteBuffer buffer, String[] strings) {
hadPreviousConfig = true;
previousValue = value;
previousFlags = flags;
previousActions = actions;
previousTextSelectionBase = textSelectionBase;
previousTextSelectionExtent = textSelectionExtent;
previousScrollPosition = scrollPosition;
previousScrollExtentMax = scrollExtentMax;
previousScrollExtentMin = scrollExtentMin;

flags = buffer.getInt();
actions = buffer.getInt();
textSelectionBase = buffer.getInt();
textSelectionExtent = buffer.getInt();
scrollPosition = buffer.getFloat();
scrollExtentMax = buffer.getFloat();
scrollExtentMin = buffer.getFloat();

int stringIndex = buffer.getInt();
label = stringIndex == -1 ? null : strings[stringIndex];
Expand Down
5 changes: 4 additions & 1 deletion shell/platform/android/platform_view_android.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() {

void PlatformViewAndroid::UpdateSemantics(
blink::SemanticsNodeUpdates update) {
constexpr size_t kBytesPerNode = 33 * sizeof(int32_t);
constexpr size_t kBytesPerNode = 36 * sizeof(int32_t);
constexpr size_t kBytesPerChild = sizeof(int32_t);

JNIEnv* env = fml::jni::AttachCurrentThread();
Expand Down Expand Up @@ -492,6 +492,9 @@ void PlatformViewAndroid::UpdateSemantics(
buffer_int32[position++] = node.actions;
buffer_int32[position++] = node.textSelectionBase;
buffer_int32[position++] = node.textSelectionExtent;
buffer_float32[position++] = (float)node.scrollPosition;
buffer_float32[position++] = (float)node.scrollExtentMax;
buffer_float32[position++] = (float)node.scrollExtentMin;
if (node.label.empty()) {
buffer_int32[position++] = -1;
} else {
Expand Down
19 changes: 15 additions & 4 deletions shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,14 @@ - (BOOL)nodeWillCauseLayoutChange:(const blink::SemanticsNode*)node {
return [self node].rect != node->rect || [self node].transform != node->transform;
}

/**
* Whether calling `setSemanticsNode:` with `node` would cause a scroll event.
*/
- (BOOL)nodeWillCauseScroll:(const blink::SemanticsNode*)node {
return !isnan([self node].scrollPosition) && !isnan(node->scrollPosition) &&
[self node].scrollPosition != node->scrollPosition;
}

- (std::vector<SemanticsObject*>*)children {
return &_children;
}
Expand Down Expand Up @@ -404,11 +412,13 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
// traversal order (top left to bottom right, with hit testing order as tie breaker).
NSMutableSet<SemanticsObject*>* childOrdersToUpdate = [[[NSMutableSet alloc] init] autorelease];
BOOL layoutChanged = NO;
BOOL scrollOccured = NO;

for (const auto& entry : nodes) {
const blink::SemanticsNode& node = entry.second;
SemanticsObject* object = GetOrCreateObject(node.id, nodes);
layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
[object setSemanticsNode:&node];
const size_t childrenCount = node.children.size();
auto& children = *[object children];
Expand Down Expand Up @@ -452,6 +462,10 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
// TODO(goderbauer): figure out which node to focus next.
UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification, nil);
}
if (scrollOccured) {
// TODO(tvolkert): provide meaningful string (e.g. "page 2 of 5")
UIAccessibilityPostNotification(UIAccessibilityPageScrolledNotification, @"");
}
}

void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, blink::SemanticsAction action) {
Expand Down Expand Up @@ -511,10 +525,7 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {

void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
NSString* type = annotatedEvent[@"type"];
if ([type isEqualToString:@"scroll"]) {
// TODO(tvolkert): provide meaningful string (e.g. "page 2 of 5")
UIAccessibilityPostNotification(UIAccessibilityPageScrolledNotification, @"");
} else if ([type isEqualToString:@"announce"]) {
if ([type isEqualToString:@"announce"]) {
NSString* message = annotatedEvent[@"data"][@"message"];
UIAccessibilityPostNotification(UIAccessibilityAnnouncementNotification, message);
} else {
Expand Down

0 comments on commit 8ac6f6e

Please sign in to comment.