From c3e2c0f32b5ceb67313c3424fd5b06648a28fd00 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Wed, 22 Jan 2020 01:12:37 -0800 Subject: [PATCH] [fuchsia] HitTesting for fuchsia a11y (#15570) --- .../fuchsia/flutter/accessibility_bridge.cc | 91 ++++++++++++++++++- .../fuchsia/flutter/accessibility_bridge.h | 47 ++++++++-- .../flutter/accessibility_bridge_unittest.cc | 70 ++++++++++++++ 3 files changed, 195 insertions(+), 13 deletions(-) diff --git a/shell/platform/fuchsia/flutter/accessibility_bridge.cc b/shell/platform/fuchsia/flutter/accessibility_bridge.cc index 8f3e28104964a..ce6bd69343f19 100644 --- a/shell/platform/fuchsia/flutter/accessibility_bridge.cc +++ b/shell/platform/fuchsia/flutter/accessibility_bridge.cc @@ -99,8 +99,8 @@ std::unordered_set AccessibilityBridge::GetDescendants( auto it = nodes_.find(id); if (it != nodes_.end()) { - auto const& children = it->second; - for (const auto& child : children) { + const auto& node = it->second; + for (const auto& child : node.children_in_hit_test_order) { if (descendents.find(child) == descendents.end()) { to_process.push_back(child); } else { @@ -180,10 +180,18 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( for (const auto& value : update) { size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node); const auto& flutter_node = value.second; - nodes_[flutter_node.id] = - std::vector(flutter_node.childrenInTraversalOrder); + // Store the nodes for later hit testing. + nodes_[flutter_node.id] = { + .id = flutter_node.id, + .flags = flutter_node.flags, + .rect = flutter_node.rect, + .transform = flutter_node.transform, + .children_in_hit_test_order = flutter_node.childrenInHitTestOrder, + }; fuchsia::accessibility::semantics::Node fuchsia_node; std::vector child_ids; + // Send the nodes in traversal order, so the manager can figure out + // traversal. for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) { child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id)); } @@ -221,6 +229,7 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( } PruneUnreachableNodes(); + UpdateScreenRects(); tree_ptr_->UpdateSemanticNodes(std::move(nodes)); // TODO(dnfield): Implement the callback here @@ -228,6 +237,51 @@ void AccessibilityBridge::AddSemanticsNodeUpdate( tree_ptr_->CommitUpdates([]() {}); } +void AccessibilityBridge::UpdateScreenRects() { + std::unordered_set visited_nodes; + UpdateScreenRects(kRootNodeId, SkMatrix44::I(), &visited_nodes); +} + +void AccessibilityBridge::UpdateScreenRects( + int32_t node_id, + SkMatrix44 parent_transform, + std::unordered_set* visited_nodes) { + auto it = nodes_.find(node_id); + if (it == nodes_.end()) { + FML_LOG(ERROR) << "UpdateScreenRects called on unknown node"; + return; + } + auto& node = it->second; + const auto& current_transform = parent_transform * node.transform; + + const auto& rect = node.rect; + FML_LOG(ERROR) << "nodeid: " << node_id; + SkScalar quad[] = { + rect.left(), rect.top(), // + rect.right(), rect.top(), // + rect.right(), rect.bottom(), // + rect.left(), rect.bottom(), // + }; + SkScalar dst[4 * 4]; + current_transform.map2(quad, 4, dst); + node.screen_rect.setLTRB(dst[0], dst[1], dst[8], dst[9]); + node.screen_rect.sort(); + std::vector points = { + current_transform * SkVector4(rect.left(), rect.top(), 0, 1), + current_transform * SkVector4(rect.right(), rect.top(), 0, 1), + current_transform * SkVector4(rect.right(), rect.bottom(), 0, 1), + current_transform * SkVector4(rect.left(), rect.bottom(), 0, 1), + }; + + visited_nodes->emplace(node_id); + + for (uint32_t child_id : node.children_in_hit_test_order) { + if (visited_nodes->find(child_id) == visited_nodes->end()) { + UpdateScreenRects(child_id, current_transform, visited_nodes); + } + } +} + // |fuchsia::accessibility::semantics::SemanticListener| void AccessibilityBridge::OnAccessibilityActionRequested( uint32_t node_id, @@ -239,7 +293,34 @@ void AccessibilityBridge::OnAccessibilityActionRequested( void AccessibilityBridge::HitTest( fuchsia::math::PointF local_point, fuchsia::accessibility::semantics::SemanticListener::HitTestCallback - callback) {} + callback) { + auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y); + FML_DCHECK(hit_node_id.has_value()); + fuchsia::accessibility::semantics::Hit hit; + hit.set_node_id(hit_node_id.value_or(kRootNodeId)); + callback(std::move(hit)); +} + +std::optional AccessibilityBridge::GetHitNode(int32_t node_id, + float x, + float y) { + auto it = nodes_.find(node_id); + if (it == nodes_.end()) { + FML_LOG(ERROR) << "Attempted to hit test unkonwn node id: " << node_id; + return {}; + } + auto const& node = it->second; + if (node.flags & + static_cast(flutter::SemanticsFlags::kIsHidden) || // + !node.screen_rect.contains(x, y)) { + return {}; + } + auto hit = node_id; + for (int32_t child_id : node.children_in_hit_test_order) { + hit = GetHitNode(child_id, x, y).value_or(hit); + } + return hit; +} // |fuchsia::accessibility::semantics::SemanticListener| void AccessibilityBridge::OnSemanticsModeChanged( diff --git a/shell/platform/fuchsia/flutter/accessibility_bridge.h b/shell/platform/fuchsia/flutter/accessibility_bridge.h index 58d53933b47f9..4072c9696ecfe 100644 --- a/shell/platform/fuchsia/flutter/accessibility_bridge.h +++ b/shell/platform/fuchsia/flutter/accessibility_bridge.h @@ -85,7 +85,24 @@ class AccessibilityBridge // Notifies the bridge of a 'hover move' touch exploration event. zx_status_t OnHoverMove(double x, double y); + // |fuchsia::accessibility::semantics::SemanticListener| + void HitTest( + fuchsia::math::PointF local_point, + fuchsia::accessibility::semantics::SemanticListener::HitTestCallback + callback) override; + private: + // Holds only the fields we need for hit testing. + // In particular, it adds a screen_rect field to flutter::SemanticsNode. + struct SemanticsNode { + int32_t id; + int32_t flags; + SkRect rect; + SkRect screen_rect; + SkMatrix44 transform; + std::vector children_in_hit_test_order; + }; + AccessibilityBridge::Delegate& delegate_; static constexpr int32_t kRootNodeId = 0; @@ -95,8 +112,8 @@ class AccessibilityBridge fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_; bool semantics_enabled_; // This is the cache of all nodes we've sent to Fuchsia's SemanticsManager. - // Assists with pruning unreachable nodes. - std::unordered_map> nodes_; + // Assists with pruning unreachable nodes and hit testing. + std::unordered_map nodes_; // Derives the BoundingBox of a Flutter semantics node from its // rect and elevation. @@ -127,6 +144,26 @@ class AccessibilityBridge // May result in a call to FuchsiaAccessibility::Commit(). void PruneUnreachableNodes(); + // Updates the on-screen positions of accessibility elements, + // starting from the root element with an identity matrix. + // + // This should be called from Update. + void UpdateScreenRects(); + + // Updates the on-screen positions of accessibility elements, starting + // from node_id and using the specified transform. + // + // Update calls this via UpdateScreenRects(). + void UpdateScreenRects(int32_t node_id, + SkMatrix44 parent_transform, + std::unordered_set* visited_nodes); + + // Traverses the semantics tree to find the node_id hit by the given x,y + // point. + // + // Assumes that SemanticsNode::screen_rect is up to date. + std::optional GetHitNode(int32_t node_id, float x, float y); + // |fuchsia::accessibility::semantics::SemanticListener| void OnAccessibilityActionRequested( uint32_t node_id, @@ -134,12 +171,6 @@ class AccessibilityBridge fuchsia::accessibility::semantics::SemanticListener:: OnAccessibilityActionRequestedCallback callback) override; - // |fuchsia::accessibility::semantics::SemanticListener| - void HitTest( - fuchsia::math::PointF local_point, - fuchsia::accessibility::semantics::SemanticListener::HitTestCallback - callback) override; - // |fuchsia::accessibility::semantics::SemanticListener| void OnSemanticsModeChanged(bool enabled, OnSemanticsModeChangedCallback callback) override; diff --git a/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc b/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc index d5c773e62daca..ef2d619271661 100644 --- a/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc +++ b/shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc @@ -90,10 +90,12 @@ TEST_F(AccessibilityBridgeTest, DeletesChildrenTransitively) { flutter::SemanticsNode node1; node1.id = 1; node1.childrenInTraversalOrder = {2}; + node1.childrenInHitTestOrder = {2}; flutter::SemanticsNode node0; node0.id = 0; node0.childrenInTraversalOrder = {1}; + node0.childrenInHitTestOrder = {1}; accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, @@ -112,6 +114,7 @@ TEST_F(AccessibilityBridgeTest, DeletesChildrenTransitively) { // Remove the children node0.childrenInTraversalOrder.clear(); + node0.childrenInHitTestOrder.clear(); accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, }); @@ -141,6 +144,7 @@ TEST_F(AccessibilityBridgeTest, TruncatesLargeLabel) { std::string(fuchsia::accessibility::semantics::MAX_LABEL_SIZE + 1, '2'); node0.childrenInTraversalOrder = {1, 2}; + node0.childrenInHitTestOrder = {1, 2}; accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, @@ -194,7 +198,9 @@ TEST_F(AccessibilityBridgeTest, SplitsLargeUpdates) { std::string(fuchsia::accessibility::semantics::MAX_LABEL_SIZE, '4'); node0.childrenInTraversalOrder = {1, 2}; + node0.childrenInHitTestOrder = {1, 2}; node1.childrenInTraversalOrder = {3, 4}; + node1.childrenInHitTestOrder = {3, 4}; accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, @@ -219,6 +225,7 @@ TEST_F(AccessibilityBridgeTest, HandlesCycles) { flutter::SemanticsNode node0; node0.id = 0; node0.childrenInTraversalOrder.push_back(0); + node0.childrenInHitTestOrder.push_back(0); accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, }); @@ -231,9 +238,11 @@ TEST_F(AccessibilityBridgeTest, HandlesCycles) { EXPECT_FALSE(semantics_manager_.UpdateOverflowed()); node0.childrenInTraversalOrder = {0, 1}; + node0.childrenInHitTestOrder = {0, 1}; flutter::SemanticsNode node1; node1.id = 1; node1.childrenInTraversalOrder = {0}; + node1.childrenInHitTestOrder = {0}; accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, {1, node1}, @@ -260,12 +269,14 @@ TEST_F(AccessibilityBridgeTest, BatchesLargeMessages) { flutter::SemanticsNode node; node.id = i; node0.childrenInTraversalOrder.push_back(i); + node0.childrenInHitTestOrder.push_back(i); for (int32_t j = 0; j < leaf_nodes; j++) { flutter::SemanticsNode leaf_node; int id = (i * child_nodes) + ((j + 1) * leaf_nodes); leaf_node.id = id; leaf_node.label = "A relatively simple label"; node.childrenInTraversalOrder.push_back(id); + node.childrenInHitTestOrder.push_back(id); update.insert(std::make_pair(id, std::move(leaf_node))); } update.insert(std::make_pair(i, std::move(node))); @@ -283,6 +294,7 @@ TEST_F(AccessibilityBridgeTest, BatchesLargeMessages) { // Remove the children node0.childrenInTraversalOrder.clear(); + node0.childrenInHitTestOrder.clear(); accessibility_bridge_->AddSemanticsNodeUpdate({ {0, node0}, }); @@ -294,4 +306,62 @@ TEST_F(AccessibilityBridgeTest, BatchesLargeMessages) { EXPECT_FALSE(semantics_manager_.DeleteOverflowed()); EXPECT_FALSE(semantics_manager_.UpdateOverflowed()); } + +TEST_F(AccessibilityBridgeTest, HitTest) { + flutter::SemanticsNode node0; + node0.id = 0; + node0.rect.setLTRB(0, 0, 100, 100); + + flutter::SemanticsNode node1; + node1.id = 1; + node1.rect.setLTRB(10, 10, 20, 20); + + flutter::SemanticsNode node2; + node2.id = 2; + node2.rect.setLTRB(25, 10, 45, 20); + + flutter::SemanticsNode node3; + node3.id = 3; + node3.rect.setLTRB(10, 25, 20, 45); + + flutter::SemanticsNode node4; + node4.id = 4; + node4.rect.setLTRB(10, 10, 20, 20); + node4.transform.setTranslate(20, 20, 0); + + node0.childrenInTraversalOrder = {1, 2, 3, 4}; + node0.childrenInHitTestOrder = {1, 2, 3, 4}; + + accessibility_bridge_->AddSemanticsNodeUpdate({ + {0, node0}, + {1, node1}, + {2, node2}, + {3, node3}, + {4, node4}, + }); + RunLoopUntilIdle(); + + uint32_t hit_node_id; + auto callback = [&hit_node_id](fuchsia::accessibility::semantics::Hit hit) { + EXPECT_TRUE(hit.has_node_id()); + hit_node_id = hit.node_id(); + }; + + // Nodes are: + // ---------- + // | 1 2 | + // | 3 4 | + // ---------- + + accessibility_bridge_->HitTest({1, 1}, callback); + EXPECT_EQ(hit_node_id, 0u); + accessibility_bridge_->HitTest({15, 15}, callback); + EXPECT_EQ(hit_node_id, 1u); + accessibility_bridge_->HitTest({30, 15}, callback); + EXPECT_EQ(hit_node_id, 2u); + accessibility_bridge_->HitTest({15, 30}, callback); + EXPECT_EQ(hit_node_id, 3u); + accessibility_bridge_->HitTest({30, 30}, callback); + EXPECT_EQ(hit_node_id, 4u); +} } // namespace flutter_runner_test