HitTesting for fuchsia a11y

flutter · Jan 14, 2020 · 1405b2b · 1405b2b
1 parent 0535b5e
commit 1405b2b
Show file tree

Hide file tree

Showing 2 changed files with 96 additions and 6 deletions.
diff --git a/shell/platform/fuchsia/flutter/accessibility_bridge.cc b/shell/platform/fuchsia/flutter/accessibility_bridge.cc
@@ -99,8 +99,8 @@ std::unordered_set<int32_t> AccessibilityBridge::GetDescendants(
 
     auto it = nodes_.find(id);
     if (it != nodes_.end()) {
-      auto const& children = it->second;
-      for (const auto& child : children) {
+      auto const& node = it->second;
+      for (const auto& child : node.children_in_hit_test_order) {
         if (descendents.find(child) == descendents.end()) {
           to_process.push_back(child);
         } else {
@@ -180,10 +180,19 @@ void AccessibilityBridge::AddSemanticsNodeUpdate(
   for (const auto& value : update) {
     size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node);
     const auto& flutter_node = value.second;
-    nodes_[flutter_node.id] =
-        std::vector<int32_t>(flutter_node.childrenInTraversalOrder);
+    // Store the nodes in hitTestOrder for later hit testing.
+    nodes_[flutter_node.id] = {
+        .id = flutter_node.id,
+        .flags = flutter_node.flags,
+        .rect = flutter_node.rect,
+        .transform = flutter_node.transform,
+        .children_in_hit_test_order =
+            std::vector<int32_t>(flutter_node.childrenInHitTestOrder),
+    };
     fuchsia::accessibility::semantics::Node fuchsia_node;
     std::vector<uint32_t> child_ids;
+    // Send the nodes in traversal order, so the manager can figure out
+    // traversal.
     for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) {
       child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id));
     }
@@ -221,13 +230,48 @@ void AccessibilityBridge::AddSemanticsNodeUpdate(
   }
 
   PruneUnreachableNodes();
+  UpdateScreenRects(kRootNodeId, SkMatrix44::I());
 
   tree_ptr_->UpdateSemanticNodes(std::move(nodes));
   // TODO(dnfield): Implement the callback here
   // https://bugs.fuchsia.dev/p/fuchsia/issues/detail?id=35718.
   tree_ptr_->CommitUpdates([]() {});
 }
 
+void AccessibilityBridge::UpdateScreenRects(int32_t node_id,
+                                            SkMatrix44 parent_transform) {
+  auto it = nodes_.find(node_id);
+  if (it == nodes_.end()) {
+    FML_LOG(ERROR) << "UpdateScreenRects called on unknown node";
+  }
+  auto& node = it->second;
+  const auto& current_transform = parent_transform * node.transform;
+
+  const auto& rect = node.rect;
+  std::vector<SkVector4> points = {
+      current_transform * SkVector4(rect.left(), rect.top(), 0, 1),
+      current_transform * SkVector4(rect.right(), rect.top(), 0, 1),
+      current_transform * SkVector4(rect.right(), rect.bottom(), 0, 1),
+      current_transform * SkVector4(rect.left(), rect.bottom(), 0, 1),
+  };
+
+  std::vector<double> x_vals;
+  std::vector<double> y_vals;
+  std::transform(points.begin(), points.end(), std::back_inserter(x_vals),
+                 [](SkVector4& point) { return point.fData[0]; });
+  std::transform(points.begin(), points.end(), std::back_inserter(y_vals),
+                 [](SkVector4& point) { return point.fData[1]; });
+
+  node.screen_rect.setLTRB(*std::min_element(x_vals.begin(), x_vals.end()),
+                           *std::min_element(y_vals.begin(), y_vals.end()),
+                           *std::max_element(x_vals.begin(), x_vals.end()),
+                           *std::max_element(y_vals.begin(), y_vals.end()));
+
+  for (uint32_t child_id : node.children_in_hit_test_order) {
+    UpdateScreenRects(child_id, current_transform);
+  }
+}
+
 // |fuchsia::accessibility::semantics::SemanticListener|
 void AccessibilityBridge::OnAccessibilityActionRequested(
     uint32_t node_id,
@@ -239,7 +283,34 @@ void AccessibilityBridge::OnAccessibilityActionRequested(
 void AccessibilityBridge::HitTest(
     fuchsia::math::PointF local_point,
     fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
-        callback) {}
+        callback) {
+  fuchsia::accessibility::semantics::Hit hit;
+  GetHitNode(kRootNodeId, local_point.x, local_point.y, &hit);
+  FML_DCHECK(hit.has_node_id());
+  callback(std::move(hit));
+}
+
+void AccessibilityBridge::GetHitNode(
+    int32_t node_id,
+    float x,
+    float y,
+    fuchsia::accessibility::semantics::Hit* hit) {
+  auto it = nodes_.find(node_id);
+  if (it == nodes_.end()) {
+    FML_LOG(ERROR) << "Attempted to hit test unkonwn node id: " << node_id;
+    return;
+  }
+  auto const& node = it->second;
+  if (node.flags &
+          static_cast<int32_t>(flutter::SemanticsFlags::kIsHidden) ||  //
+      !node.screen_rect.contains(x, y)) {
+    return;
+  }
+  hit->set_node_id(node_id);
+  for (int32_t child_id : node.children_in_hit_test_order) {
+    GetHitNode(child_id, x, y, hit);
+  }
+}
 
 // |fuchsia::accessibility::semantics::SemanticListener|
 void AccessibilityBridge::OnSemanticsModeChanged(

diff --git a/shell/platform/fuchsia/flutter/accessibility_bridge.h b/shell/platform/fuchsia/flutter/accessibility_bridge.h
@@ -86,6 +86,15 @@ class AccessibilityBridge
   zx_status_t OnHoverMove(double x, double y);
 
  private:
+  struct SemanticsNode {
+    int32_t id;
+    int32_t flags;
+    SkRect rect;
+    SkRect screen_rect;
+    SkMatrix44 transform;
+    std::vector<int32_t> children_in_hit_test_order;
+  };
+
   AccessibilityBridge::Delegate& delegate_;
 
   static constexpr int32_t kRootNodeId = 0;
@@ -96,7 +105,7 @@ class AccessibilityBridge
   bool semantics_enabled_;
   // This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
   // Assists with pruning unreachable nodes.
-  std::unordered_map<int32_t, std::vector<int32_t>> nodes_;
+  std::unordered_map<int32_t, SemanticsNode> nodes_;
 
   // Derives the BoundingBox of a Flutter semantics node from its
   // rect and elevation.
@@ -127,6 +136,16 @@ class AccessibilityBridge
   // May result in a call to FuchsiaAccessibility::Commit().
   void PruneUnreachableNodes();
 
+  // Updates the on-screen positions of accessibility elements.
+  //
+  // This should be called from Update
+  void UpdateScreenRects(int32_t node_id, SkMatrix44 parent_transform);
+
+  void GetHitNode(int32_t node_id,
+                  float x,
+                  float y,
+                  fuchsia::accessibility::semantics::Hit* hit);
+
   // |fuchsia::accessibility::semantics::SemanticListener|
   void OnAccessibilityActionRequested(
       uint32_t node_id,