[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / ad097706e44d63dacd97d2ebad898e34ff2ca2f9 / . / chrome / renderer / accessibility / ax_tree_distiller.cc
blob: ed1f43549214c2b5be656f465b5c9775e280ce06 [file] [log] [blame]
Avi Drissman4e1b7bc2022-09-15 14:03:50[diff] [blame]1// Copyright 2022 The Chromium Authors
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]5#include "chrome/renderer/accessibility/ax_tree_distiller.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]6
Abigail Kleine8c97f22022-04-04 21:39:11[diff] [blame]7#include <memory>
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]8#include <queue>
Abigail Kleine9c8e3c2022-07-22 15:47:25[diff] [blame]9#include <utility>
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]10#include <vector>
11
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]12#include "base/containers/contains.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]13#include "base/strings/utf_string_conversions.h"
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]14#include "content/public/renderer/render_frame.h"
15#include "third_party/blink/public/common/browser_interface_broker_proxy.h"
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]16#include "ui/accessibility/accessibility_features.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]17#include "ui/accessibility/ax_node.h"
18#include "ui/accessibility/ax_tree.h"
19
20namespace {
21
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]22// TODO: Consider moving this to AXNodeProperties.
23static const ax::mojom::Role kContentRoles[]{
24 ax::mojom::Role::kHeading,
25 ax::mojom::Role::kParagraph,
Kristi Saneya99fff32023-05-17 17:32:15[diff] [blame]26 ax::mojom::Role::kNote,
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]27};
28
29// TODO: Consider moving this to AXNodeProperties.
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]30static const ax::mojom::Role kRolesToSkip[]{
31 ax::mojom::Role::kAudio,
32 ax::mojom::Role::kBanner,
33 ax::mojom::Role::kButton,
34 ax::mojom::Role::kComplementary,
35 ax::mojom::Role::kContentInfo,
36 ax::mojom::Role::kFooter,
37 ax::mojom::Role::kFooterAsNonLandmark,
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]38 ax::mojom::Role::kImage,
39 ax::mojom::Role::kLabelText,
40 ax::mojom::Role::kNavigation,
41};
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]42
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]43// Find all of the main and article nodes.
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]44// TODO(crbug.com/1266555): Replace this with a call to
45// OneShotAccessibilityTreeSearch.
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]46void GetContentRootNodes(const ui::AXNode* root,
47 std::vector<const ui::AXNode*>* content_root_nodes) {
Kristi Saney414dd79b2023-04-20 20:45:44[diff] [blame]48 if (!root) {
49 return;
50 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]51 std::queue<const ui::AXNode*> queue;
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]52 queue.push(root);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]53 while (!queue.empty()) {
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]54 const ui::AXNode* node = queue.front();
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]55 queue.pop();
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]56 // If a main or article node is found, add it to the list of content root
57 // nodes and continue. Do not explore children for nested article nodes.
58 if (node->GetRole() == ax::mojom::Role::kMain ||
59 node->GetRole() == ax::mojom::Role::kArticle) {
60 content_root_nodes->push_back(node);
61 continue;
62 }
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]63 for (auto iter = node->UnignoredChildrenBegin();
64 iter != node->UnignoredChildrenEnd(); ++iter) {
65 queue.push(iter.get());
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]66 }
67 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]68}
69
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]70// Recurse through the root node, searching for content nodes (any node whose
71// role is in kContentRoles). Skip branches which begin with a node with role
72// in kRolesToSkip. Once a content node is identified, add it to the vector
73// |content_node_ids|, whose pointer is passed through the recursion.
74void AddContentNodesToVector(const ui::AXNode* node,
75 std::vector<ui::AXNodeID>* content_node_ids) {
76 if (base::Contains(kContentRoles, node->GetRole())) {
77 content_node_ids->emplace_back(node->id());
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]78 return;
79 }
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]80 if (base::Contains(kRolesToSkip, node->GetRole()))
81 return;
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]82 for (auto iter = node->UnignoredChildrenBegin();
83 iter != node->UnignoredChildrenEnd(); ++iter) {
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]84 AddContentNodesToVector(iter.get(), content_node_ids);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]85 }
86}
87
88} // namespace
89
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]90AXTreeDistiller::AXTreeDistiller(
91 content::RenderFrame* render_frame,
92 OnAXTreeDistilledCallback on_ax_tree_distilled_callback)
93 : render_frame_(render_frame),
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]94 on_ax_tree_distilled_callback_(on_ax_tree_distilled_callback) {}
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]95
96AXTreeDistiller::~AXTreeDistiller() = default;
97
Abigail Kleinfdeb3d72023-01-19 15:20:19[diff] [blame]98void AXTreeDistiller::Distill(const ui::AXTree& tree,
Abigail Kleindfdde352023-01-27 21:03:10[diff] [blame]99 const ui::AXTreeUpdate& snapshot,
100 const ukm::SourceId& ukm_source_id) {
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]101 // Try with the algorithm first.
102 std::vector<ui::AXNodeID> content_node_ids;
103 DistillViaAlgorithm(tree, &content_node_ids);
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]104
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]105 // If Read Anything with Screen 2x is enabled and the main content extractor
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]106 // is bound, kick off Screen 2x run, which distills the AXTree in the
107 // utility process using ML.
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]108#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]109 if (features::IsReadAnythingWithScreen2xEnabled() &&
110 main_content_extractor_.is_bound()) {
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame^]111 DistillViaScreen2x(tree, snapshot, ukm_source_id, &content_node_ids);
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]112 return;
113 }
114#endif
115
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]116 // Ensure we still callback if Screen2x is not available.
117 on_ax_tree_distilled_callback_.Run(tree.GetAXTreeID(), content_node_ids);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]118}
119
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]120void AXTreeDistiller::DistillViaAlgorithm(
121 const ui::AXTree& tree,
122 std::vector<ui::AXNodeID>* content_node_ids) {
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]123 std::vector<const ui::AXNode*> content_root_nodes;
Abigail Kleinfdeb3d72023-01-19 15:20:19[diff] [blame]124 GetContentRootNodes(tree.root(), &content_root_nodes);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]125 for (const ui::AXNode* content_root_node : content_root_nodes) {
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]126 AddContentNodesToVector(content_root_node, content_node_ids);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]127 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]128}
129
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]130#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame^]131void AXTreeDistiller::DistillViaScreen2x(
132 const ui::AXTree& tree,
133 const ui::AXTreeUpdate& snapshot,
134 const ukm::SourceId& ukm_source_id,
135 std::vector<ui::AXNodeID>* content_node_ids_algorithm) {
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]136 DCHECK(main_content_extractor_.is_bound());
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame^]137 // Make a copy of |content_node_ids_algorithm| rather than sending a pointer.
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]138 main_content_extractor_->ExtractMainContent(
Abigail Kleindfdde352023-01-27 21:03:10[diff] [blame]139 snapshot, ukm_source_id,
Abigail Klein34db57e2023-05-01 22:37:37[diff] [blame]140 base::BindOnce(&AXTreeDistiller::ProcessScreen2xResult,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame^]141 weak_ptr_factory_.GetWeakPtr(), tree.GetAXTreeID(),
142 *content_node_ids_algorithm));
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]143}
144
145void AXTreeDistiller::ProcessScreen2xResult(
Abigail Klein34db57e2023-05-01 22:37:37[diff] [blame]146 const ui::AXTreeID& tree_id,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame^]147 std::vector<ui::AXNodeID> content_node_ids_algorithm,
148 const std::vector<ui::AXNodeID>& content_node_ids_screen2x) {
149 // Merge the results from the algorithm and from screen2x.
150 for (ui::AXNodeID content_node_id_screen2x : content_node_ids_screen2x) {
151 if (!base::Contains(content_node_ids_algorithm, content_node_id_screen2x)) {
152 content_node_ids_algorithm.push_back(content_node_id_screen2x);
153 }
154 }
155 on_ax_tree_distilled_callback_.Run(tree_id, content_node_ids_algorithm);
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]156
Kristi Saney84a78642023-05-10 18:41:54[diff] [blame]157 // TODO(crbug.com/1266555): If no content nodes were identified, and
Abigail Kleind5fef9a2022-11-07 15:03:50[diff] [blame]158 // there is a selection, try sending Screen2x a partial tree just containing
159 // the selected nodes.
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]160}
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]161
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]162void AXTreeDistiller::ScreenAIServiceReady() {
163 if (main_content_extractor_.is_bound()) {
164 return;
165 }
166 render_frame_->GetBrowserInterfaceBroker()->GetInterface(
167 main_content_extractor_.BindNewPipeAndPassReceiver());
168 main_content_extractor_.set_disconnect_handler(
169 base::BindOnce(&AXTreeDistiller::OnMainContentExtractorDisconnected,
170 weak_ptr_factory_.GetWeakPtr()));
171}
172
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]173void AXTreeDistiller::OnMainContentExtractorDisconnected() {
Abigail Klein3bd99e02023-01-24 00:29:18[diff] [blame]174 on_ax_tree_distilled_callback_.Run(ui::AXTreeIDUnknown(),
175 std::vector<ui::AXNodeID>());
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]176}
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]177#endif