[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / 1ba3a622b3d80d630443b8c38ba7fc60667ea134 / . / chrome / renderer / accessibility / ax_tree_distiller.cc
blob: 3dadd45796a8c0c912c1e18f6cbd6965dc244545 [file] [log] [blame]
Avi Drissman4e1b7bc2022-09-15 14:03:50[diff] [blame]1// Copyright 2022 The Chromium Authors
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]5#include "chrome/renderer/accessibility/ax_tree_distiller.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]6
Abigail Kleine8c97f22022-04-04 21:39:11[diff] [blame]7#include <memory>
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]8#include <queue>
Abigail Kleine9c8e3c2022-07-22 15:47:25[diff] [blame]9#include <utility>
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]10#include <vector>
11
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]12#include "base/containers/contains.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]13#include "base/strings/utf_string_conversions.h"
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]14#include "content/public/renderer/render_frame.h"
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]15#include "content/public/renderer/render_thread.h"
16#include "services/metrics/public/cpp/mojo_ukm_recorder.h"
17#include "services/metrics/public/cpp/ukm_builders.h"
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]18#include "third_party/blink/public/common/browser_interface_broker_proxy.h"
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]19#include "ui/accessibility/accessibility_features.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]20#include "ui/accessibility/ax_node.h"
21#include "ui/accessibility/ax_tree.h"
22
23namespace {
24
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]25// TODO: Consider moving this to AXNodeProperties.
26static const ax::mojom::Role kContentRoles[]{
27 ax::mojom::Role::kHeading,
28 ax::mojom::Role::kParagraph,
Kristi Saneya99fff32023-05-17 17:32:15[diff] [blame]29 ax::mojom::Role::kNote,
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]30};
31
32// TODO: Consider moving this to AXNodeProperties.
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]33static const ax::mojom::Role kRolesToSkip[]{
34 ax::mojom::Role::kAudio,
35 ax::mojom::Role::kBanner,
36 ax::mojom::Role::kButton,
37 ax::mojom::Role::kComplementary,
38 ax::mojom::Role::kContentInfo,
39 ax::mojom::Role::kFooter,
40 ax::mojom::Role::kFooterAsNonLandmark,
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]41 ax::mojom::Role::kImage,
42 ax::mojom::Role::kLabelText,
43 ax::mojom::Role::kNavigation,
44};
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]45
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]46// Find all of the main and article nodes.
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]47// TODO(crbug.com/1266555): Replace this with a call to
48// OneShotAccessibilityTreeSearch.
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]49void GetContentRootNodes(const ui::AXNode* root,
50 std::vector<const ui::AXNode*>* content_root_nodes) {
Kristi Saney414dd79b2023-04-20 20:45:44[diff] [blame]51 if (!root) {
52 return;
53 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]54 std::queue<const ui::AXNode*> queue;
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]55 queue.push(root);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]56 while (!queue.empty()) {
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]57 const ui::AXNode* node = queue.front();
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]58 queue.pop();
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]59 // If a main or article node is found, add it to the list of content root
60 // nodes and continue. Do not explore children for nested article nodes.
61 if (node->GetRole() == ax::mojom::Role::kMain ||
62 node->GetRole() == ax::mojom::Role::kArticle) {
63 content_root_nodes->push_back(node);
64 continue;
65 }
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]66 for (auto iter = node->UnignoredChildrenBegin();
67 iter != node->UnignoredChildrenEnd(); ++iter) {
68 queue.push(iter.get());
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]69 }
70 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]71}
72
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]73// Recurse through the root node, searching for content nodes (any node whose
74// role is in kContentRoles). Skip branches which begin with a node with role
75// in kRolesToSkip. Once a content node is identified, add it to the vector
76// |content_node_ids|, whose pointer is passed through the recursion.
77void AddContentNodesToVector(const ui::AXNode* node,
78 std::vector<ui::AXNodeID>* content_node_ids) {
79 if (base::Contains(kContentRoles, node->GetRole())) {
80 content_node_ids->emplace_back(node->id());
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]81 return;
82 }
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]83 if (base::Contains(kRolesToSkip, node->GetRole()))
84 return;
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]85 for (auto iter = node->UnignoredChildrenBegin();
86 iter != node->UnignoredChildrenEnd(); ++iter) {
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]87 AddContentNodesToVector(iter.get(), content_node_ids);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]88 }
89}
90
91} // namespace
92
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]93AXTreeDistiller::AXTreeDistiller(
94 content::RenderFrame* render_frame,
95 OnAXTreeDistilledCallback on_ax_tree_distilled_callback)
96 : render_frame_(render_frame),
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]97 on_ax_tree_distilled_callback_(on_ax_tree_distilled_callback) {
98 // TODO(crbug.com/1450930): Use a global ukm recorder instance instead.
99 mojo::Remote<ukm::mojom::UkmRecorderFactory> factory;
100 content::RenderThread::Get()->BindHostReceiver(
101 factory.BindNewPipeAndPassReceiver());
102 ukm_recorder_ = ukm::MojoUkmRecorder::Create(*factory);
103}
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]104
105AXTreeDistiller::~AXTreeDistiller() = default;
106
Abigail Kleinfdeb3d72023-01-19 15:20:19[diff] [blame]107void AXTreeDistiller::Distill(const ui::AXTree& tree,
Abigail Kleindfdde352023-01-27 21:03:10[diff] [blame]108 const ui::AXTreeUpdate& snapshot,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]109 const ukm::SourceId ukm_source_id) {
110#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
111 base::TimeTicks start_time = base::TimeTicks::Now();
112#endif
Jacob Francis1ba3a622023-11-21 22:18:21[diff] [blame^]113
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]114 std::vector<ui::AXNodeID> content_node_ids;
Jacob Francis1ba3a622023-11-21 22:18:21[diff] [blame^]115 if (features::IsReadAnythingWithAlgorithmEnabled()) {
116 // Try with the algorithm first.
117 DistillViaAlgorithm(tree, ukm_source_id, &content_node_ids);
118 }
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]119
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]120 // If Read Anything with Screen 2x is enabled and the main content extractor
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]121 // is bound, kick off Screen 2x run, which distills the AXTree in the
122 // utility process using ML.
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]123#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]124 if (features::IsReadAnythingWithScreen2xEnabled() &&
125 main_content_extractor_.is_bound()) {
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]126 DistillViaScreen2x(tree, snapshot, ukm_source_id, start_time,
127 &content_node_ids);
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]128 return;
129 }
130#endif
131
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]132 // Ensure we still callback if Screen2x is not available.
133 on_ax_tree_distilled_callback_.Run(tree.GetAXTreeID(), content_node_ids);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]134}
135
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]136void AXTreeDistiller::DistillViaAlgorithm(
137 const ui::AXTree& tree,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]138 const ukm::SourceId ukm_source_id,
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]139 std::vector<ui::AXNodeID>* content_node_ids) {
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]140 base::TimeTicks start_time = base::TimeTicks::Now();
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]141 std::vector<const ui::AXNode*> content_root_nodes;
Abigail Kleinfdeb3d72023-01-19 15:20:19[diff] [blame]142 GetContentRootNodes(tree.root(), &content_root_nodes);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]143 for (const ui::AXNode* content_root_node : content_root_nodes) {
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]144 AddContentNodesToVector(content_root_node, content_node_ids);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]145 }
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]146 RecordRulesMetrics(ukm_source_id, base::TimeTicks::Now() - start_time,
147 !content_node_ids->empty());
148}
149
150void AXTreeDistiller::RecordRulesMetrics(ukm::SourceId ukm_source_id,
151 base::TimeDelta elapsed_time,
152 bool success) {
153 if (success) {
154 base::UmaHistogramTimes(
155 "Accessibility.ReadAnything.RulesDistillationTime.Success",
156 elapsed_time);
157 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
158 .SetRulesDistillationTime_Success(elapsed_time.InMilliseconds())
159 .Record(ukm_recorder_.get());
160 } else {
161 base::UmaHistogramTimes(
162 "Accessibility.ReadAnything.RulesDistillationTime.Failure",
163 elapsed_time);
164 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
165 .SetRulesDistillationTime_Failure(elapsed_time.InMilliseconds())
166 .Record(ukm_recorder_.get());
167 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]168}
169
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]170#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]171void AXTreeDistiller::DistillViaScreen2x(
172 const ui::AXTree& tree,
173 const ui::AXTreeUpdate& snapshot,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]174 const ukm::SourceId ukm_source_id,
175 base::TimeTicks start_time,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]176 std::vector<ui::AXNodeID>* content_node_ids_algorithm) {
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]177 DCHECK(main_content_extractor_.is_bound());
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]178 // Make a copy of |content_node_ids_algorithm| rather than sending a pointer.
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]179 main_content_extractor_->ExtractMainContent(
Abigail Kleindfdde352023-01-27 21:03:10[diff] [blame]180 snapshot, ukm_source_id,
Abigail Klein34db57e2023-05-01 22:37:37[diff] [blame]181 base::BindOnce(&AXTreeDistiller::ProcessScreen2xResult,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]182 weak_ptr_factory_.GetWeakPtr(), tree.GetAXTreeID(),
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]183 ukm_source_id, start_time, *content_node_ids_algorithm));
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]184}
185
186void AXTreeDistiller::ProcessScreen2xResult(
Abigail Klein34db57e2023-05-01 22:37:37[diff] [blame]187 const ui::AXTreeID& tree_id,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]188 const ukm::SourceId ukm_source_id,
189 base::TimeTicks start_time,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]190 std::vector<ui::AXNodeID> content_node_ids_algorithm,
191 const std::vector<ui::AXNodeID>& content_node_ids_screen2x) {
192 // Merge the results from the algorithm and from screen2x.
193 for (ui::AXNodeID content_node_id_screen2x : content_node_ids_screen2x) {
194 if (!base::Contains(content_node_ids_algorithm, content_node_id_screen2x)) {
195 content_node_ids_algorithm.push_back(content_node_id_screen2x);
196 }
197 }
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]198 RecordMergedMetrics(ukm_source_id, base::TimeTicks::Now() - start_time,
199 !content_node_ids_algorithm.empty());
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]200 on_ax_tree_distilled_callback_.Run(tree_id, content_node_ids_algorithm);
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]201
Kristi Saney84a78642023-05-10 18:41:54[diff] [blame]202 // TODO(crbug.com/1266555): If no content nodes were identified, and
Abigail Kleind5fef9a2022-11-07 15:03:50[diff] [blame]203 // there is a selection, try sending Screen2x a partial tree just containing
204 // the selected nodes.
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]205}
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]206
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]207void AXTreeDistiller::ScreenAIServiceReady() {
208 if (main_content_extractor_.is_bound()) {
209 return;
210 }
211 render_frame_->GetBrowserInterfaceBroker()->GetInterface(
212 main_content_extractor_.BindNewPipeAndPassReceiver());
213 main_content_extractor_.set_disconnect_handler(
214 base::BindOnce(&AXTreeDistiller::OnMainContentExtractorDisconnected,
215 weak_ptr_factory_.GetWeakPtr()));
216}
217
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]218void AXTreeDistiller::OnMainContentExtractorDisconnected() {
Abigail Klein3bd99e02023-01-24 00:29:18[diff] [blame]219 on_ax_tree_distilled_callback_.Run(ui::AXTreeIDUnknown(),
220 std::vector<ui::AXNodeID>());
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]221}
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]222
223void AXTreeDistiller::RecordMergedMetrics(ukm::SourceId ukm_source_id,
224 base::TimeDelta elapsed_time,
225 bool success) {
226 if (success) {
227 base::UmaHistogramTimes(
228 "Accessibility.ReadAnything.MergedDistillationTime.Success",
229 elapsed_time);
230 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
231 .SetMergedDistillationTime_Success(elapsed_time.InMilliseconds())
232 .Record(ukm_recorder_.get());
233 } else {
234 base::UmaHistogramTimes(
235 "Accessibility.ReadAnything.MergedDistillationTime.Failure",
236 elapsed_time);
237 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
238 .SetMergedDistillationTime_Failure(elapsed_time.InMilliseconds())
239 .Record(ukm_recorder_.get());
240 }
241}
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]242#endif