[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / refs/tags/123.0.6312.47 / . / chrome / renderer / accessibility / ax_tree_distiller.cc
blob: 505149384a0669d0da741ae893d4047d34ba19bd [file] [log] [blame]
Avi Drissman4e1b7bc2022-09-15 14:03:50[diff] [blame]1// Copyright 2022 The Chromium Authors
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]5#include "chrome/renderer/accessibility/ax_tree_distiller.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]6
Abigail Kleine8c97f22022-04-04 21:39:11[diff] [blame]7#include <memory>
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]8#include <queue>
Abigail Kleine9c8e3c2022-07-22 15:47:25[diff] [blame]9#include <utility>
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]10#include <vector>
11
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]12#include "base/containers/contains.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]13#include "base/strings/utf_string_conversions.h"
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]14#include "content/public/renderer/render_frame.h"
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]15#include "content/public/renderer/render_thread.h"
16#include "services/metrics/public/cpp/mojo_ukm_recorder.h"
17#include "services/metrics/public/cpp/ukm_builders.h"
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]18#include "third_party/blink/public/common/browser_interface_broker_proxy.h"
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]19#include "ui/accessibility/accessibility_features.h"
Jacob Francise11099b2024-02-17 04:34:04[diff] [blame]20#include "ui/accessibility/ax_enums.mojom-shared.h"
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]21#include "ui/accessibility/ax_node.h"
22#include "ui/accessibility/ax_tree.h"
23
24namespace {
25
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]26// TODO: Consider moving this to AXNodeProperties.
27static const ax::mojom::Role kContentRoles[]{
Jacob Francise11099b2024-02-17 04:34:04[diff] [blame]28 ax::mojom::Role::kHeading, ax::mojom::Role::kParagraph,
29 ax::mojom::Role::kNote, ax::mojom::Role::kImage,
30 ax::mojom::Role::kFigcaption};
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]31
32// TODO: Consider moving this to AXNodeProperties.
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]33static const ax::mojom::Role kRolesToSkip[]{
34 ax::mojom::Role::kAudio,
35 ax::mojom::Role::kBanner,
36 ax::mojom::Role::kButton,
37 ax::mojom::Role::kComplementary,
38 ax::mojom::Role::kContentInfo,
39 ax::mojom::Role::kFooter,
40 ax::mojom::Role::kFooterAsNonLandmark,
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]41 ax::mojom::Role::kLabelText,
42 ax::mojom::Role::kNavigation,
43};
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]44
Abigail Kleinffd37a22023-11-29 21:52:14[diff] [blame]45// Find all of the main and article nodes. Also, include unignored heading nodes
46// which lie outside of the main and article node.
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]47// TODO(crbug.com/1266555): Replace this with a call to
48// OneShotAccessibilityTreeSearch.
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]49void GetContentRootNodes(const ui::AXNode* root,
50 std::vector<const ui::AXNode*>* content_root_nodes) {
Kristi Saney414dd79b2023-04-20 20:45:44[diff] [blame]51 if (!root) {
52 return;
53 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]54 std::queue<const ui::AXNode*> queue;
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]55 queue.push(root);
Abigail Kleinffd37a22023-11-29 21:52:14[diff] [blame]56 bool has_main_or_heading = false;
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]57 while (!queue.empty()) {
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]58 const ui::AXNode* node = queue.front();
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]59 queue.pop();
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]60 // If a main or article node is found, add it to the list of content root
61 // nodes and continue. Do not explore children for nested article nodes.
62 if (node->GetRole() == ax::mojom::Role::kMain ||
63 node->GetRole() == ax::mojom::Role::kArticle) {
64 content_root_nodes->push_back(node);
Abigail Kleinffd37a22023-11-29 21:52:14[diff] [blame]65 has_main_or_heading = true;
66 continue;
67 }
68 // If a heading node is found, add it to the list of content root nodes,
69 // too. It may be removed later if the tree doesn't contain a main or
70 // article node.
71 if (node->GetRole() == ax::mojom::Role::kHeading) {
72 content_root_nodes->push_back(node);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]73 continue;
74 }
Abigail Klein930f02c2022-11-07 15:03:50[diff] [blame]75 for (auto iter = node->UnignoredChildrenBegin();
76 iter != node->UnignoredChildrenEnd(); ++iter) {
77 queue.push(iter.get());
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]78 }
79 }
Abigail Kleinffd37a22023-11-29 21:52:14[diff] [blame]80 if (!has_main_or_heading) {
81 content_root_nodes->clear();
82 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]83}
84
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]85// Recurse through the root node, searching for content nodes (any node whose
86// role is in kContentRoles). Skip branches which begin with a node with role
87// in kRolesToSkip. Once a content node is identified, add it to the vector
88// |content_node_ids|, whose pointer is passed through the recursion.
89void AddContentNodesToVector(const ui::AXNode* node,
90 std::vector<ui::AXNodeID>* content_node_ids) {
Jacob Francise11099b2024-02-17 04:34:04[diff] [blame]91 const auto& role = node->GetRole();
92 if (base::Contains(kContentRoles, role)) {
93 // TODO(1464340): Remove when flag is no longer necessary. Skip these roles
94 // if the flag is not enabled.
95 if (!features::IsReadAnythingImagesViaAlgorithmEnabled() &&
96 (role == ax::mojom::Role::kFigcaption ||
97 role == ax::mojom::Role::kImage)) {
98 return;
99 }
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]100 content_node_ids->emplace_back(node->id());
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]101 return;
102 }
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]103 if (base::Contains(kRolesToSkip, node->GetRole()))
104 return;
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]105 for (auto iter = node->UnignoredChildrenBegin();
106 iter != node->UnignoredChildrenEnd(); ++iter) {
Abigail Klein277fe7c22022-04-11 21:38:45[diff] [blame]107 AddContentNodesToVector(iter.get(), content_node_ids);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]108 }
109}
110
111} // namespace
112
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]113AXTreeDistiller::AXTreeDistiller(
Abigail Klein1ed22242022-12-16 16:48:43[diff] [blame]114 OnAXTreeDistilledCallback on_ax_tree_distilled_callback)
Jocelyn Tran2696ed22024-01-17 16:47:14[diff] [blame]115 : on_ax_tree_distilled_callback_(on_ax_tree_distilled_callback) {
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]116 // TODO(crbug.com/1450930): Use a global ukm recorder instance instead.
117 mojo::Remote<ukm::mojom::UkmRecorderFactory> factory;
118 content::RenderThread::Get()->BindHostReceiver(
119 factory.BindNewPipeAndPassReceiver());
120 ukm_recorder_ = ukm::MojoUkmRecorder::Create(*factory);
121}
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]122
123AXTreeDistiller::~AXTreeDistiller() = default;
124
Abigail Kleinfdeb3d72023-01-19 15:20:19[diff] [blame]125void AXTreeDistiller::Distill(const ui::AXTree& tree,
Abigail Kleindfdde352023-01-27 21:03:10[diff] [blame]126 const ui::AXTreeUpdate& snapshot,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]127 const ukm::SourceId ukm_source_id) {
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]128 base::TimeTicks start_time = base::TimeTicks::Now();
Jacob Francis1ba3a622023-11-21 22:18:21[diff] [blame]129
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]130 std::vector<ui::AXNodeID> content_node_ids;
Jacob Francis1ba3a622023-11-21 22:18:21[diff] [blame]131 if (features::IsReadAnythingWithAlgorithmEnabled()) {
132 // Try with the algorithm first.
133 DistillViaAlgorithm(tree, ukm_source_id, &content_node_ids);
134 }
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]135
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]136 // If Read Anything with Screen 2x is enabled and the main content extractor
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]137 // is bound, kick off Screen 2x run, which distills the AXTree in the
138 // utility process using ML.
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]139 if (features::IsReadAnythingWithScreen2xEnabled() &&
140 main_content_extractor_.is_bound()) {
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]141 DistillViaScreen2x(tree, snapshot, ukm_source_id, start_time,
142 &content_node_ids);
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]143 return;
144 }
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]145
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]146 // Ensure we still callback if Screen2x is not available.
147 on_ax_tree_distilled_callback_.Run(tree.GetAXTreeID(), content_node_ids);
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]148}
149
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]150void AXTreeDistiller::DistillViaAlgorithm(
151 const ui::AXTree& tree,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]152 const ukm::SourceId ukm_source_id,
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]153 std::vector<ui::AXNodeID>* content_node_ids) {
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]154 base::TimeTicks start_time = base::TimeTicks::Now();
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]155 std::vector<const ui::AXNode*> content_root_nodes;
Abigail Kleinfdeb3d72023-01-19 15:20:19[diff] [blame]156 GetContentRootNodes(tree.root(), &content_root_nodes);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]157 for (const ui::AXNode* content_root_node : content_root_nodes) {
Kristi Saney4e5438b2023-05-16 21:00:58[diff] [blame]158 AddContentNodesToVector(content_root_node, content_node_ids);
Abigail Klein41d20e52022-11-28 19:24:06[diff] [blame]159 }
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]160 RecordRulesMetrics(ukm_source_id, base::TimeTicks::Now() - start_time,
161 !content_node_ids->empty());
162}
163
164void AXTreeDistiller::RecordRulesMetrics(ukm::SourceId ukm_source_id,
165 base::TimeDelta elapsed_time,
166 bool success) {
167 if (success) {
168 base::UmaHistogramTimes(
169 "Accessibility.ReadAnything.RulesDistillationTime.Success",
170 elapsed_time);
171 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
172 .SetRulesDistillationTime_Success(elapsed_time.InMilliseconds())
173 .Record(ukm_recorder_.get());
174 } else {
175 base::UmaHistogramTimes(
176 "Accessibility.ReadAnything.RulesDistillationTime.Failure",
177 elapsed_time);
178 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
179 .SetRulesDistillationTime_Failure(elapsed_time.InMilliseconds())
180 .Record(ukm_recorder_.get());
181 }
Abigail Kleina303f272022-03-31 22:32:57[diff] [blame]182}
183
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]184void AXTreeDistiller::DistillViaScreen2x(
185 const ui::AXTree& tree,
186 const ui::AXTreeUpdate& snapshot,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]187 const ukm::SourceId ukm_source_id,
188 base::TimeTicks start_time,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]189 std::vector<ui::AXNodeID>* content_node_ids_algorithm) {
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]190 DCHECK(main_content_extractor_.is_bound());
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]191 // Make a copy of |content_node_ids_algorithm| rather than sending a pointer.
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]192 main_content_extractor_->ExtractMainContent(
Abigail Kleindfdde352023-01-27 21:03:10[diff] [blame]193 snapshot, ukm_source_id,
Abigail Klein34db57e2023-05-01 22:37:37[diff] [blame]194 base::BindOnce(&AXTreeDistiller::ProcessScreen2xResult,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]195 weak_ptr_factory_.GetWeakPtr(), tree.GetAXTreeID(),
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]196 ukm_source_id, start_time, *content_node_ids_algorithm));
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]197}
198
199void AXTreeDistiller::ProcessScreen2xResult(
Abigail Klein34db57e2023-05-01 22:37:37[diff] [blame]200 const ui::AXTreeID& tree_id,
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]201 const ukm::SourceId ukm_source_id,
202 base::TimeTicks start_time,
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]203 std::vector<ui::AXNodeID> content_node_ids_algorithm,
204 const std::vector<ui::AXNodeID>& content_node_ids_screen2x) {
205 // Merge the results from the algorithm and from screen2x.
206 for (ui::AXNodeID content_node_id_screen2x : content_node_ids_screen2x) {
207 if (!base::Contains(content_node_ids_algorithm, content_node_id_screen2x)) {
208 content_node_ids_algorithm.push_back(content_node_id_screen2x);
209 }
210 }
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]211 RecordMergedMetrics(ukm_source_id, base::TimeTicks::Now() - start_time,
212 !content_node_ids_algorithm.empty());
Abigail Kleinad0977062023-05-19 18:21:08[diff] [blame]213 on_ax_tree_distilled_callback_.Run(tree_id, content_node_ids_algorithm);
Abigail Kleinaf1b5162022-12-01 01:40:40[diff] [blame]214
Kristi Saney84a78642023-05-10 18:41:54[diff] [blame]215 // TODO(crbug.com/1266555): If no content nodes were identified, and
Abigail Kleind5fef9a2022-11-07 15:03:50[diff] [blame]216 // there is a selection, try sending Screen2x a partial tree just containing
217 // the selected nodes.
Ramin Halavatieddadb62022-05-04 17:29:49[diff] [blame]218}
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]219
Jocelyn Tran2696ed22024-01-17 16:47:14[diff] [blame]220void AXTreeDistiller::ScreenAIServiceReady(content::RenderFrame* render_frame) {
221 if (main_content_extractor_.is_bound() || !render_frame) {
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]222 return;
223 }
Jocelyn Tran2696ed22024-01-17 16:47:14[diff] [blame]224 render_frame->GetBrowserInterfaceBroker()->GetInterface(
Abigail Klein598726242023-02-22 18:44:14[diff] [blame]225 main_content_extractor_.BindNewPipeAndPassReceiver());
226 main_content_extractor_.set_disconnect_handler(
227 base::BindOnce(&AXTreeDistiller::OnMainContentExtractorDisconnected,
228 weak_ptr_factory_.GetWeakPtr()));
229}
230
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]231void AXTreeDistiller::OnMainContentExtractorDisconnected() {
Abigail Klein3bd99e02023-01-24 00:29:18[diff] [blame]232 on_ax_tree_distilled_callback_.Run(ui::AXTreeIDUnknown(),
233 std::vector<ui::AXNodeID>());
Abigail Klein7849d1a2023-01-19 15:20:19[diff] [blame]234}
Jocelyn Trandab4ca602023-06-05 18:32:30[diff] [blame]235
236void AXTreeDistiller::RecordMergedMetrics(ukm::SourceId ukm_source_id,
237 base::TimeDelta elapsed_time,
238 bool success) {
239 if (success) {
240 base::UmaHistogramTimes(
241 "Accessibility.ReadAnything.MergedDistillationTime.Success",
242 elapsed_time);
243 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
244 .SetMergedDistillationTime_Success(elapsed_time.InMilliseconds())
245 .Record(ukm_recorder_.get());
246 } else {
247 base::UmaHistogramTimes(
248 "Accessibility.ReadAnything.MergedDistillationTime.Failure",
249 elapsed_time);
250 ukm::builders::Accessibility_ReadAnything(ukm_source_id)
251 .SetMergedDistillationTime_Failure(elapsed_time.InMilliseconds())
252 .Record(ukm_recorder_.get());
253 }
254}