/TalkBack/src/com/google/android/marvin/talkback/formatter/TouchExplorationFormatter.java
Java | 532 lines | 306 code | 82 blank | 144 comment | 85 complexity | 7edc929cdf977f8a470cacd2045c49fa MD5 | raw file
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.google.android.marvin.talkback.formatter; 18 19import android.content.Context; 20import android.graphics.Rect; 21import android.os.Bundle; 22import android.text.TextUtils; 23import android.util.Log; 24import android.view.accessibility.AccessibilityEvent; 25import android.view.accessibility.AccessibilityNodeInfo; 26import android.widget.AdapterView; 27 28import com.google.android.marvin.talkback.AccessibilityNodeInfoUtils; 29import com.google.android.marvin.talkback.AccessibilityNodeInfoUtils.NodeFilter; 30import com.google.android.marvin.talkback.Formatter; 31import com.google.android.marvin.talkback.speechrules.NodeProcessor; 32import com.google.android.marvin.talkback.LogUtils; 33import com.google.android.marvin.talkback.R; 34import com.google.android.marvin.talkback.Utils; 35import com.google.android.marvin.talkback.Utterance; 36 37import java.util.ArrayList; 38import java.util.Collections; 39import java.util.Comparator; 40import java.util.List; 41 42/** 43 * This class is a formatter for handling touch exploration events. Current 44 * implementation is simple and handles only hover enter events. 45 */ 46public final class TouchExplorationFormatter implements Formatter { 47 private static final Comparator<AccessibilityNodeInfo> COMPARATOR = 48 new TopToBottomLeftToRightComparator(); 49 private static final String SEPARATOR = " "; 50 51 /** Whether the last region the user explored was scrollable. */ 52 private boolean mUserCanScroll; 53 54 /** 55 * Whether the user is currently touch exploring. We can still receive 56 * {@link AccessibilityEvent#TYPE_VIEW_HOVER_ENTER} events when this is 57 * {@code false}. 58 */ 59 private boolean mIsTouchExploring; 60 61 /** The most recently announced node. Used for duplicate detection. */ 62 private AccessibilityNodeInfo mLastAnnouncedNode; 63 64 /** The node processor. Used to get spoken descriptions. */ 65 private NodeProcessor mNodeProcessor; 66 67 /** Filters out actionable nodes. Used to pick child nodes to read. */ 68 private final NodeFilter mNonActionableInfoFilter = new NodeFilter() { 69 @Override 70 public boolean accept(AccessibilityNodeInfo node) { 71 return !AccessibilityNodeInfoUtils.isActionable(node); 72 } 73 }; 74 75 /** 76 * Formatter that returns an utterance to announce touch exploration. 77 */ 78 @Override 79 public boolean format(AccessibilityEvent event, Context context, Utterance utterance, 80 Bundle args) { 81 final int eventType = event.getEventType(); 82 83 if (eventType == AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START) { 84 onTouchExplorationStarted(utterance); 85 return true; 86 } 87 88 if (eventType == AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END 89 || (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT && !mIsTouchExploring)) { 90 onTouchExplorationEnded(utterance); 91 return true; 92 } 93 94 if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT) { 95 // Don't actually do any processing for exit events. 96 return true; 97 } 98 99 final AccessibilityNodeInfo source = event.getSource(); 100 if (source == null) { 101 LogUtils.log(TouchExplorationFormatter.class, Log.INFO, 102 "Failed to obtain source for event"); 103 return false; 104 } 105 106 // Starting from the current node go up and find the best node to 107 // announce. 108 final ArrayList<AccessibilityNodeInfo> sourceAndPredecessors = 109 new ArrayList<AccessibilityNodeInfo>(); 110 sourceAndPredecessors.add(source); 111 AccessibilityNodeInfoUtils.getPredecessors(source, sourceAndPredecessors); 112 113 final AccessibilityNodeInfo announcedNode = computeAnnouncedNode(sourceAndPredecessors); 114 115 if (announcedNode != null) { 116 LogUtils.log(TouchExplorationFormatter.class, Log.DEBUG, "Announcing node: %s", 117 announcedNode); 118 } 119 120 // HACK: The announced node will always be null for an AdapterView, 121 // but in the case of a FOCUSED or SELECTED event, we still want to 122 // read the event text. Otherwise, we shouldn't be reading layouts. 123 if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER && announcedNode == null 124 && Utils.isViewGroup(event, source)) { 125 LogUtils.log(TouchExplorationFormatter.class, Log.INFO, 126 "No node to announce, ignoring view with children"); 127 128 AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors); 129 AccessibilityNodeInfoUtils.recycleNodes(announcedNode); 130 return false; 131 } 132 133 // Do not announce the same node twice in a row. 134 if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER && announcedNode != null 135 && announcedNode.equals(mLastAnnouncedNode)) { 136 LogUtils.log(TouchExplorationFormatter.class, Log.INFO, 137 "Same as last announced node, not speaking"); 138 139 AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors); 140 AccessibilityNodeInfoUtils.recycleNodes(announcedNode); 141 return false; 142 } 143 144 // If neither succeeded, abort. 145 if (!addDescription(context, event, utterance, source, announcedNode)) { 146 LogUtils.log(TouchExplorationFormatter.class, Log.INFO, 147 "Failed to populate utterance, not speaking"); 148 149 AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors); 150 AccessibilityNodeInfoUtils.recycleNodes(announcedNode); 151 return false; 152 } 153 154 addEarcons(announcedNode, sourceAndPredecessors, event, utterance); 155 156 setLastAnnouncedNode(announcedNode); 157 158 AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors); 159 AccessibilityNodeInfoUtils.recycleNodes(announcedNode); 160 return true; 161 } 162 163 /** 164 * Sets the most recently announced node. 165 * 166 * @param node The node to set. 167 */ 168 private void setLastAnnouncedNode(AccessibilityNodeInfo node) { 169 if (mLastAnnouncedNode != null) { 170 mLastAnnouncedNode.recycle(); 171 } 172 173 if (node != null) { 174 mLastAnnouncedNode = AccessibilityNodeInfo.obtain(node); 175 } else { 176 mLastAnnouncedNode = null; 177 } 178 } 179 180 private enum DescriptionResult { 181 SUCCESS, FAILURE, ABORT 182 } 183 184 /** 185 * Populates an utterance with text, either from the node or event. 186 * 187 * @param context The parent context. 188 * @param event The source event. 189 * @param utterance The target utterance. 190 * @param source The source node. 191 * @param announcedNode The computed announced node. 192 * @return {@code true} if the utterance was populated with text. 193 */ 194 private boolean addDescription(Context context, AccessibilityEvent event, Utterance utterance, 195 AccessibilityNodeInfo source, AccessibilityNodeInfo announcedNode) { 196 // Attempt to populate with node description. 197 final DescriptionResult result = 198 addNodeDescription(context, event, utterance, source, announcedNode); 199 200 // If the node description failed (but didn't abort) try the event. 201 if (result == DescriptionResult.FAILURE) { 202 return addEventDescription(context, event, utterance); 203 } 204 205 return (result == DescriptionResult.SUCCESS); 206 } 207 208 /** 209 * Populates an utterance with text from an event. 210 * 211 * @param context The parent context. 212 * @param event The source event. 213 * @param utterance The target utterance. 214 */ 215 private boolean addEventDescription(Context context, AccessibilityEvent event, 216 Utterance utterance) { 217 final CharSequence eventText = Utils.getEventText(context, event); 218 219 if (TextUtils.isEmpty(eventText)) { 220 return false; 221 } 222 223 utterance.getText().append(eventText); 224 225 return true; 226 } 227 228 /** 229 * Adds a description to an utterance for the specified node. 230 * 231 * @param context The parent context. 232 * @param event The source event. 233 * @param utterance The output utterance. 234 * @param source The source node. 235 * @param announcedNode The node to announce. 236 */ 237 private DescriptionResult addNodeDescription(Context context, AccessibilityEvent event, 238 Utterance utterance, AccessibilityNodeInfo source, 239 AccessibilityNodeInfo announcedNode) { 240 final StringBuilder builder = utterance.getText(); 241 242 if (announcedNode == null) { 243 return DescriptionResult.FAILURE; 244 } 245 246 final CharSequence desc = announcedNode.getContentDescription(); 247 248 // HACK: If the node to announce is a view group and already has a 249 // content description, don't bother fetching its children. 250 if (!TextUtils.isEmpty(desc) && Utils.isViewGroup(null, announcedNode)) { 251 builder.append(desc); 252 return DescriptionResult.SUCCESS; 253 } 254 255 final AccessibilityNodeInfo announcedNodeClone = 256 AccessibilityNodeInfo.obtain(announcedNode); 257 final ArrayList<AccessibilityNodeInfo> announcedSubtreeNodes = 258 new ArrayList<AccessibilityNodeInfo>(); 259 260 // Fetch the subtree of the node to announce. 261 announcedSubtreeNodes.add(announcedNodeClone); 262 263 final int failedFilter = 264 AccessibilityNodeInfoUtils.addDescendantsBfs(announcedNode, announcedSubtreeNodes, 265 mNonActionableInfoFilter); 266 267 Collections.sort(announcedSubtreeNodes, COMPARATOR); 268 269 // Make sure we have a node processor. 270 // TODO(alanv): We should do this on initialization. 271 if (mNodeProcessor == null) { 272 mNodeProcessor = new NodeProcessor(context); 273 } 274 275 // Add text for the nodes. If we have text (or there are children that 276 // failed the filter) then we've handled the node. 277 final boolean handled = addNodesText(event, source, announcedSubtreeNodes, utterance); 278 279 AccessibilityNodeInfoUtils.recycleNodeList(announcedSubtreeNodes); 280 281 if (handled) { 282 return DescriptionResult.SUCCESS; 283 } else if (failedFilter > 0) { 284 return DescriptionResult.ABORT; 285 } else { 286 return DescriptionResult.FAILURE; 287 } 288 } 289 290 private void onTouchExplorationStarted(Utterance utterance) { 291 if (!mIsTouchExploring) { 292 utterance.getEarcons().add(R.raw.explore_begin); 293 } 294 295 mUserCanScroll = false; 296 mIsTouchExploring = true; 297 } 298 299 private void onTouchExplorationEnded(Utterance utterance) { 300 if (mIsTouchExploring) { 301 utterance.getEarcons().add(R.raw.explore_end); 302 } 303 304 if (mLastAnnouncedNode != null) { 305 mLastAnnouncedNode.recycle(); 306 mLastAnnouncedNode = null; 307 } 308 309 mIsTouchExploring = false; 310 } 311 312 /** 313 * Adds earcons when moving between scrollable and non-scrollable views. 314 * 315 * @param announcedNode The node that is announces. 316 * @param eventSourcePredecessors The event source predecessors. 317 * @param event The received accessibility event. 318 * @param utterance The utterance to which to add the earcons. 319 */ 320 private void addEarcons(AccessibilityNodeInfo announcedNode, 321 ArrayList<AccessibilityNodeInfo> eventSourcePredecessors, AccessibilityEvent event, 322 Utterance utterance) { 323 // If the announced node is in a scrollable container - add an earcon to 324 // convey that. 325 final boolean userCanScroll = 326 isScrollableOrHasScrollablePredecessor(announcedNode, eventSourcePredecessors); 327 if (mUserCanScroll != userCanScroll) { 328 mUserCanScroll = userCanScroll; 329 if (userCanScroll) { 330 utterance.getEarcons().add(R.raw.chime_up); 331 } else { 332 utterance.getEarcons().add(R.raw.chime_down); 333 } 334 } 335 336 final boolean actionable = 337 isActionableOrHasActionablePredecessor(announcedNode, eventSourcePredecessors); 338 if (actionable) { 339 utterance.getCustomEarcons().add(R.string.pref_sounds_actionable_key); 340 utterance.getCustomVibrations().add(R.string.pref_patterns_actionable_key); 341 } else { 342 utterance.getCustomEarcons().add(R.string.pref_sounds_hover_key); 343 utterance.getCustomVibrations().add(R.string.pref_patterns_hover_key); 344 } 345 } 346 347 /** 348 * Computes the node that is to be announced. 349 * 350 * @param nodes The candidate nodes. 351 * @return The node to announce or null if no such is found. 352 */ 353 private AccessibilityNodeInfo computeAnnouncedNode(ArrayList<AccessibilityNodeInfo> nodes) { 354 for (AccessibilityNodeInfo node : nodes) { 355 if (shouldAnnounceNode(node)) { 356 return AccessibilityNodeInfo.obtain(node); 357 } 358 } 359 360 return null; 361 } 362 363 /** 364 * Determines whether a node should to be announced. This is used when 365 * traversing up from a touched node to ensure that enough information is 366 * spoken to provide context for what will happen if the user performs an 367 * action on the item. 368 * 369 * @param node The examined node. 370 * @return True if the node is to be announced. 371 */ 372 private boolean shouldAnnounceNode(AccessibilityNodeInfo node) { 373 // Do not announce AdapterView or anything that extends it. 374 if (AccessibilityNodeInfoUtils.nodeMatchesClassByType(node, AdapterView.class)) { 375 return false; 376 } 377 378 // Always announce "actionable" nodes. 379 if (AccessibilityNodeInfoUtils.isActionable(node)) { 380 return true; 381 } 382 383 // If the parent is an AdapterView, then the node is a list item, so 384 // announce it. 385 final AccessibilityNodeInfo parent = node.getParent(); 386 387 if (parent == null) { 388 return false; 389 } 390 391 final boolean isListItem = 392 AccessibilityNodeInfoUtils.nodeMatchesClassByType(parent, AdapterView.class); 393 parent.recycle(); 394 395 return isListItem; 396 } 397 398 /** 399 * Adds the text of the given nodes to the specified utterance. 400 * 401 * @param nodes The nodes whose text to add. 402 * @param utterance The utterance to which to add text. 403 */ 404 private boolean addNodesText(AccessibilityEvent event, AccessibilityNodeInfo source, 405 ArrayList<AccessibilityNodeInfo> nodes, Utterance utterance) { 406 final StringBuilder builder = utterance.getText(); 407 408 for (AccessibilityNodeInfo node : nodes) { 409 final CharSequence nodeDesc; 410 411 if (node.equals(source)) { 412 nodeDesc = mNodeProcessor.process(node, event); 413 } else { 414 nodeDesc = mNodeProcessor.process(node, null); 415 } 416 417 if (TextUtils.isEmpty(nodeDesc)) { 418 continue; 419 } 420 421 builder.append(nodeDesc); 422 builder.append(SEPARATOR); 423 } 424 425 return !TextUtils.isEmpty(builder); 426 } 427 428 /** 429 * Check whether a given node is actionable or has an actionable 430 * predecessor. 431 * 432 * @param source The announced node. 433 * @param predecessors The predecessors of the event source node. 434 * @return True if the node or some of its predecessors is actionable. 435 */ 436 private boolean isActionableOrHasActionablePredecessor(AccessibilityNodeInfo source, 437 List<AccessibilityNodeInfo> predecessors) { 438 // Nothing is always non-actionable. 439 if (source == null || predecessors == null) { 440 return false; 441 } 442 443 // Check whether the node is actionable. 444 if (AccessibilityNodeInfoUtils.isActionable(source)) { 445 return true; 446 } 447 448 // Check whether the any of the node predecessors is actionable. 449 for (AccessibilityNodeInfo predecessor : predecessors) { 450 if (AccessibilityNodeInfoUtils.isActionable(predecessor)) { 451 return true; 452 } 453 } 454 455 return false; 456 } 457 458 /** 459 * Check whether a given node is scrollable or has a scrollable predecessor. 460 * 461 * @param source The announced node. 462 * @param predecessors The predecessors of the event source node. 463 * @return True if the node or some of its predecessors is scrollable. 464 */ 465 private boolean isScrollableOrHasScrollablePredecessor(AccessibilityNodeInfo source, 466 List<AccessibilityNodeInfo> predecessors) { 467 // Nothing is always non-actionable. 468 if (source == null || predecessors == null) { 469 return false; 470 } 471 472 // Check whether the node is scrollable. 473 if (source.isScrollable()) { 474 return true; 475 } 476 477 // Check whether the any of the node predecessors is scrollable. 478 for (AccessibilityNodeInfo predecessor : predecessors) { 479 if (predecessor.isScrollable()) { 480 return true; 481 } 482 } 483 484 return false; 485 } 486 487 /** 488 * Compares two AccessibilityNodeInfos in left-to-right and top-to-bottom 489 * fashion. 490 */ 491 private static class TopToBottomLeftToRightComparator implements 492 Comparator<AccessibilityNodeInfo> { 493 private final Rect mFirstBounds = new Rect(); 494 private final Rect mSecondBounds = new Rect(); 495 496 @Override 497 public int compare(AccessibilityNodeInfo first, AccessibilityNodeInfo second) { 498 Rect firstBounds = mFirstBounds; 499 first.getBoundsInScreen(firstBounds); 500 501 Rect secondBounds = mSecondBounds; 502 second.getBoundsInScreen(secondBounds); 503 504 // top - bottom 505 final int topDifference = firstBounds.top - secondBounds.top; 506 if (topDifference != 0) { 507 return topDifference; 508 } 509 // left - right 510 final int leftDifference = firstBounds.left - secondBounds.left; 511 if (leftDifference != 0) { 512 return leftDifference; 513 } 514 // break tie by height 515 final int firstHeight = firstBounds.bottom - firstBounds.top; 516 final int secondHeight = secondBounds.bottom - secondBounds.top; 517 final int heightDiference = firstHeight - secondHeight; 518 if (heightDiference != 0) { 519 return heightDiference; 520 } 521 // break tie by width 522 final int firstWidth = firstBounds.right - firstBounds.left; 523 final int secondWidth = secondBounds.right - secondBounds.left; 524 int widthDiference = firstWidth - secondWidth; 525 if (widthDiference != 0) { 526 return widthDiference; 527 } 528 // do not return 0 to avoid losing data 529 return 1; 530 } 531 } 532}