/TalkBack/src/com/google/android/marvin/talkback/formatter/TouchExplorationFormatter.java

http://eyes-free.googlecode.com/ · Java · 532 lines · 306 code · 82 blank · 144 comment · 85 complexity · 7edc929cdf977f8a470cacd2045c49fa MD5 · raw file

  1. /*
  2. * Copyright (C) 2011 The Android Open Source Project
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. package com.google.android.marvin.talkback.formatter;
  17. import android.content.Context;
  18. import android.graphics.Rect;
  19. import android.os.Bundle;
  20. import android.text.TextUtils;
  21. import android.util.Log;
  22. import android.view.accessibility.AccessibilityEvent;
  23. import android.view.accessibility.AccessibilityNodeInfo;
  24. import android.widget.AdapterView;
  25. import com.google.android.marvin.talkback.AccessibilityNodeInfoUtils;
  26. import com.google.android.marvin.talkback.AccessibilityNodeInfoUtils.NodeFilter;
  27. import com.google.android.marvin.talkback.Formatter;
  28. import com.google.android.marvin.talkback.speechrules.NodeProcessor;
  29. import com.google.android.marvin.talkback.LogUtils;
  30. import com.google.android.marvin.talkback.R;
  31. import com.google.android.marvin.talkback.Utils;
  32. import com.google.android.marvin.talkback.Utterance;
  33. import java.util.ArrayList;
  34. import java.util.Collections;
  35. import java.util.Comparator;
  36. import java.util.List;
  37. /**
  38. * This class is a formatter for handling touch exploration events. Current
  39. * implementation is simple and handles only hover enter events.
  40. */
  41. public final class TouchExplorationFormatter implements Formatter {
  42. private static final Comparator<AccessibilityNodeInfo> COMPARATOR =
  43. new TopToBottomLeftToRightComparator();
  44. private static final String SEPARATOR = " ";
  45. /** Whether the last region the user explored was scrollable. */
  46. private boolean mUserCanScroll;
  47. /**
  48. * Whether the user is currently touch exploring. We can still receive
  49. * {@link AccessibilityEvent#TYPE_VIEW_HOVER_ENTER} events when this is
  50. * {@code false}.
  51. */
  52. private boolean mIsTouchExploring;
  53. /** The most recently announced node. Used for duplicate detection. */
  54. private AccessibilityNodeInfo mLastAnnouncedNode;
  55. /** The node processor. Used to get spoken descriptions. */
  56. private NodeProcessor mNodeProcessor;
  57. /** Filters out actionable nodes. Used to pick child nodes to read. */
  58. private final NodeFilter mNonActionableInfoFilter = new NodeFilter() {
  59. @Override
  60. public boolean accept(AccessibilityNodeInfo node) {
  61. return !AccessibilityNodeInfoUtils.isActionable(node);
  62. }
  63. };
  64. /**
  65. * Formatter that returns an utterance to announce touch exploration.
  66. */
  67. @Override
  68. public boolean format(AccessibilityEvent event, Context context, Utterance utterance,
  69. Bundle args) {
  70. final int eventType = event.getEventType();
  71. if (eventType == AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START) {
  72. onTouchExplorationStarted(utterance);
  73. return true;
  74. }
  75. if (eventType == AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END
  76. || (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT && !mIsTouchExploring)) {
  77. onTouchExplorationEnded(utterance);
  78. return true;
  79. }
  80. if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT) {
  81. // Don't actually do any processing for exit events.
  82. return true;
  83. }
  84. final AccessibilityNodeInfo source = event.getSource();
  85. if (source == null) {
  86. LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
  87. "Failed to obtain source for event");
  88. return false;
  89. }
  90. // Starting from the current node go up and find the best node to
  91. // announce.
  92. final ArrayList<AccessibilityNodeInfo> sourceAndPredecessors =
  93. new ArrayList<AccessibilityNodeInfo>();
  94. sourceAndPredecessors.add(source);
  95. AccessibilityNodeInfoUtils.getPredecessors(source, sourceAndPredecessors);
  96. final AccessibilityNodeInfo announcedNode = computeAnnouncedNode(sourceAndPredecessors);
  97. if (announcedNode != null) {
  98. LogUtils.log(TouchExplorationFormatter.class, Log.DEBUG, "Announcing node: %s",
  99. announcedNode);
  100. }
  101. // HACK: The announced node will always be null for an AdapterView,
  102. // but in the case of a FOCUSED or SELECTED event, we still want to
  103. // read the event text. Otherwise, we shouldn't be reading layouts.
  104. if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER && announcedNode == null
  105. && Utils.isViewGroup(event, source)) {
  106. LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
  107. "No node to announce, ignoring view with children");
  108. AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
  109. AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
  110. return false;
  111. }
  112. // Do not announce the same node twice in a row.
  113. if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER && announcedNode != null
  114. && announcedNode.equals(mLastAnnouncedNode)) {
  115. LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
  116. "Same as last announced node, not speaking");
  117. AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
  118. AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
  119. return false;
  120. }
  121. // If neither succeeded, abort.
  122. if (!addDescription(context, event, utterance, source, announcedNode)) {
  123. LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
  124. "Failed to populate utterance, not speaking");
  125. AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
  126. AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
  127. return false;
  128. }
  129. addEarcons(announcedNode, sourceAndPredecessors, event, utterance);
  130. setLastAnnouncedNode(announcedNode);
  131. AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
  132. AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
  133. return true;
  134. }
  135. /**
  136. * Sets the most recently announced node.
  137. *
  138. * @param node The node to set.
  139. */
  140. private void setLastAnnouncedNode(AccessibilityNodeInfo node) {
  141. if (mLastAnnouncedNode != null) {
  142. mLastAnnouncedNode.recycle();
  143. }
  144. if (node != null) {
  145. mLastAnnouncedNode = AccessibilityNodeInfo.obtain(node);
  146. } else {
  147. mLastAnnouncedNode = null;
  148. }
  149. }
  150. private enum DescriptionResult {
  151. SUCCESS, FAILURE, ABORT
  152. }
  153. /**
  154. * Populates an utterance with text, either from the node or event.
  155. *
  156. * @param context The parent context.
  157. * @param event The source event.
  158. * @param utterance The target utterance.
  159. * @param source The source node.
  160. * @param announcedNode The computed announced node.
  161. * @return {@code true} if the utterance was populated with text.
  162. */
  163. private boolean addDescription(Context context, AccessibilityEvent event, Utterance utterance,
  164. AccessibilityNodeInfo source, AccessibilityNodeInfo announcedNode) {
  165. // Attempt to populate with node description.
  166. final DescriptionResult result =
  167. addNodeDescription(context, event, utterance, source, announcedNode);
  168. // If the node description failed (but didn't abort) try the event.
  169. if (result == DescriptionResult.FAILURE) {
  170. return addEventDescription(context, event, utterance);
  171. }
  172. return (result == DescriptionResult.SUCCESS);
  173. }
  174. /**
  175. * Populates an utterance with text from an event.
  176. *
  177. * @param context The parent context.
  178. * @param event The source event.
  179. * @param utterance The target utterance.
  180. */
  181. private boolean addEventDescription(Context context, AccessibilityEvent event,
  182. Utterance utterance) {
  183. final CharSequence eventText = Utils.getEventText(context, event);
  184. if (TextUtils.isEmpty(eventText)) {
  185. return false;
  186. }
  187. utterance.getText().append(eventText);
  188. return true;
  189. }
  190. /**
  191. * Adds a description to an utterance for the specified node.
  192. *
  193. * @param context The parent context.
  194. * @param event The source event.
  195. * @param utterance The output utterance.
  196. * @param source The source node.
  197. * @param announcedNode The node to announce.
  198. */
  199. private DescriptionResult addNodeDescription(Context context, AccessibilityEvent event,
  200. Utterance utterance, AccessibilityNodeInfo source,
  201. AccessibilityNodeInfo announcedNode) {
  202. final StringBuilder builder = utterance.getText();
  203. if (announcedNode == null) {
  204. return DescriptionResult.FAILURE;
  205. }
  206. final CharSequence desc = announcedNode.getContentDescription();
  207. // HACK: If the node to announce is a view group and already has a
  208. // content description, don't bother fetching its children.
  209. if (!TextUtils.isEmpty(desc) && Utils.isViewGroup(null, announcedNode)) {
  210. builder.append(desc);
  211. return DescriptionResult.SUCCESS;
  212. }
  213. final AccessibilityNodeInfo announcedNodeClone =
  214. AccessibilityNodeInfo.obtain(announcedNode);
  215. final ArrayList<AccessibilityNodeInfo> announcedSubtreeNodes =
  216. new ArrayList<AccessibilityNodeInfo>();
  217. // Fetch the subtree of the node to announce.
  218. announcedSubtreeNodes.add(announcedNodeClone);
  219. final int failedFilter =
  220. AccessibilityNodeInfoUtils.addDescendantsBfs(announcedNode, announcedSubtreeNodes,
  221. mNonActionableInfoFilter);
  222. Collections.sort(announcedSubtreeNodes, COMPARATOR);
  223. // Make sure we have a node processor.
  224. // TODO(alanv): We should do this on initialization.
  225. if (mNodeProcessor == null) {
  226. mNodeProcessor = new NodeProcessor(context);
  227. }
  228. // Add text for the nodes. If we have text (or there are children that
  229. // failed the filter) then we've handled the node.
  230. final boolean handled = addNodesText(event, source, announcedSubtreeNodes, utterance);
  231. AccessibilityNodeInfoUtils.recycleNodeList(announcedSubtreeNodes);
  232. if (handled) {
  233. return DescriptionResult.SUCCESS;
  234. } else if (failedFilter > 0) {
  235. return DescriptionResult.ABORT;
  236. } else {
  237. return DescriptionResult.FAILURE;
  238. }
  239. }
  240. private void onTouchExplorationStarted(Utterance utterance) {
  241. if (!mIsTouchExploring) {
  242. utterance.getEarcons().add(R.raw.explore_begin);
  243. }
  244. mUserCanScroll = false;
  245. mIsTouchExploring = true;
  246. }
  247. private void onTouchExplorationEnded(Utterance utterance) {
  248. if (mIsTouchExploring) {
  249. utterance.getEarcons().add(R.raw.explore_end);
  250. }
  251. if (mLastAnnouncedNode != null) {
  252. mLastAnnouncedNode.recycle();
  253. mLastAnnouncedNode = null;
  254. }
  255. mIsTouchExploring = false;
  256. }
  257. /**
  258. * Adds earcons when moving between scrollable and non-scrollable views.
  259. *
  260. * @param announcedNode The node that is announces.
  261. * @param eventSourcePredecessors The event source predecessors.
  262. * @param event The received accessibility event.
  263. * @param utterance The utterance to which to add the earcons.
  264. */
  265. private void addEarcons(AccessibilityNodeInfo announcedNode,
  266. ArrayList<AccessibilityNodeInfo> eventSourcePredecessors, AccessibilityEvent event,
  267. Utterance utterance) {
  268. // If the announced node is in a scrollable container - add an earcon to
  269. // convey that.
  270. final boolean userCanScroll =
  271. isScrollableOrHasScrollablePredecessor(announcedNode, eventSourcePredecessors);
  272. if (mUserCanScroll != userCanScroll) {
  273. mUserCanScroll = userCanScroll;
  274. if (userCanScroll) {
  275. utterance.getEarcons().add(R.raw.chime_up);
  276. } else {
  277. utterance.getEarcons().add(R.raw.chime_down);
  278. }
  279. }
  280. final boolean actionable =
  281. isActionableOrHasActionablePredecessor(announcedNode, eventSourcePredecessors);
  282. if (actionable) {
  283. utterance.getCustomEarcons().add(R.string.pref_sounds_actionable_key);
  284. utterance.getCustomVibrations().add(R.string.pref_patterns_actionable_key);
  285. } else {
  286. utterance.getCustomEarcons().add(R.string.pref_sounds_hover_key);
  287. utterance.getCustomVibrations().add(R.string.pref_patterns_hover_key);
  288. }
  289. }
  290. /**
  291. * Computes the node that is to be announced.
  292. *
  293. * @param nodes The candidate nodes.
  294. * @return The node to announce or null if no such is found.
  295. */
  296. private AccessibilityNodeInfo computeAnnouncedNode(ArrayList<AccessibilityNodeInfo> nodes) {
  297. for (AccessibilityNodeInfo node : nodes) {
  298. if (shouldAnnounceNode(node)) {
  299. return AccessibilityNodeInfo.obtain(node);
  300. }
  301. }
  302. return null;
  303. }
  304. /**
  305. * Determines whether a node should to be announced. This is used when
  306. * traversing up from a touched node to ensure that enough information is
  307. * spoken to provide context for what will happen if the user performs an
  308. * action on the item.
  309. *
  310. * @param node The examined node.
  311. * @return True if the node is to be announced.
  312. */
  313. private boolean shouldAnnounceNode(AccessibilityNodeInfo node) {
  314. // Do not announce AdapterView or anything that extends it.
  315. if (AccessibilityNodeInfoUtils.nodeMatchesClassByType(node, AdapterView.class)) {
  316. return false;
  317. }
  318. // Always announce "actionable" nodes.
  319. if (AccessibilityNodeInfoUtils.isActionable(node)) {
  320. return true;
  321. }
  322. // If the parent is an AdapterView, then the node is a list item, so
  323. // announce it.
  324. final AccessibilityNodeInfo parent = node.getParent();
  325. if (parent == null) {
  326. return false;
  327. }
  328. final boolean isListItem =
  329. AccessibilityNodeInfoUtils.nodeMatchesClassByType(parent, AdapterView.class);
  330. parent.recycle();
  331. return isListItem;
  332. }
  333. /**
  334. * Adds the text of the given nodes to the specified utterance.
  335. *
  336. * @param nodes The nodes whose text to add.
  337. * @param utterance The utterance to which to add text.
  338. */
  339. private boolean addNodesText(AccessibilityEvent event, AccessibilityNodeInfo source,
  340. ArrayList<AccessibilityNodeInfo> nodes, Utterance utterance) {
  341. final StringBuilder builder = utterance.getText();
  342. for (AccessibilityNodeInfo node : nodes) {
  343. final CharSequence nodeDesc;
  344. if (node.equals(source)) {
  345. nodeDesc = mNodeProcessor.process(node, event);
  346. } else {
  347. nodeDesc = mNodeProcessor.process(node, null);
  348. }
  349. if (TextUtils.isEmpty(nodeDesc)) {
  350. continue;
  351. }
  352. builder.append(nodeDesc);
  353. builder.append(SEPARATOR);
  354. }
  355. return !TextUtils.isEmpty(builder);
  356. }
  357. /**
  358. * Check whether a given node is actionable or has an actionable
  359. * predecessor.
  360. *
  361. * @param source The announced node.
  362. * @param predecessors The predecessors of the event source node.
  363. * @return True if the node or some of its predecessors is actionable.
  364. */
  365. private boolean isActionableOrHasActionablePredecessor(AccessibilityNodeInfo source,
  366. List<AccessibilityNodeInfo> predecessors) {
  367. // Nothing is always non-actionable.
  368. if (source == null || predecessors == null) {
  369. return false;
  370. }
  371. // Check whether the node is actionable.
  372. if (AccessibilityNodeInfoUtils.isActionable(source)) {
  373. return true;
  374. }
  375. // Check whether the any of the node predecessors is actionable.
  376. for (AccessibilityNodeInfo predecessor : predecessors) {
  377. if (AccessibilityNodeInfoUtils.isActionable(predecessor)) {
  378. return true;
  379. }
  380. }
  381. return false;
  382. }
  383. /**
  384. * Check whether a given node is scrollable or has a scrollable predecessor.
  385. *
  386. * @param source The announced node.
  387. * @param predecessors The predecessors of the event source node.
  388. * @return True if the node or some of its predecessors is scrollable.
  389. */
  390. private boolean isScrollableOrHasScrollablePredecessor(AccessibilityNodeInfo source,
  391. List<AccessibilityNodeInfo> predecessors) {
  392. // Nothing is always non-actionable.
  393. if (source == null || predecessors == null) {
  394. return false;
  395. }
  396. // Check whether the node is scrollable.
  397. if (source.isScrollable()) {
  398. return true;
  399. }
  400. // Check whether the any of the node predecessors is scrollable.
  401. for (AccessibilityNodeInfo predecessor : predecessors) {
  402. if (predecessor.isScrollable()) {
  403. return true;
  404. }
  405. }
  406. return false;
  407. }
  408. /**
  409. * Compares two AccessibilityNodeInfos in left-to-right and top-to-bottom
  410. * fashion.
  411. */
  412. private static class TopToBottomLeftToRightComparator implements
  413. Comparator<AccessibilityNodeInfo> {
  414. private final Rect mFirstBounds = new Rect();
  415. private final Rect mSecondBounds = new Rect();
  416. @Override
  417. public int compare(AccessibilityNodeInfo first, AccessibilityNodeInfo second) {
  418. Rect firstBounds = mFirstBounds;
  419. first.getBoundsInScreen(firstBounds);
  420. Rect secondBounds = mSecondBounds;
  421. second.getBoundsInScreen(secondBounds);
  422. // top - bottom
  423. final int topDifference = firstBounds.top - secondBounds.top;
  424. if (topDifference != 0) {
  425. return topDifference;
  426. }
  427. // left - right
  428. final int leftDifference = firstBounds.left - secondBounds.left;
  429. if (leftDifference != 0) {
  430. return leftDifference;
  431. }
  432. // break tie by height
  433. final int firstHeight = firstBounds.bottom - firstBounds.top;
  434. final int secondHeight = secondBounds.bottom - secondBounds.top;
  435. final int heightDiference = firstHeight - secondHeight;
  436. if (heightDiference != 0) {
  437. return heightDiference;
  438. }
  439. // break tie by width
  440. final int firstWidth = firstBounds.right - firstBounds.left;
  441. final int secondWidth = secondBounds.right - secondBounds.left;
  442. int widthDiference = firstWidth - secondWidth;
  443. if (widthDiference != 0) {
  444. return widthDiference;
  445. }
  446. // do not return 0 to avoid losing data
  447. return 1;
  448. }
  449. }
  450. }