/TalkBack/src/com/google/android/marvin/talkback/formatter/TouchExplorationFormatter.java

  1/*
  2 * Copyright (C) 2011 The Android Open Source Project
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License");
  5 * you may not use this file except in compliance with the License.
  6 * You may obtain a copy of the License at
  7 *
  8 *      http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS,
 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 * See the License for the specific language governing permissions and
 14 * limitations under the License.
 15 */
 16
 17package com.google.android.marvin.talkback.formatter;
 18
 19import android.content.Context;
 20import android.graphics.Rect;
 21import android.os.Bundle;
 22import android.text.TextUtils;
 23import android.util.Log;
 24import android.view.accessibility.AccessibilityEvent;
 25import android.view.accessibility.AccessibilityNodeInfo;
 26import android.widget.AdapterView;
 27
 28import com.google.android.marvin.talkback.AccessibilityNodeInfoUtils;
 29import com.google.android.marvin.talkback.AccessibilityNodeInfoUtils.NodeFilter;
 30import com.google.android.marvin.talkback.Formatter;
 31import com.google.android.marvin.talkback.speechrules.NodeProcessor;
 32import com.google.android.marvin.talkback.LogUtils;
 33import com.google.android.marvin.talkback.R;
 34import com.google.android.marvin.talkback.Utils;
 35import com.google.android.marvin.talkback.Utterance;
 36
 37import java.util.ArrayList;
 38import java.util.Collections;
 39import java.util.Comparator;
 40import java.util.List;
 41
 42/**
 43 * This class is a formatter for handling touch exploration events. Current
 44 * implementation is simple and handles only hover enter events.
 45 */
 46public final class TouchExplorationFormatter implements Formatter {
 47    private static final Comparator<AccessibilityNodeInfo> COMPARATOR =
 48            new TopToBottomLeftToRightComparator();
 49    private static final String SEPARATOR = " ";
 50
 51    /** Whether the last region the user explored was scrollable. */
 52    private boolean mUserCanScroll;
 53
 54    /**
 55     * Whether the user is currently touch exploring. We can still receive
 56     * {@link AccessibilityEvent#TYPE_VIEW_HOVER_ENTER} events when this is
 57     * {@code false}.
 58     */
 59    private boolean mIsTouchExploring;
 60
 61    /** The most recently announced node. Used for duplicate detection. */
 62    private AccessibilityNodeInfo mLastAnnouncedNode;
 63
 64    /** The node processor. Used to get spoken descriptions. */
 65    private NodeProcessor mNodeProcessor;
 66
 67    /** Filters out actionable nodes. Used to pick child nodes to read. */
 68    private final NodeFilter mNonActionableInfoFilter = new NodeFilter() {
 69        @Override
 70        public boolean accept(AccessibilityNodeInfo node) {
 71            return !AccessibilityNodeInfoUtils.isActionable(node);
 72        }
 73    };
 74
 75    /**
 76     * Formatter that returns an utterance to announce touch exploration.
 77     */
 78    @Override
 79    public boolean format(AccessibilityEvent event, Context context, Utterance utterance,
 80            Bundle args) {
 81        final int eventType = event.getEventType();
 82
 83        if (eventType == AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_START) {
 84            onTouchExplorationStarted(utterance);
 85            return true;
 86        }
 87
 88        if (eventType == AccessibilityEvent.TYPE_TOUCH_EXPLORATION_GESTURE_END
 89                || (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT && !mIsTouchExploring)) {
 90            onTouchExplorationEnded(utterance);
 91            return true;
 92        }
 93
 94        if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_EXIT) {
 95            // Don't actually do any processing for exit events.
 96            return true;
 97        }
 98
 99        final AccessibilityNodeInfo source = event.getSource();
100        if (source == null) {
101            LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
102                    "Failed to obtain source for event");
103            return false;
104        }
105
106        // Starting from the current node go up and find the best node to
107        // announce.
108        final ArrayList<AccessibilityNodeInfo> sourceAndPredecessors =
109                new ArrayList<AccessibilityNodeInfo>();
110        sourceAndPredecessors.add(source);
111        AccessibilityNodeInfoUtils.getPredecessors(source, sourceAndPredecessors);
112
113        final AccessibilityNodeInfo announcedNode = computeAnnouncedNode(sourceAndPredecessors);
114
115        if (announcedNode != null) {
116            LogUtils.log(TouchExplorationFormatter.class, Log.DEBUG, "Announcing node: %s",
117                    announcedNode);
118        }
119
120        // HACK: The announced node will always be null for an AdapterView,
121        // but in the case of a FOCUSED or SELECTED event, we still want to
122        // read the event text. Otherwise, we shouldn't be reading layouts.
123        if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER && announcedNode == null
124                && Utils.isViewGroup(event, source)) {
125            LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
126                    "No node to announce, ignoring view with children");
127
128            AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
129            AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
130            return false;
131        }
132
133        // Do not announce the same node twice in a row.
134        if (eventType == AccessibilityEvent.TYPE_VIEW_HOVER_ENTER && announcedNode != null
135                && announcedNode.equals(mLastAnnouncedNode)) {
136            LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
137                    "Same as last announced node, not speaking");
138
139            AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
140            AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
141            return false;
142        }
143
144        // If neither succeeded, abort.
145        if (!addDescription(context, event, utterance, source, announcedNode)) {
146            LogUtils.log(TouchExplorationFormatter.class, Log.INFO,
147                    "Failed to populate utterance, not speaking");
148
149            AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
150            AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
151            return false;
152        }
153
154        addEarcons(announcedNode, sourceAndPredecessors, event, utterance);
155
156        setLastAnnouncedNode(announcedNode);
157
158        AccessibilityNodeInfoUtils.recycleNodeList(sourceAndPredecessors);
159        AccessibilityNodeInfoUtils.recycleNodes(announcedNode);
160        return true;
161    }
162
163    /**
164     * Sets the most recently announced node.
165     * 
166     * @param node The node to set.
167     */
168    private void setLastAnnouncedNode(AccessibilityNodeInfo node) {
169        if (mLastAnnouncedNode != null) {
170            mLastAnnouncedNode.recycle();
171        }
172
173        if (node != null) {
174            mLastAnnouncedNode = AccessibilityNodeInfo.obtain(node);
175        } else {
176            mLastAnnouncedNode = null;
177        }
178    }
179
180    private enum DescriptionResult {
181        SUCCESS, FAILURE, ABORT
182    }
183
184    /**
185     * Populates an utterance with text, either from the node or event.
186     * 
187     * @param context The parent context.
188     * @param event The source event.
189     * @param utterance The target utterance.
190     * @param source The source node.
191     * @param announcedNode The computed announced node.
192     * @return {@code true} if the utterance was populated with text.
193     */
194    private boolean addDescription(Context context, AccessibilityEvent event, Utterance utterance,
195            AccessibilityNodeInfo source, AccessibilityNodeInfo announcedNode) {
196        // Attempt to populate with node description.
197        final DescriptionResult result =
198                addNodeDescription(context, event, utterance, source, announcedNode);
199
200        // If the node description failed (but didn't abort) try the event.
201        if (result == DescriptionResult.FAILURE) {
202            return addEventDescription(context, event, utterance);
203        }
204
205        return (result == DescriptionResult.SUCCESS);
206    }
207
208    /**
209     * Populates an utterance with text from an event.
210     * 
211     * @param context The parent context.
212     * @param event The source event.
213     * @param utterance The target utterance.
214     */
215    private boolean addEventDescription(Context context, AccessibilityEvent event,
216            Utterance utterance) {
217        final CharSequence eventText = Utils.getEventText(context, event);
218
219        if (TextUtils.isEmpty(eventText)) {
220            return false;
221        }
222
223        utterance.getText().append(eventText);
224
225        return true;
226    }
227
228    /**
229     * Adds a description to an utterance for the specified node.
230     * 
231     * @param context The parent context.
232     * @param event The source event.
233     * @param utterance The output utterance.
234     * @param source The source node.
235     * @param announcedNode The node to announce.
236     */
237    private DescriptionResult addNodeDescription(Context context, AccessibilityEvent event,
238            Utterance utterance, AccessibilityNodeInfo source,
239            AccessibilityNodeInfo announcedNode) {
240        final StringBuilder builder = utterance.getText();
241
242        if (announcedNode == null) {
243            return DescriptionResult.FAILURE;
244        }
245
246        final CharSequence desc = announcedNode.getContentDescription();
247
248        // HACK: If the node to announce is a view group and already has a
249        // content description, don't bother fetching its children.
250        if (!TextUtils.isEmpty(desc) && Utils.isViewGroup(null, announcedNode)) {
251            builder.append(desc);
252            return DescriptionResult.SUCCESS;
253        }
254
255        final AccessibilityNodeInfo announcedNodeClone =
256                AccessibilityNodeInfo.obtain(announcedNode);
257        final ArrayList<AccessibilityNodeInfo> announcedSubtreeNodes =
258                new ArrayList<AccessibilityNodeInfo>();
259
260        // Fetch the subtree of the node to announce.
261        announcedSubtreeNodes.add(announcedNodeClone);
262
263        final int failedFilter =
264                AccessibilityNodeInfoUtils.addDescendantsBfs(announcedNode, announcedSubtreeNodes,
265                        mNonActionableInfoFilter);
266
267        Collections.sort(announcedSubtreeNodes, COMPARATOR);
268
269        // Make sure we have a node processor.
270        // TODO(alanv): We should do this on initialization.
271        if (mNodeProcessor == null) {
272            mNodeProcessor = new NodeProcessor(context);
273        }
274
275        // Add text for the nodes. If we have text (or there are children that
276        // failed the filter) then we've handled the node.
277        final boolean handled = addNodesText(event, source, announcedSubtreeNodes, utterance);
278
279        AccessibilityNodeInfoUtils.recycleNodeList(announcedSubtreeNodes);
280
281        if (handled) {
282            return DescriptionResult.SUCCESS;
283        } else if (failedFilter > 0) {
284            return DescriptionResult.ABORT;
285        } else {
286            return DescriptionResult.FAILURE;
287        }
288    }
289
290    private void onTouchExplorationStarted(Utterance utterance) {
291        if (!mIsTouchExploring) {
292            utterance.getEarcons().add(R.raw.explore_begin);
293        }
294
295        mUserCanScroll = false;
296        mIsTouchExploring = true;
297    }
298
299    private void onTouchExplorationEnded(Utterance utterance) {
300        if (mIsTouchExploring) {
301            utterance.getEarcons().add(R.raw.explore_end);
302        }
303
304        if (mLastAnnouncedNode != null) {
305            mLastAnnouncedNode.recycle();
306            mLastAnnouncedNode = null;
307        }
308
309        mIsTouchExploring = false;
310    }
311
312    /**
313     * Adds earcons when moving between scrollable and non-scrollable views.
314     * 
315     * @param announcedNode The node that is announces.
316     * @param eventSourcePredecessors The event source predecessors.
317     * @param event The received accessibility event.
318     * @param utterance The utterance to which to add the earcons.
319     */
320    private void addEarcons(AccessibilityNodeInfo announcedNode,
321            ArrayList<AccessibilityNodeInfo> eventSourcePredecessors, AccessibilityEvent event,
322            Utterance utterance) {
323        // If the announced node is in a scrollable container - add an earcon to
324        // convey that.
325        final boolean userCanScroll =
326                isScrollableOrHasScrollablePredecessor(announcedNode, eventSourcePredecessors);
327        if (mUserCanScroll != userCanScroll) {
328            mUserCanScroll = userCanScroll;
329            if (userCanScroll) {
330                utterance.getEarcons().add(R.raw.chime_up);
331            } else {
332                utterance.getEarcons().add(R.raw.chime_down);
333            }
334        }
335
336        final boolean actionable =
337                isActionableOrHasActionablePredecessor(announcedNode, eventSourcePredecessors);
338        if (actionable) {
339            utterance.getCustomEarcons().add(R.string.pref_sounds_actionable_key);
340            utterance.getCustomVibrations().add(R.string.pref_patterns_actionable_key);
341        } else {
342            utterance.getCustomEarcons().add(R.string.pref_sounds_hover_key);
343            utterance.getCustomVibrations().add(R.string.pref_patterns_hover_key);
344        }
345    }
346
347    /**
348     * Computes the node that is to be announced.
349     * 
350     * @param nodes The candidate nodes.
351     * @return The node to announce or null if no such is found.
352     */
353    private AccessibilityNodeInfo computeAnnouncedNode(ArrayList<AccessibilityNodeInfo> nodes) {
354        for (AccessibilityNodeInfo node : nodes) {
355            if (shouldAnnounceNode(node)) {
356                return AccessibilityNodeInfo.obtain(node);
357            }
358        }
359
360        return null;
361    }
362
363    /**
364     * Determines whether a node should to be announced. This is used when
365     * traversing up from a touched node to ensure that enough information is
366     * spoken to provide context for what will happen if the user performs an
367     * action on the item.
368     * 
369     * @param node The examined node.
370     * @return True if the node is to be announced.
371     */
372    private boolean shouldAnnounceNode(AccessibilityNodeInfo node) {
373        // Do not announce AdapterView or anything that extends it.
374        if (AccessibilityNodeInfoUtils.nodeMatchesClassByType(node, AdapterView.class)) {
375            return false;
376        }
377
378        // Always announce "actionable" nodes.
379        if (AccessibilityNodeInfoUtils.isActionable(node)) {
380            return true;
381        }
382
383        // If the parent is an AdapterView, then the node is a list item, so
384        // announce it.
385        final AccessibilityNodeInfo parent = node.getParent();
386
387        if (parent == null) {
388            return false;
389        }
390
391        final boolean isListItem =
392                AccessibilityNodeInfoUtils.nodeMatchesClassByType(parent, AdapterView.class);
393        parent.recycle();
394
395        return isListItem;
396    }
397
398    /**
399     * Adds the text of the given nodes to the specified utterance.
400     * 
401     * @param nodes The nodes whose text to add.
402     * @param utterance The utterance to which to add text.
403     */
404    private boolean addNodesText(AccessibilityEvent event, AccessibilityNodeInfo source,
405            ArrayList<AccessibilityNodeInfo> nodes, Utterance utterance) {
406        final StringBuilder builder = utterance.getText();
407
408        for (AccessibilityNodeInfo node : nodes) {
409            final CharSequence nodeDesc;
410
411            if (node.equals(source)) {
412                nodeDesc = mNodeProcessor.process(node, event);
413            } else {
414                nodeDesc = mNodeProcessor.process(node, null);
415            }
416
417            if (TextUtils.isEmpty(nodeDesc)) {
418                continue;
419            }
420
421            builder.append(nodeDesc);
422            builder.append(SEPARATOR);
423        }
424
425        return !TextUtils.isEmpty(builder);
426    }
427
428    /**
429     * Check whether a given node is actionable or has an actionable
430     * predecessor.
431     * 
432     * @param source The announced node.
433     * @param predecessors The predecessors of the event source node.
434     * @return True if the node or some of its predecessors is actionable.
435     */
436    private boolean isActionableOrHasActionablePredecessor(AccessibilityNodeInfo source,
437            List<AccessibilityNodeInfo> predecessors) {
438        // Nothing is always non-actionable.
439        if (source == null || predecessors == null) {
440            return false;
441        }
442
443        // Check whether the node is actionable.
444        if (AccessibilityNodeInfoUtils.isActionable(source)) {
445            return true;
446        }
447
448        // Check whether the any of the node predecessors is actionable.
449        for (AccessibilityNodeInfo predecessor : predecessors) {
450            if (AccessibilityNodeInfoUtils.isActionable(predecessor)) {
451                return true;
452            }
453        }
454
455        return false;
456    }
457
458    /**
459     * Check whether a given node is scrollable or has a scrollable predecessor.
460     * 
461     * @param source The announced node.
462     * @param predecessors The predecessors of the event source node.
463     * @return True if the node or some of its predecessors is scrollable.
464     */
465    private boolean isScrollableOrHasScrollablePredecessor(AccessibilityNodeInfo source,
466            List<AccessibilityNodeInfo> predecessors) {
467        // Nothing is always non-actionable.
468        if (source == null || predecessors == null) {
469            return false;
470        }
471
472        // Check whether the node is scrollable.
473        if (source.isScrollable()) {
474            return true;
475        }
476
477        // Check whether the any of the node predecessors is scrollable.
478        for (AccessibilityNodeInfo predecessor : predecessors) {
479            if (predecessor.isScrollable()) {
480                return true;
481            }
482        }
483
484        return false;
485    }
486
487    /**
488     * Compares two AccessibilityNodeInfos in left-to-right and top-to-bottom
489     * fashion.
490     */
491    private static class TopToBottomLeftToRightComparator implements
492            Comparator<AccessibilityNodeInfo> {
493        private final Rect mFirstBounds = new Rect();
494        private final Rect mSecondBounds = new Rect();
495
496        @Override
497        public int compare(AccessibilityNodeInfo first, AccessibilityNodeInfo second) {
498            Rect firstBounds = mFirstBounds;
499            first.getBoundsInScreen(firstBounds);
500
501            Rect secondBounds = mSecondBounds;
502            second.getBoundsInScreen(secondBounds);
503
504            // top - bottom
505            final int topDifference = firstBounds.top - secondBounds.top;
506            if (topDifference != 0) {
507                return topDifference;
508            }
509            // left - right
510            final int leftDifference = firstBounds.left - secondBounds.left;
511            if (leftDifference != 0) {
512                return leftDifference;
513            }
514            // break tie by height
515            final int firstHeight = firstBounds.bottom - firstBounds.top;
516            final int secondHeight = secondBounds.bottom - secondBounds.top;
517            final int heightDiference = firstHeight - secondHeight;
518            if (heightDiference != 0) {
519                return heightDiference;
520            }
521            // break tie by width
522            final int firstWidth = firstBounds.right - firstBounds.left;
523            final int secondWidth = secondBounds.right - secondBounds.left;
524            int widthDiference = firstWidth - secondWidth;
525            if (widthDiference != 0) {
526                return widthDiference;
527            }
528            // do not return 0 to avoid losing data
529            return 1;
530        }
531    }
532}