/ime/latinime/src/com/googlecode/eyesfree/inputmethod/voice/RecognitionView.java

  1/*
  2 * Copyright (C) 2009 Google Inc.
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5 * use this file except in compliance with the License. You may obtain a copy of
  6 * the License at
  7 *
  8 * http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 13 * License for the specific language governing permissions and limitations under
 14 * the License.
 15 */
 16
 17package com.googlecode.eyesfree.inputmethod.voice;
 18
 19import android.content.ContentResolver;
 20import android.content.Context;
 21import android.content.res.Resources;
 22import android.graphics.Bitmap;
 23import android.graphics.Canvas;
 24import android.graphics.CornerPathEffect;
 25import android.graphics.Paint;
 26import android.graphics.Path;
 27import android.graphics.PathEffect;
 28import android.graphics.drawable.Drawable;
 29import android.os.Handler;
 30import android.util.TypedValue;
 31import android.view.LayoutInflater;
 32import android.view.View;
 33import android.view.View.OnClickListener;
 34import android.view.ViewGroup.MarginLayoutParams;
 35import android.widget.ImageView;
 36import android.widget.ProgressBar;
 37import android.widget.TextView;
 38
 39import com.googlecode.eyesfree.inputmethod.latin.R;
 40
 41import java.io.ByteArrayOutputStream;
 42import java.nio.ByteBuffer;
 43import java.nio.ByteOrder;
 44import java.nio.ShortBuffer;
 45import java.util.ArrayList;
 46import java.util.List;
 47
 48/**
 49 * The user interface for the "Speak now" and "working" states.
 50 * Displays a recognition dialog (with waveform, voice meter, etc.),
 51 * plays beeps, shows errors, etc.
 52 */
 53public class RecognitionView {
 54    private Handler mUiHandler;  // Reference to UI thread
 55    private View mView;
 56    private Context mContext;
 57
 58    private ImageView mImage;
 59    private TextView mText;
 60    private View mButton;
 61    private TextView mButtonText;
 62    private View mProgress;
 63
 64    private Drawable mInitializing;
 65    private Drawable mError;
 66    private List<Drawable> mSpeakNow;
 67
 68    private float mVolume = 0.0f;
 69    private int mLevel = 0;
 70
 71    private enum State {LISTENING, WORKING, READY}
 72    private State mState = State.READY;
 73
 74    private float mMinMicrophoneLevel;
 75    private float mMaxMicrophoneLevel;
 76
 77    /** Updates the microphone icon to show user their volume.*/
 78    private Runnable mUpdateVolumeRunnable = new Runnable() {
 79        public void run() {
 80            if (mState != State.LISTENING) {
 81                return;
 82            }
 83
 84            final float min = mMinMicrophoneLevel;
 85            final float max = mMaxMicrophoneLevel;
 86            final int maxLevel = mSpeakNow.size() - 1;
 87
 88            int index = (int) ((mVolume - min) / (max - min) * maxLevel);
 89            final int level = Math.min(Math.max(0, index), maxLevel);
 90
 91            if (level != mLevel) {
 92                mImage.setImageDrawable(mSpeakNow.get(level));
 93                mLevel = level;
 94            }
 95            mUiHandler.postDelayed(mUpdateVolumeRunnable, 50);
 96        }
 97      };
 98
 99    public RecognitionView(Context context, OnClickListener clickListener) {
100        mUiHandler = new Handler();
101
102        LayoutInflater inflater = (LayoutInflater) context.getSystemService(
103            Context.LAYOUT_INFLATER_SERVICE);
104        mView = inflater.inflate(R.layout.recognition_status, null);
105        ContentResolver cr = context.getContentResolver();
106        mMinMicrophoneLevel = SettingsUtil.getSettingsFloat(
107                cr, SettingsUtil.LATIN_IME_MIN_MICROPHONE_LEVEL, 15.f);
108        mMaxMicrophoneLevel = SettingsUtil.getSettingsFloat(
109                cr, SettingsUtil.LATIN_IME_MAX_MICROPHONE_LEVEL, 30.f);
110
111        // Pre-load volume level images
112        Resources r = context.getResources();
113
114        mSpeakNow = new ArrayList<Drawable>();
115        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level0));
116        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level1));
117        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level2));
118        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level3));
119        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level4));
120        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level5));
121        mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level6));
122
123        mInitializing = r.getDrawable(R.drawable.mic_slash);
124        mError = r.getDrawable(R.drawable.caution);
125
126        mImage = (ImageView) mView.findViewById(R.id.image);
127        mButton = mView.findViewById(R.id.button);
128        mButton.setOnClickListener(clickListener);
129        mText = (TextView) mView.findViewById(R.id.text);
130        mButtonText = (TextView) mView.findViewById(R.id.button_text);
131        mProgress = mView.findViewById(R.id.progress);
132
133        mContext = context;
134    }
135
136    public View getView() {
137        return mView;
138    }
139
140    public void restoreState() {
141        mUiHandler.post(new Runnable() {
142            public void run() {
143                // Restart the spinner
144                if (mState == State.WORKING) {
145                    ((ProgressBar)mProgress).setIndeterminate(false);
146                    ((ProgressBar)mProgress).setIndeterminate(true);
147                }
148            }
149        });
150    }
151
152    public void showInitializing() {
153        mUiHandler.post(new Runnable() {
154            public void run() {
155                prepareDialog(false, mContext.getText(R.string.voice_initializing), mInitializing,
156                        mContext.getText(R.string.cancel)); 
157            }
158          });
159    }
160
161    public void showListening() {
162        mUiHandler.post(new Runnable() {
163            public void run() {
164                mState = State.LISTENING;
165                prepareDialog(false, mContext.getText(R.string.voice_listening), mSpeakNow.get(0),
166                        mContext.getText(R.string.cancel));
167            }
168          });
169        mUiHandler.postDelayed(mUpdateVolumeRunnable, 50);
170    }
171
172    public void updateVoiceMeter(final float rmsdB) {
173        mVolume = rmsdB;
174    }
175
176    public void showError(final String message) {
177        mUiHandler.post(new Runnable() {
178            public void run() {
179                mState = State.READY;
180                prepareDialog(false, message, mError, mContext.getText(R.string.ok));
181            }
182          });
183    }
184
185    public void showWorking(
186        final ByteArrayOutputStream waveBuffer,
187        final int speechStartPosition,
188        final int speechEndPosition) {
189
190        mUiHandler.post(new Runnable() {
191            public void run() {
192                mState = State.WORKING;
193                prepareDialog(true, mContext.getText(R.string.voice_working), null, mContext
194                        .getText(R.string.cancel));
195                final ShortBuffer buf = ByteBuffer.wrap(waveBuffer.toByteArray()).order(
196                        ByteOrder.nativeOrder()).asShortBuffer();
197                buf.position(0);
198                waveBuffer.reset();
199                showWave(buf, speechStartPosition / 2, speechEndPosition / 2);
200            }
201          });
202    }
203    
204    private void prepareDialog(boolean spinVisible, CharSequence text, Drawable image,
205            CharSequence btnTxt) {
206        if (spinVisible) {
207            mProgress.setVisibility(View.VISIBLE);
208            mImage.setVisibility(View.GONE);
209        } else {
210            mProgress.setVisibility(View.GONE);
211            mImage.setImageDrawable(image);
212            mImage.setVisibility(View.VISIBLE);
213        }
214        mText.setText(text);
215        mButtonText.setText(btnTxt);
216    }
217
218    /**
219     * @return an average abs of the specified buffer.
220     */
221    private static int getAverageAbs(ShortBuffer buffer, int start, int i, int npw) {
222        int from = start + i * npw;
223        int end = from + npw;
224        int total = 0;
225        for (int x = from; x < end; x++) {
226            total += Math.abs(buffer.get(x));
227        }
228        return total / npw;
229    }
230
231
232    /**
233     * Shows waveform of input audio.
234     *
235     * Copied from version in VoiceSearch's RecognitionActivity.
236     *
237     * TODO: adjust stroke width based on the size of data.
238     * TODO: use dip rather than pixels.
239     */
240    private void showWave(ShortBuffer waveBuffer, int startPosition, int endPosition) {
241        final int w = ((View) mImage.getParent()).getWidth();
242        final int h = mImage.getHeight();
243        if (w <= 0 || h <= 0) {
244            // view is not visible this time. Skip drawing.
245            return;
246        }
247        final Bitmap b = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888);
248        final Canvas c = new Canvas(b);
249        final Paint paint = new Paint();
250        paint.setColor(0xFFFFFFFF); // 0xAARRGGBB
251        paint.setAntiAlias(true);
252        paint.setStyle(Paint.Style.STROKE);
253        paint.setAlpha(0x90);
254
255        final PathEffect effect = new CornerPathEffect(3);
256        paint.setPathEffect(effect);
257
258        final int numSamples = waveBuffer.remaining();
259        int endIndex;
260        if (endPosition == 0) {
261            endIndex = numSamples;
262        } else {
263            endIndex = Math.min(endPosition, numSamples);
264        }
265
266        int startIndex = startPosition - 2000; // include 250ms before speech
267        if (startIndex < 0) {
268            startIndex = 0;
269        }
270        final int numSamplePerWave = 200;  // 8KHz 25ms = 200 samples
271        final float scale = 10.0f / 65536.0f;
272
273        final int count = (endIndex - startIndex) / numSamplePerWave;
274        final float deltaX = 1.0f * w / count;
275        int yMax = h / 2 - 8;
276        Path path = new Path();
277        c.translate(0, yMax);
278        float x = 0;
279        path.moveTo(x, 0);
280        for (int i = 0; i < count; i++) {
281            final int avabs = getAverageAbs(waveBuffer, startIndex, i , numSamplePerWave);
282            int sign = ( (i & 01) == 0) ? -1 : 1;
283            final float y = Math.min(yMax, avabs * h * scale) * sign;
284            path.lineTo(x, y);
285            x += deltaX;
286            path.lineTo(x, y);
287        }
288        if (deltaX > 4) {
289            paint.setStrokeWidth(3);
290        } else {
291            paint.setStrokeWidth(Math.max(1, (int) (deltaX -.05)));
292        }
293        c.drawPath(path, paint);
294        mImage.setImageBitmap(b);
295        mImage.setVisibility(View.VISIBLE);
296        MarginLayoutParams mProgressParams = (MarginLayoutParams)mProgress.getLayoutParams();
297        mProgressParams.topMargin = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_PX,
298                -h , mContext.getResources().getDisplayMetrics());
299
300        // Tweak the padding manually to fill out the whole view horizontally.
301        // TODO: Do this in the xml layout instead.
302        ((View) mImage.getParent()).setPadding(4, ((View) mImage.getParent()).getPaddingTop(), 3,
303                ((View) mImage.getParent()).getPaddingBottom());
304        mProgress.setLayoutParams(mProgressParams);
305    }
306
307
308    public void finish() {
309        mUiHandler.post(new Runnable() {
310            public void run() {
311                mState = State.READY;
312                exitWorking();
313            }
314          });
315    }
316
317    private void exitWorking() {
318        mProgress.setVisibility(View.GONE);
319        mImage.setVisibility(View.VISIBLE);
320    }
321}