/ocr/ocrservice/src/com/googlecode/eyesfree/opticflow/OpticalFlowProcessor.java

  1/*
  2 * Copyright (C) 2011 Google Inc.
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5 * use this file except in compliance with the License. You may obtain a copy of
  6 * the License at
  7 *
  8 * http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 13 * License for the specific language governing permissions and limitations under
 14 * the License.
 15 */
 16
 17package com.googlecode.eyesfree.opticflow;
 18
 19import android.graphics.Canvas;
 20import android.graphics.Color;
 21import android.graphics.Paint;
 22import android.graphics.PointF;
 23import android.graphics.Typeface;
 24import android.os.SystemClock;
 25
 26import com.googlecode.eyesfree.env.Size;
 27
 28import java.util.Vector;
 29
 30/**
 31 * Frame processor that tracks optical flow.
 32 *
 33 * Modified by Alan Viverette from Andrew Harp's original source.
 34 *
 35 * @author Andrew Harp
 36 * @author alanv@google.com (Alan Viverette)
 37 */
 38public class OpticalFlowProcessor extends FrameProcessor {
 39    private static final boolean DRAW_TEXT = false;
 40
 41    // How many history points to keep track of and draw.
 42    private static final int MAX_HISTORY_SIZE = 30;
 43
 44    // The reduction factor in image dimensions for optical flow processing.
 45    private static final int DOWNSAMPLE_FACTOR = 4;
 46
 47    private int frameWidth;
 48
 49    private int frameHeight;
 50
 51    private final Paint p;
 52
 53    private final Vector<PointF> history;
 54
 55    private FrameChange lastFeatures;
 56
 57    private long lastTimestamp;
 58
 59    private OpticalFlow mOpticalFlow;
 60
 61    public OpticalFlowProcessor() {
 62        history = new Vector<PointF>(MAX_HISTORY_SIZE);
 63
 64        p = new Paint();
 65        p.setAntiAlias(false);
 66        p.setTypeface(Typeface.SERIF);
 67
 68        mOpticalFlow = new OpticalFlow();
 69    }
 70
 71    public static class Feature {
 72        public final float x;
 73
 74        public final float y;
 75
 76        public final float score;
 77
 78        public final int type;
 79
 80        public Feature(final float x, final float y) {
 81            this.x = x;
 82            this.y = y;
 83            this.score = 0;
 84            this.type = -1;
 85        }
 86
 87        public Feature(final float x, final float y, final float score, final int type) {
 88            this.x = x;
 89            this.y = y;
 90            this.score = score;
 91            this.type = type;
 92        }
 93
 94        Feature delta(final Feature other) {
 95            return new Feature(this.x - other.x, this.y - other.y);
 96        }
 97    }
 98
 99    public static class PointChange {
100        public final Feature featureA;
101
102        public final Feature featureB;
103
104        Feature pointDelta;
105
106        public PointChange(final float x1, final float y1, final float x2, final float y2,
107                final float score, final int type) {
108            featureA = new Feature(x1, y1, score, type);
109            featureB = new Feature(x2, y2);
110        }
111
112        public Feature getDelta() {
113            if (pointDelta == null) {
114                pointDelta = featureB.delta(featureA);
115            }
116            return pointDelta;
117        }
118    }
119
120    /**
121     * A class that records a timestamped frame translation delta for optical
122     * flow.
123     */
124    public static class FrameChange {
125        public final Vector<PointChange> pointDeltas;
126
127        private final float minScore;
128
129        private final float maxScore;
130
131        public FrameChange(final float[] framePoints) {
132            final int featureStep = 7;
133
134            float minScore = 100.0f;
135            float maxScore = -100.0f;
136
137            pointDeltas = new Vector<PointChange>(framePoints.length / featureStep);
138            float totalChangeX = 0.0f;
139            float totalChangeY = 0.0f;
140            for (int i = 0; i < framePoints.length; i += featureStep) {
141                final float x1 = framePoints[i + 0];
142                final float y1 = framePoints[i + 1];
143
144                final float x2 = framePoints[i + 3];
145                final float y2 = framePoints[i + 4];
146                final float score = framePoints[i + 5];
147                final int type = (int) framePoints[i + 6];
148
149                minScore = Math.min(minScore, score);
150                maxScore = Math.max(maxScore, score);
151
152                final PointChange pointDelta = new PointChange(x1, y1, x2, y2, score, type);
153                final Feature change = pointDelta.getDelta();
154
155                totalChangeX += change.x;
156                totalChangeY += change.y;
157
158                pointDeltas.add(pointDelta);
159            }
160
161            this.minScore = minScore;
162            this.maxScore = maxScore;
163        }
164    }
165
166    @Override
167    protected synchronized void onPreprocess(final TimestampedFrame frame) {
168        mOpticalFlow.setImage(frame.getRawData(), frame.getTimestamp());
169    }
170
171    @Override
172    protected synchronized void onProcessFrame(final TimestampedFrame frame) {
173        mOpticalFlow.computeOpticalFlow();
174
175        if (DebugView.isVisible) {
176            updateHistory();
177        }
178    }
179
180    public OpticalFlow getOpticalFlow() {
181        return mOpticalFlow;
182    }
183
184    @Override
185    protected synchronized void onInit(final Size size) {
186        this.frameWidth = size.width;
187        this.frameHeight = size.height;
188
189        mOpticalFlow.initialize(frameWidth, frameHeight, DOWNSAMPLE_FACTOR);
190    }
191
192    @Override
193    protected synchronized void onShutdown() {
194        mOpticalFlow = null;
195    }
196
197    private void drawHistory(final Canvas canvas) {
198        drawHistoryPoint(canvas, canvas.getWidth() / 2, canvas.getHeight() / 2);
199    }
200
201    private void drawHistoryPoint(final Canvas canvas, final float startX, final float startY) {
202        // Draw the center circle.
203        p.setColor(Color.GREEN);
204        canvas.drawCircle(startX, startY, 3.0f, p);
205
206        p.setColor(Color.RED);
207        p.setStrokeWidth(2.0f);
208
209        float x1 = startX;
210        float y1 = startY;
211
212        // Iterate through in backwards order.
213        synchronized (history) {
214            final int numPoints = history.size();
215            for (int featureNum = numPoints - 1; featureNum >= 0; --featureNum) {
216                final PointF delta = history.get(featureNum);
217
218                final float x2 = x1 + delta.x;
219                final float y2 = y1 + delta.y;
220
221                canvas.drawLine(x1, y1, x2, y2, p);
222
223                x1 = x2;
224                y1 = y2;
225            }
226        }
227    }
228
229    private static int floatToChar(final float value) {
230        return Math.max(0, Math.min((int) (value * 255.999f), 255));
231    }
232
233    private void drawFeatures(final Canvas canvas) {
234        if (lastFeatures == null) {
235            return;
236        }
237        final int featureSize = 3;
238
239        final float minScore = lastFeatures.minScore;
240        final float maxScore = lastFeatures.maxScore;
241
242        for (final PointChange feature : lastFeatures.pointDeltas) {
243            final int r = floatToChar((feature.featureA.score - minScore) / (maxScore - minScore));
244            final int b = floatToChar(
245                    1.0f - (feature.featureA.score - minScore) / (maxScore - minScore));
246
247            final int color = 0xFF000000 | (r << 16) | b;
248            p.setColor(color);
249
250            canvas.drawRect(feature.featureB.x - featureSize, feature.featureB.y - featureSize,
251                    feature.featureB.x + featureSize, feature.featureB.y + featureSize, p);
252
253            p.setColor(Color.CYAN);
254            canvas.drawLine(feature.featureB.x, feature.featureB.y, feature.featureA.x,
255                    feature.featureA.y, p);
256
257            if (DRAW_TEXT) {
258                p.setColor(Color.WHITE);
259                canvas.drawText(feature.featureA.type + ": " + feature.featureA.score,
260                        feature.featureA.x, feature.featureA.y, p);
261            }
262        }
263    }
264
265    private void updateHistory() {
266        final float[] features = mOpticalFlow.getFeatures(true);
267
268        lastFeatures = new FrameChange(features);
269
270        final long timestamp = SystemClock.uptimeMillis();
271
272        final PointF delta = mOpticalFlow.getAccumulatedDelta(
273                lastTimestamp, frameWidth / 2, frameHeight / 2, 100);
274        lastTimestamp = timestamp;
275
276        synchronized (history) {
277            history.add(delta);
278
279            while (history.size() > MAX_HISTORY_SIZE) {
280                history.remove(0);
281            }
282        }
283    }
284
285    @Override
286    protected synchronized void onDrawDebug(final Canvas canvas) {
287        drawHistory(canvas);
288        drawFeatures(canvas);
289    }
290
291    @Override
292    protected Vector<String> getDebugText() {
293        final Vector<String> lines = new Vector<String>();
294
295        if (lastFeatures != null) {
296            lines.add("Num features " + lastFeatures.pointDeltas.size());
297            lines.add("Min score: " + lastFeatures.minScore);
298            lines.add("Max score: " + lastFeatures.maxScore);
299        }
300
301        return lines;
302    }
303}