/opengl/tests/gl2_cameraeye/src/com/android/gl2cameraeye/GL2CameraEye.java

  1/*
  2 * Copyright (C) 2011 The Android Open Source Project
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License");
  5 * you may not use this file except in compliance with the License.
  6 * You may obtain a copy of the License at
  7 *
  8 *      http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS,
 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 * See the License for the specific language governing permissions and
 14 * limitations under the License.
 15 */
 16
 17package com.android.gl2cameraeye;
 18
 19import java.io.IOException;
 20import java.nio.ByteBuffer;
 21import java.nio.ByteOrder;
 22import java.nio.FloatBuffer;
 23
 24import javax.microedition.khronos.egl.EGLConfig;
 25import javax.microedition.khronos.opengles.GL10;
 26
 27import android.app.Activity;
 28import android.content.pm.ActivityInfo;
 29import android.os.Bundle;
 30import android.view.MotionEvent;
 31import android.content.Context;
 32import android.util.Log;
 33
 34import android.opengl.GLES20;
 35import android.opengl.GLSurfaceView;
 36import android.opengl.GLUtils;
 37import android.opengl.Matrix;
 38
 39import android.graphics.SurfaceTexture;
 40
 41import android.hardware.Camera;
 42import android.hardware.SensorManager;
 43import android.hardware.SensorEvent;
 44import android.hardware.SensorEventListener;
 45import android.hardware.Sensor;
 46
 47public class GL2CameraEye extends Activity {
 48    @Override
 49    protected void onCreate(Bundle savedInstanceState) {
 50        super.onCreate(savedInstanceState);
 51        mGLView = new CamGLSurfaceView(this);
 52        setContentView(mGLView);
 53        setRequestedOrientation (ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE);
 54    }
 55
 56    @Override
 57    protected void onPause() {
 58        super.onPause();
 59        mGLView.onPause();
 60    }
 61
 62    @Override
 63    protected void onResume() {
 64        super.onResume();
 65        mGLView.onResume();
 66    }
 67
 68    private GLSurfaceView mGLView;
 69}
 70
 71class CamGLSurfaceView extends GLSurfaceView implements SensorEventListener {
 72    public CamGLSurfaceView(Context context) {
 73        super(context);
 74        setEGLContextClientVersion(2);
 75        mRenderer = new CamRenderer(context);
 76        setRenderer(mRenderer);
 77
 78        mSensorManager = (SensorManager)context.getSystemService(Context.SENSOR_SERVICE);
 79        mAcceleration = mSensorManager.getDefaultSensor(Sensor.TYPE_LINEAR_ACCELERATION);
 80    }
 81
 82    public boolean onTouchEvent(final MotionEvent event) {
 83        queueEvent(new Runnable(){
 84                public void run() {
 85                mRenderer.setPosition(event.getX() / getWidth(),
 86                                      event.getY() / getHeight());
 87            }});
 88        return true;
 89    }
 90
 91    @Override
 92    public void onPause() {
 93        super.onPause();
 94        mCamera.stopPreview();
 95        mCamera.release();
 96
 97        mSensorManager.unregisterListener(this);
 98    }
 99
100    @Override
101    public void onResume() {
102        mCamera = Camera.open();
103        Camera.Parameters p = mCamera.getParameters();
104        // No changes to default camera parameters
105        mCamera.setParameters(p);
106
107        queueEvent(new Runnable(){
108                public void run() {
109                    mRenderer.setCamera(mCamera);
110                }});
111
112        mSensorManager.registerListener(this, mAcceleration, SensorManager.SENSOR_DELAY_GAME);
113        super.onResume();
114    }
115
116    public void onSensorChanged(SensorEvent event) {
117        if (event.sensor.getType() == Sensor.TYPE_LINEAR_ACCELERATION) {
118            final float[] accelerationVector = event.values;
119            queueEvent(new Runnable(){
120                    public void run() {
121                        mRenderer.setAcceleration(accelerationVector);
122                    }});
123        }
124    }
125
126    public void onAccuracyChanged(Sensor sensor, int accuracy) {
127        // Ignoring sensor accuracy changes.
128    }
129
130    CamRenderer mRenderer;
131    Camera mCamera;
132
133    SensorManager mSensorManager;
134    Sensor mAcceleration;
135}
136
137class CamRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
138
139    public CamRenderer(Context context) {
140        mContext = context;
141
142        mTriangleVertices = ByteBuffer.allocateDirect(mTriangleVerticesData.length
143                * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
144        mTriangleVertices.put(mTriangleVerticesData).position(0);
145
146        Matrix.setIdentityM(mSTMatrix, 0);
147        Matrix.setIdentityM(mMMatrix, 0);
148
149        float[] defaultAcceleration = {0.f,0.f,0.f};
150        setAcceleration(defaultAcceleration);
151        mPos[0] = 0.f;
152        mPos[1] = 0.f;
153        mPos[2] = 0.f;
154        mVel[0] = 0.f;
155        mVel[1] = 0.f;
156        mVel[2] = 0.f;
157
158    }
159
160    /* The following set methods are not synchronized, so should only
161     * be called within the rendering thread context. Use GLSurfaceView.queueEvent for safe access.
162     */
163    public void setPosition(float x, float y) {
164        /* Map from screen (0,0)-(1,1) to scene coordinates */
165        mPos[0] = (x*2-1)*mRatio;
166        mPos[1] = (-y)*2+1;
167        mPos[2] = 0.f;
168        mVel[0] = 0;
169        mVel[1] = 0;
170        mVel[2] = 0;
171    }
172
173    public void setCamera(Camera camera) {
174        mCamera = camera;
175        Camera.Size previewSize = camera.getParameters().getPreviewSize();
176        mCameraRatio = (float)previewSize.width/previewSize.height;
177    }
178
179    public void setAcceleration(float[] accelerationVector) {
180        mGForce[0] = accelerationVector[0];
181        mGForce[1] = accelerationVector[1];
182        mGForce[2] = accelerationVector[2];
183    }
184
185    public void onDrawFrame(GL10 glUnused) {
186        synchronized(this) {
187            if (updateSurface) {
188                mSurface.updateTexImage();
189
190                mSurface.getTransformMatrix(mSTMatrix);
191                long timestamp = mSurface.getTimestamp();
192                doPhysics(timestamp);
193
194                updateSurface = false;
195            }
196        }
197
198        // Ignore the passed-in GL10 interface, and use the GLES20
199        // class's static methods instead.
200        GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
201        GLES20.glUseProgram(mProgram);
202        checkGlError("glUseProgram");
203
204        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
205        GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
206
207        mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
208        GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
209                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
210        checkGlError("glVertexAttribPointer maPosition");
211        GLES20.glEnableVertexAttribArray(maPositionHandle);
212        checkGlError("glEnableVertexAttribArray maPositionHandle");
213
214        mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
215        GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
216                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
217        checkGlError("glVertexAttribPointer maTextureHandle");
218        GLES20.glEnableVertexAttribArray(maTextureHandle);
219        checkGlError("glEnableVertexAttribArray maTextureHandle");
220
221        Matrix.multiplyMM(mMVPMatrix, 0, mVMatrix, 0, mMMatrix, 0);
222        Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVPMatrix, 0);
223
224        GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
225        GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
226        GLES20.glUniform1f(muCRatioHandle, mCameraRatio);
227
228        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
229        checkGlError("glDrawArrays");
230    }
231
232    public void onSurfaceChanged(GL10 glUnused, int width, int height) {
233        // Ignore the passed-in GL10 interface, and use the GLES20
234        // class's static methods instead.
235        GLES20.glViewport(0, 0, width, height);
236        mRatio = (float) width / height;
237        Matrix.frustumM(mProjMatrix, 0, -mRatio, mRatio, -1, 1, 3, 7);
238    }
239
240    public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
241        // Ignore the passed-in GL10 interface, and use the GLES20
242        // class's static methods instead.
243
244        /* Set up alpha blending and an Android background color */
245        GLES20.glEnable(GLES20.GL_BLEND);
246        GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
247        GLES20.glClearColor(0.643f, 0.776f, 0.223f, 1.0f);
248
249        /* Set up shaders and handles to their variables */
250        mProgram = createProgram(mVertexShader, mFragmentShader);
251        if (mProgram == 0) {
252            return;
253        }
254        maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
255        checkGlError("glGetAttribLocation aPosition");
256        if (maPositionHandle == -1) {
257            throw new RuntimeException("Could not get attrib location for aPosition");
258        }
259        maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
260        checkGlError("glGetAttribLocation aTextureCoord");
261        if (maTextureHandle == -1) {
262            throw new RuntimeException("Could not get attrib location for aTextureCoord");
263        }
264
265        muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
266        checkGlError("glGetUniformLocation uMVPMatrix");
267        if (muMVPMatrixHandle == -1) {
268            throw new RuntimeException("Could not get attrib location for uMVPMatrix");
269        }
270
271        muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
272        checkGlError("glGetUniformLocation uSTMatrix");
273        if (muMVPMatrixHandle == -1) {
274            throw new RuntimeException("Could not get attrib location for uSTMatrix");
275        }
276
277        muCRatioHandle = GLES20.glGetUniformLocation(mProgram, "uCRatio");
278        checkGlError("glGetUniformLocation uCRatio");
279        if (muMVPMatrixHandle == -1) {
280            throw new RuntimeException("Could not get attrib location for uCRatio");
281        }
282
283        /*
284         * Create our texture. This has to be done each time the
285         * surface is created.
286         */
287
288        int[] textures = new int[1];
289        GLES20.glGenTextures(1, textures, 0);
290
291        mTextureID = textures[0];
292        GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
293        checkGlError("glBindTexture mTextureID");
294
295        // Can't do mipmapping with camera source
296        GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
297                GLES20.GL_NEAREST);
298        GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
299                GLES20.GL_LINEAR);
300        // Clamp to edge is the only option
301        GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
302                GLES20.GL_CLAMP_TO_EDGE);
303        GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
304                GLES20.GL_CLAMP_TO_EDGE);
305        checkGlError("glTexParameteri mTextureID");
306
307        /*
308         * Create the SurfaceTexture that will feed this textureID, and pass it to the camera
309         */
310
311        mSurface = new SurfaceTexture(mTextureID);
312        mSurface.setOnFrameAvailableListener(this);
313        try {
314            mCamera.setPreviewTexture(mSurface);
315        } catch (IOException t) {
316            Log.e(TAG, "Cannot set preview texture target!");
317        }
318
319        /* Start the camera */
320        mCamera.startPreview();
321
322        Matrix.setLookAtM(mVMatrix, 0, 0, 0, 5f, 0f, 0f, 0f, 0f, 1.0f, 0.0f);
323
324        mLastTime = 0;
325
326        synchronized(this) {
327            updateSurface = false;
328        }
329    }
330
331    synchronized public void onFrameAvailable(SurfaceTexture surface) {
332        /* For simplicity, SurfaceTexture calls here when it has new
333         * data available.  Call may come in from some random thread,
334         * so let's be safe and use synchronize. No OpenGL calls can be done here.
335         */
336        updateSurface = true;
337    }
338
339    private void doPhysics(long timestamp) {
340        /*
341         * Move the camera surface around based on some simple spring physics with drag
342         */
343
344        if (mLastTime == 0)
345            mLastTime = timestamp;
346
347        float deltaT = (timestamp - mLastTime)/1000000000.f; // To seconds
348
349        float springStrength = 20.f;
350        float frictionCoeff = 10.f;
351        float mass = 10.f;
352        float gMultiplier = 4.f;
353        /* Only update physics every 30 ms */
354        if (deltaT > 0.030f) {
355            mLastTime = timestamp;
356
357            float[] totalForce = new float[3];
358            totalForce[0] = -mPos[0] * springStrength - mVel[0]*frictionCoeff + gMultiplier*mGForce[0]*mass;
359            totalForce[1] = -mPos[1] * springStrength - mVel[1]*frictionCoeff + gMultiplier*mGForce[1]*mass;
360            totalForce[2] = -mPos[2] * springStrength - mVel[2]*frictionCoeff + gMultiplier*mGForce[2]*mass;
361
362            float[] accel = new float[3];
363            accel[0] = totalForce[0]/mass;
364            accel[1] = totalForce[1]/mass;
365            accel[2] = totalForce[2]/mass;
366
367            /* Not a very accurate integrator */
368            mVel[0] = mVel[0] + accel[0]*deltaT;
369            mVel[1] = mVel[1] + accel[1]*deltaT;
370            mVel[2] = mVel[2] + accel[2]*deltaT;
371
372            mPos[0] = mPos[0] + mVel[0]*deltaT;
373            mPos[1] = mPos[1] + mVel[1]*deltaT;
374            mPos[2] = mPos[2] + mVel[2]*deltaT;
375
376            Matrix.setIdentityM(mMMatrix, 0);
377            Matrix.translateM(mMMatrix, 0, mPos[0], mPos[1], mPos[2]);
378        }
379
380    }
381
382    private int loadShader(int shaderType, String source) {
383        int shader = GLES20.glCreateShader(shaderType);
384        if (shader != 0) {
385            GLES20.glShaderSource(shader, source);
386            GLES20.glCompileShader(shader);
387            int[] compiled = new int[1];
388            GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
389            if (compiled[0] == 0) {
390                Log.e(TAG, "Could not compile shader " + shaderType + ":");
391                Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
392                GLES20.glDeleteShader(shader);
393                shader = 0;
394            }
395        }
396        return shader;
397    }
398
399    private int createProgram(String vertexSource, String fragmentSource) {
400        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
401        if (vertexShader == 0) {
402            return 0;
403        }
404        int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
405        if (pixelShader == 0) {
406            return 0;
407        }
408
409        int program = GLES20.glCreateProgram();
410        if (program != 0) {
411            GLES20.glAttachShader(program, vertexShader);
412            checkGlError("glAttachShader");
413            GLES20.glAttachShader(program, pixelShader);
414            checkGlError("glAttachShader");
415            GLES20.glLinkProgram(program);
416            int[] linkStatus = new int[1];
417            GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
418            if (linkStatus[0] != GLES20.GL_TRUE) {
419                Log.e(TAG, "Could not link program: ");
420                Log.e(TAG, GLES20.glGetProgramInfoLog(program));
421                GLES20.glDeleteProgram(program);
422                program = 0;
423            }
424        }
425        return program;
426    }
427
428    private void checkGlError(String op) {
429        int error;
430        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
431            Log.e(TAG, op + ": glError " + error);
432            throw new RuntimeException(op + ": glError " + error);
433        }
434    }
435
436    private static final int FLOAT_SIZE_BYTES = 4;
437    private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
438    private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
439    private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
440    private final float[] mTriangleVerticesData = {
441        // X, Y, Z, U, V
442        -1.0f, -1.0f, 0, 0.f, 0.f,
443        1.0f, -1.0f, 0, 1.f, 0.f,
444        -1.0f,  1.0f, 0, 0.f, 1.f,
445        1.0f,   1.0f, 0, 1.f, 1.f,
446    };
447
448    private FloatBuffer mTriangleVertices;
449
450    private final String mVertexShader =
451        "uniform mat4 uMVPMatrix;\n" +
452        "uniform mat4 uSTMatrix;\n" +
453        "uniform float uCRatio;\n" +
454        "attribute vec4 aPosition;\n" +
455        "attribute vec4 aTextureCoord;\n" +
456        "varying vec2 vTextureCoord;\n" +
457        "varying vec2 vTextureNormCoord;\n" +
458        "void main() {\n" +
459        "  vec4 scaledPos = aPosition;\n" +
460        "  scaledPos.x = scaledPos.x * uCRatio;\n" +
461        "  gl_Position = uMVPMatrix * scaledPos;\n" +
462        "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
463        "  vTextureNormCoord = aTextureCoord.xy;\n" +
464        "}\n";
465
466    private final String mFragmentShader =
467        "#extension GL_OES_EGL_image_external : require\n" +
468        "precision mediump float;\n" +
469        "varying vec2 vTextureCoord;\n" +
470        "varying vec2 vTextureNormCoord;\n" +
471        "uniform samplerExternalOES sTexture;\n" +
472        "void main() {\n" +
473        "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
474        "  gl_FragColor.a = 1.0-min(length(vTextureNormCoord-0.5)*2.0,1.0);\n" +
475        "}\n";
476
477    private float[] mMVPMatrix = new float[16];
478    private float[] mProjMatrix = new float[16];
479    private float[] mMMatrix = new float[16];
480    private float[] mVMatrix = new float[16];
481    private float[] mSTMatrix = new float[16];
482
483    private int mProgram;
484    private int mTextureID;
485    private int muMVPMatrixHandle;
486    private int muSTMatrixHandle;
487    private int muCRatioHandle;
488    private int maPositionHandle;
489    private int maTextureHandle;
490
491    private float mRatio = 1.0f;
492    private float mCameraRatio = 1.0f;
493    private float[] mVel = new float[3];
494    private float[] mPos = new float[3];
495    private float[] mGForce = new float[3];
496
497    private long mLastTime;
498
499    private SurfaceTexture mSurface;
500    private Camera mCamera;
501    private boolean updateSurface = false;
502
503    private Context mContext;
504    private static String TAG = "CamRenderer";
505
506    // Magic key
507    private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
508}