PageRenderTime 55ms CodeModel.GetById 17ms RepoModel.GetById 0ms app.codeStats 1ms

/Mind-body-Interaction/Main.hx

http://github.com/andyli/AugmentedMirror
Haxe | 608 lines | 488 code | 98 blank | 22 comment | 16 complexity | 5e76a3a3268b29bb84c799a0975ad36d MD5 | raw file
  1. package;
  2. import haxe.FastList;
  3. import flash.display.Bitmap;
  4. import flash.display.BitmapData;
  5. import flash.display.Graphics;
  6. import flash.display.GradientType;
  7. import flash.display.SpreadMethod;
  8. import flash.display.Sprite;
  9. import flash.display.StageAlign;
  10. import flash.display.StageScaleMode;
  11. import flash.events.Event;
  12. import flash.events.MouseEvent;
  13. import flash.events.KeyboardEvent;
  14. import flash.geom.ColorTransform;
  15. import flash.geom.Point;
  16. import flash.geom.Rectangle;
  17. import flash.geom.Matrix;
  18. import flash.geom.Vector3D;
  19. import flash.ui.Keyboard;
  20. import flash.utils.ByteArray;
  21. import flash.Vector;
  22. #if flash
  23. import flash.display.BlendMode;
  24. import flash.display.Shader;
  25. import flash.display.ShaderJob;
  26. import flash.filters.ColorMatrixFilter;
  27. #end
  28. import sandy.HaxeTypes;
  29. import sandy.core.Scene3D;
  30. import sandy.core.data.Point3D;
  31. import sandy.core.data.Matrix4;
  32. import sandy.core.data.Vertex;
  33. import sandy.core.scenegraph.ATransformable;
  34. import sandy.core.scenegraph.Camera3D;
  35. import sandy.core.scenegraph.Group;
  36. import sandy.core.scenegraph.TransformGroup;
  37. import sandy.core.scenegraph.Sprite2D;
  38. import sandy.materials.Appearance;
  39. import sandy.materials.BitmapMaterial;
  40. import sandy.materials.Material;
  41. import sandy.materials.ColorMaterial;
  42. import sandy.materials.WireFrameMaterial;
  43. import sandy.materials.attributes.LightAttributes;
  44. import sandy.materials.attributes.LineAttributes;
  45. import sandy.materials.attributes.MaterialAttributes;
  46. import sandy.primitive.Box;
  47. import sandy.primitive.Line3D;
  48. import sandy.primitive.Plane3D;
  49. import sandy.primitive.Cone;
  50. import sandy.primitive.Sphere;
  51. import org.casalib.math.Percent;
  52. import org.casalib.math.geom.Point3d;
  53. import org.casalib.ui.Key;
  54. import org.casalib.util.StageReference;
  55. import org.casalib.util.NumberUtil;
  56. using org.casalib.util.NumberUtil;
  57. using hxSteeringBehaviors.Float3DTools;
  58. import hxSteeringBehaviors.Vehicle;
  59. import hxSteeringBehaviors.VehicleGroup;
  60. import hxSteeringBehaviors.behavior.Arrival;
  61. import hxSteeringBehaviors.behavior.Seek;
  62. import hxSteeringBehaviors.behavior.Separation;
  63. import hxSteeringBehaviors.behavior.UnalignedCollisionAvoidance;
  64. import hxSteeringBehaviors.behavior.Wander;
  65. import hxSteeringBehaviors.behavior.combined.Flocking;
  66. import hxSteeringBehaviors.behavior.combined.LeaderFollowing;
  67. class Main extends Sprite {
  68. inline static public var firstRenderSize:Int = 800;
  69. inline static public var unwrapAnd3dFile:ByteArray = haxe.Resource.getBytes("unwrapAnd3d.pbj").getData();
  70. inline static public var unwrapFile:ByteArray = haxe.Resource.getBytes("unwrap.pbj").getData();
  71. public var scene:Scene3D;
  72. public var camera:Camera3D;
  73. public var camPos0:Cone;
  74. public var camPos1:Cone;
  75. public var camTT:TransformGroup;
  76. public var camT:TransformGroup;
  77. public var sandyRoot:Group;
  78. public var rootT:TransformGroup;
  79. public var realWorld:Plane3D;
  80. public var realWorldModelMatrix:Matrix4;
  81. public var mirror:Plane3D;
  82. public var mirrorTexture0:BitmapData;
  83. public var mirrorTexture1:BitmapData;
  84. public var mirrorTexture_blank:ByteArray;
  85. public var fxGroup:Group;
  86. public var vehicleGroup:VehicleGroup;
  87. public var fxBigLights:FastList<Vehicle>;
  88. public var fxSmallLights:FastList<Vehicle>;
  89. public var realWorldPassiveBoids:FastList<Vehicle>;
  90. public var headPos:Vector3D;
  91. public var cameraUpdater:CameraUpdater;
  92. public var renderBD:BitmapData;
  93. public var renderLayer:Sprite;
  94. inline static public var ct1:ColorTransform = new ColorTransform(0, 1 - 0.125, 1 - 0.05);
  95. inline static public var ct2:ColorTransform = new ColorTransform(1, 0, 0);
  96. public var emotionStates:EmotionStates;
  97. public var infoLayer:Sprite;
  98. public var emotionStatesVis:EmotionStatesVis;
  99. public var headDetectionViz:HeadDetectionViz;
  100. public var manualCam:Bool;
  101. public var showRealWorld:Bool;
  102. public var singleEye:Bool;
  103. public var traceBG:Sprite;
  104. public var key:Key;
  105. public function new(){
  106. super();
  107. infoLayer = new Sprite();
  108. singleEye = false;
  109. var stage = flash.Lib.current.stage;
  110. StageReference.setStage(stage);
  111. stage.scaleMode = StageScaleMode.NO_SCALE;
  112. stage.align = StageAlign.TOP_LEFT;
  113. camera = new Camera3D(firstRenderSize,firstRenderSize,45,0,100000000);
  114. camera.z = -300;
  115. camera.lookAt(0, 0, 0);
  116. initRoot();
  117. scene = new Scene3D("scene", new Sprite(), camera, sandyRoot);
  118. scene.render();
  119. realWorldModelMatrix = realWorld.modelMatrix;
  120. sandyRoot.removeChild(rootT);
  121. showRealWorld = false;
  122. headPos = new Vector3D();
  123. initFxGroup();
  124. renderBD = new BitmapData(768,1024, false, 0);
  125. addEventListener(Event.ENTER_FRAME, enterFrameHandler);
  126. stage.addEventListener(MouseEvent.MOUSE_MOVE, onMouseMove);
  127. key = Key.getInstance();
  128. key.addEventListener(KeyboardEvent.KEY_DOWN, onKeyDown);
  129. #if (cpp && HXCPP_MULTI_THREADED)
  130. cpp.Sys.setCwd("helper");
  131. ThreadedCmd.run("./openHelpers.sh");
  132. cpp.Sys.sleep(1);
  133. cpp.Sys.setCwd("..");
  134. #end
  135. emotionStates = new EmotionStates();
  136. new PlaneUpdater(realWorld,"192.168.1.100:11999").start();
  137. cameraUpdater = new CameraUpdater(camPos0,camPos1,"192.168.1.100:11998");
  138. cameraUpdater.start();
  139. new EmotionUpdater(emotionStates,"192.168.1.101:11995").start();
  140. renderLayer = new Sprite();
  141. renderLayer.addChild(new Bitmap(renderBD));
  142. this.addChild(renderLayer);
  143. traceBG = new Sprite();
  144. traceBG.graphics.beginFill(0xFFFFFF,1);
  145. traceBG.graphics.drawRect(0,0,firstRenderSize,firstRenderSize);
  146. traceBG.graphics.endFill();
  147. traceBG.visible = false;
  148. this.addChild(traceBG);
  149. manualCam = false;
  150. /*
  151. info display
  152. */
  153. this.addChild(infoLayer);
  154. emotionStatesVis = new EmotionStatesVis(emotionStates);
  155. emotionStatesVis.x = 80;
  156. emotionStatesVis.y = 900 - emotionStatesVis.drawArea.height;
  157. infoLayer.addChild(emotionStatesVis);
  158. headDetectionViz = new HeadDetectionViz(cameraUpdater);
  159. headDetectionViz.x = 80 + emotionStatesVis.drawArea.width + 20;
  160. headDetectionViz.y = 900 - emotionStatesVis.drawArea.height;
  161. infoLayer.addChild(headDetectionViz);
  162. }
  163. private function initRoot():Void {
  164. sandyRoot = new Group();
  165. var material:Material;
  166. var materialAttr:MaterialAttributes;
  167. var appearance:Appearance;
  168. realWorld = new Plane3D("plane", 240, 320, 32-1, 24-1);
  169. realWorld.enableBackFaceCulling = false;
  170. realWorld.container.filters = [new ColorMatrixFilter([0.5,0,0,0,0, 0,0.5,0,0,0 ,0,0,0.5,0,0, 0,0,0,1,0])];
  171. camPos0 = new Cone("camPos0",2,4);
  172. camPos1 = new Cone("camPos1",2,4);
  173. camPos0.visible = camPos1.visible = false;
  174. rootT = new TransformGroup();
  175. rootT.addChild(realWorld);
  176. rootT.x = 10;
  177. rootT.y = 105;
  178. rootT.z = -20;
  179. rootT.rotateX = 65;
  180. rootT.rotateY = 0;
  181. rootT.rotateZ = 0;
  182. rootT.scaleX = 1.9;
  183. rootT.scaleY = 1.9;
  184. rootT.scaleZ = -3.5;
  185. sandyRoot.addChild(rootT);
  186. camT = new TransformGroup();
  187. camT.addChild(camPos0);
  188. camT.addChild(camPos1);
  189. camT.x = 0;
  190. camT.y = 60;
  191. camT.z = 15;
  192. camT.rotateX = 70;
  193. camT.rotateY = 10;
  194. camT.rotateZ = 5;
  195. camT.scaleX = 1.2;
  196. camT.scaleY = 1;
  197. camT.scaleZ = -1.3;
  198. camTT = new TransformGroup();
  199. camTT.addChild(camT);
  200. camTT.scaleZ = -1;
  201. sandyRoot.addChild(camTT);
  202. mirror = new Plane3D("mirror",98,78);
  203. mirror.x = 5;
  204. mirror.y = -20;
  205. mirrorTexture0 = new BitmapData(firstRenderSize,firstRenderSize, false, 0);
  206. mirrorTexture1 = mirrorTexture0.clone();
  207. mirrorTexture_blank = mirrorTexture0.getPixels(mirrorTexture0.rect);
  208. sandyRoot.addChild(mirror);
  209. }
  210. private function initFxGroup():Void {
  211. fxGroup = new Group();
  212. sandyRoot.addChild(fxGroup);
  213. //infoLayer.addChild(sp);
  214. var flocking = new Flocking();
  215. var leaderFollowing = new LeaderFollowing();
  216. var separation = new Separation();
  217. vehicleGroup = new VehicleGroup([]);
  218. fxBigLights = new FastList<Vehicle>();
  219. fxSmallLights = new FastList<Vehicle>();
  220. for (i in 0...10) {
  221. //var s = new Sphere(null, 1, 2 , 2);
  222. var sp = new Sprite();
  223. //sp.graphics.beginFill(0,1);
  224. var matr = new Matrix();
  225. matr.createGradientBox(10, 10, 0, 5, 5);
  226. sp.graphics.beginGradientFill(GradientType.RADIAL, [0xFFFFFF,0x999999,0x1111111,0x000000], [1,1,1,1], [0,30,230,255], matr, SpreadMethod.PAD);
  227. sp.graphics.drawCircle(10,10,10);
  228. sp.graphics.endFill();
  229. var s = new Sprite2D(sp);
  230. s.setPosition(0, 0, 100);
  231. //s.container.blendMode = BlendMode.SCREEN;
  232. fxGroup.addChild(s);
  233. var veh:Boid = new Boid(s);
  234. veh.vehicleRadius = 5;
  235. veh.maxSpeed = 5;
  236. veh.maxForce = 2;
  237. veh.position.setTo(0,0,0);
  238. veh.edgeBehavior = Vehicle.EDGE_BOUNCE;
  239. veh.boundsCentre.setTo(0,0,50);
  240. veh.boundsRadius = 50;
  241. veh.velocity.setUnitRandom();
  242. veh.velocity.scaleBy(3);
  243. veh.behaviorList = [leaderFollowing, flocking];
  244. fxBigLights.add(veh);
  245. /*
  246. small lights
  247. */
  248. var sveh:Boid = null;
  249. for (s in 0...2){
  250. var sp = new Sprite();
  251. //sp.graphics.beginFill(0,1);
  252. var matr = new Matrix();
  253. matr.createGradientBox(4, 4, 0, 2, 2);
  254. sp.graphics.beginGradientFill(GradientType.RADIAL, [0x555555,0x333333,0x050505,0x000000], [1,1,1,1], [0,30,230,255], matr, SpreadMethod.PAD);
  255. sp.graphics.drawCircle(4,4,4);
  256. sp.graphics.endFill();
  257. var s = new Sprite2D(sp);
  258. s.setPosition(0, 0, 100);
  259. //s.container.blendMode = BlendMode.SCREEN;
  260. fxGroup.addChild(s);
  261. var leader = sveh == null ? veh : sveh;
  262. sveh = new Boid(s);
  263. sveh.vehicleRadius = 0.5;
  264. sveh.maxSpeed = 6;
  265. sveh.maxForce = 1.5;
  266. sveh.position.setTo(0,0,0);
  267. sveh.edgeBehavior = Vehicle.EDGE_BOUNCE;
  268. sveh.boundsCentre.setTo(0,0,50);
  269. sveh.boundsRadius = 50;
  270. sveh.velocity.setUnitRandom();
  271. sveh.velocity.scaleBy(3);
  272. var list = new FastList<Vehicle>();
  273. list.add(sveh);
  274. var sleaderFollowing = new Seek(leader.position);
  275. sveh.behaviorList = cast [sleaderFollowing];
  276. fxSmallLights.add(sveh);
  277. }
  278. }
  279. vehicleGroup.addVehicles(fxBigLights);
  280. vehicleGroup.addVehicles(fxSmallLights);
  281. flocking.separate.separateList = flocking.align.alignList = flocking.cohere.cohereList = fxBigLights;
  282. flocking.cohere.cohereDist = 20;
  283. flocking.cohere.cohereStrength = 5;
  284. flocking.align.alignDist = 30;
  285. flocking.align.alignStrength = 10;
  286. flocking.separate.separateDist = 5;
  287. flocking.separate.separateStrength = 20;
  288. var leader = new PassiveBoid(headPos);
  289. vehicleGroup.addVehicle(leader);
  290. leaderFollowing.vehicleList = fxBigLights;
  291. leaderFollowing.vehicleLeader = leader;
  292. realWorldPassiveBoids = new FastList<Vehicle>();
  293. for (v in realWorld.geometry.aVertex) {
  294. var pb = new PassiveBoid(v,realWorldModelMatrix);
  295. realWorldPassiveBoids.add(pb);
  296. vehicleGroup.addVehicle(pb);
  297. }
  298. separation.separateList = realWorldPassiveBoids;
  299. separation.separateStrength = 15;
  300. separation.separateDist = 10;
  301. }
  302. private function renderOneEye(x:Float,y:Float,z:Float):Void {
  303. camera.setPosition(x,y,z);
  304. camera.fov = 120;
  305. mirror.visible = true;
  306. realWorld.visible = false;
  307. fxGroup.visible = false;
  308. scene.render();
  309. var camTaget = camera.getPosition(ABSOLUTE);
  310. camTaget.add(Point3D.UNIT_Z);
  311. camera.lookAtPoint(camTaget);
  312. mirror.visible = false;
  313. realWorld.visible = true;
  314. fxGroup.visible = true;
  315. scene.render();
  316. }
  317. private function enterFrameHandler(evt:Event):Void {
  318. var p0 = camPos0.getPosition(ABSOLUTE);
  319. var p1 = camPos1.getPosition(ABSOLUTE);
  320. var _p0 = new Point3d(p0.x,p0.y,p0.z);
  321. var _p1 = new Point3d(p1.x,p1.y,p1.z);
  322. headPos.setToVector(Point3d.interpolate(_p0,_p1,new Percent(0.4)));
  323. headPos.z = Math.abs(headPos.z);
  324. for (big in fxBigLights){
  325. var veh:Boid = cast big;
  326. veh.maxSpeed = 5 * emotionStates.excitementLongTerm;
  327. veh.maxForce = 2 * (1-emotionStates.frustration);
  328. var s:Sprite2D = cast veh.shape3D;
  329. s.container.alpha = NumberUtil.randomWithinRange(0.2,0.5+emotionStates.meditation*0.5);
  330. }
  331. for (small in fxSmallLights){
  332. var veh:Boid = cast small;
  333. veh.maxSpeed = 6 * emotionStates.excitementShortTerm;
  334. veh.maxForce = 1.5 * (1-emotionStates.frustration);
  335. var s:Sprite2D = cast veh.shape3D;
  336. s.container.alpha = NumberUtil.randomWithinRange(0.2,0.5+emotionStates.engagementBoredom*0.5);
  337. }
  338. vehicleGroup.update();
  339. emotionStatesVis.draw();
  340. headDetectionViz.draw();
  341. if (manualCam){
  342. scene.render();
  343. return;
  344. }
  345. var pts = new Vector<Array<Float>>();
  346. //var halfEyeDist = Point3d.distance(_p0,_p1)*0.25; //3.2
  347. renderBD.lock();
  348. /*
  349. 1st eye
  350. */
  351. var ipt = Point3d.interpolate(_p0,_p1,new Percent(0.27));
  352. renderOneEye(ipt.x,ipt.y,ipt.z);
  353. for (i in 0...mirror.geometry.aVertex.length){
  354. var v = mirror.geometry.aVertex[i];
  355. pts.push([v.sx,v.sy]);
  356. }
  357. mirrorTexture_blank.position = 0;
  358. mirrorTexture0.setPixels(mirrorTexture0.rect,mirrorTexture_blank);
  359. mirrorTexture0.draw(scene.container);
  360. if (singleEye){
  361. var shader = new Shader(unwrapFile);
  362. shader.data.src.input = mirrorTexture0;
  363. shader.data.tl.value = pts[2];
  364. shader.data.tr.value = pts[3];
  365. shader.data.bl.value = pts[0];
  366. shader.data.br.value = pts[1];
  367. shader.data.size.value = [renderBD.width,renderBD.height];
  368. new ShaderJob(shader,renderBD).start(true);
  369. renderBD.unlock();
  370. return;
  371. }
  372. /*
  373. 2nd eye
  374. */
  375. ipt = Point3d.interpolate(_p0,_p1,new Percent(0.5));
  376. renderOneEye(ipt.x,ipt.y,ipt.z);
  377. for (i in 0...mirror.geometry.aVertex.length){
  378. var v = mirror.geometry.aVertex[i];
  379. pts.push([v.sx,v.sy]);
  380. }
  381. mirrorTexture_blank.position = 0;
  382. mirrorTexture1.setPixels(mirrorTexture1.rect,mirrorTexture_blank);
  383. mirrorTexture1.draw(scene.container);
  384. /*
  385. unwrap and make 3d
  386. */
  387. var shader = new Shader(unwrapAnd3dFile);
  388. shader.data.src0.input = mirrorTexture0;
  389. shader.data.tl0.value = pts[2];
  390. shader.data.tr0.value = pts[3];
  391. shader.data.bl0.value = pts[0];
  392. shader.data.br0.value = pts[1];
  393. shader.data.src1.input = mirrorTexture1;
  394. shader.data.tl1.value = pts[6];
  395. shader.data.tr1.value = pts[7];
  396. shader.data.bl1.value = pts[4];
  397. shader.data.br1.value = pts[5];
  398. shader.data.size.value = [renderBD.width,renderBD.height];
  399. new ShaderJob(shader,renderBD).start(true);
  400. renderBD.unlock();
  401. }
  402. private function onMouseMove(evt:MouseEvent):Void {
  403. if (manualCam){
  404. camera.x = evt.stageX.map(0,stage.stageWidth,-1000,1000);
  405. camera.y = evt.stageY.map(0,stage.stageHeight,1000,-1000);
  406. camera.z = -300;
  407. camera.lookAt(0, 0, 0);
  408. camera.fov = 60;
  409. }
  410. }
  411. private var transformGroup:ATransformable;
  412. private function onKeyDown(evt:KeyboardEvent):Void {
  413. switch(evt.keyCode) {
  414. case Keyboard.NUMPAD_0:
  415. traceBG.visible = !traceBG.visible;
  416. case Keyboard.NUMPAD_1:
  417. transformGroup = camT;
  418. case Keyboard.NUMPAD_2:
  419. transformGroup = rootT;
  420. case Keyboard.NUMPAD_3:
  421. transformGroup = mirror;
  422. case Keyboard.UP:
  423. camera.z += 10;
  424. case Keyboard.DOWN:
  425. camera.z -= 10;
  426. case Keyboard.SPACE:
  427. trace("x = "+transformGroup.x);
  428. trace("y = "+transformGroup.y);
  429. trace("z = "+transformGroup.z);
  430. trace("rotateX = "+transformGroup.rotateX);
  431. trace("rotateY = "+transformGroup.rotateY);
  432. trace("rotateZ = "+transformGroup.rotateZ);
  433. trace("scaleX = "+transformGroup.scaleX);
  434. trace("scaleY = "+transformGroup.scaleY);
  435. trace("scaleZ = "+transformGroup.scaleZ);
  436. case Keyboard.ENTER:
  437. mirror.visible = camPos0.visible = camPos1.visible = manualCam = !manualCam;
  438. if (manualCam){
  439. this.addChild(scene.container);
  440. camTT.scaleZ = 1;
  441. renderLayer.visible = false;
  442. } else {
  443. this.removeChild(scene.container);
  444. camTT.scaleZ = -1;
  445. renderLayer.visible = true;
  446. }
  447. case Keyboard.DELETE:
  448. showRealWorld = !showRealWorld;
  449. if (showRealWorld) {
  450. sandyRoot.addChild(rootT);
  451. } else {
  452. sandyRoot.removeChild(rootT);
  453. }
  454. case Keyboard.INSERT:
  455. singleEye = !singleEye;
  456. default:
  457. }
  458. switch(evt.charCode) {
  459. case "a".charCodeAt(0):
  460. transformGroup.x += 5;
  461. case "q".charCodeAt(0):
  462. transformGroup.x -= 5;
  463. case "w".charCodeAt(0):
  464. transformGroup.y += 5;
  465. case "s".charCodeAt(0):
  466. transformGroup.y -= 5;
  467. case "e".charCodeAt(0):
  468. transformGroup.z += 5;
  469. case "d".charCodeAt(0):
  470. transformGroup.z -= 5;
  471. case "r".charCodeAt(0):
  472. transformGroup.rotateX += 5;
  473. case "f".charCodeAt(0):
  474. transformGroup.rotateX -= 5;
  475. case "t".charCodeAt(0):
  476. transformGroup.rotateY += 5;
  477. case "g".charCodeAt(0):
  478. transformGroup.rotateY -= 5;
  479. case "y".charCodeAt(0):
  480. transformGroup.rotateZ += 5;
  481. case "h".charCodeAt(0):
  482. transformGroup.rotateZ -= 5;
  483. case "u".charCodeAt(0):
  484. transformGroup.scaleX += 0.05;
  485. case "j".charCodeAt(0):
  486. transformGroup.scaleX -= 0.05;
  487. case "i".charCodeAt(0):
  488. transformGroup.scaleY += 0.05;
  489. case "k".charCodeAt(0):
  490. transformGroup.scaleY -= 0.05;
  491. case "o".charCodeAt(0):
  492. transformGroup.scaleZ += 0.05;
  493. case "l".charCodeAt(0):
  494. transformGroup.scaleZ -= 0.05;
  495. default:
  496. }
  497. }
  498. public static function main():Void {
  499. #if cpp
  500. neash.Lib.Init("Mind-Body Interaction", 600, 800);
  501. #end
  502. flash.Lib.current.addChild(new Main());
  503. #if myDebug
  504. neash.Lib.ShowFPS(true);
  505. #end
  506. #if cpp
  507. neash.Lib.Run();
  508. #end
  509. }
  510. }