PageRenderTime 39ms CodeModel.GetById 16ms RepoModel.GetById 1ms app.codeStats 0ms

/src/org/thoughtcrime/redphone/audio/CallAudioProvider.java

https://github.com/blrhc/RedPhone
Java | 273 lines | 195 code | 39 blank | 39 comment | 49 complexity | f9907c262ec943cfefa4788d252b084b MD5 | raw file
Possible License(s): GPL-3.0
  1. /*
  2. * Copyright (C) 2011 Whisper Systems
  3. *
  4. * This program is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation, either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. */
  17. package org.thoughtcrime.redphone.audio;
  18. import android.util.Log;
  19. import org.thoughtcrime.redphone.codec.AudioCodec;
  20. import org.thoughtcrime.redphone.monitor.CallMonitor;
  21. import org.thoughtcrime.redphone.profiling.PacketLogger;
  22. import org.thoughtcrime.redphone.profiling.StatisticsWatcher;
  23. import java.util.TreeMap;
  24. /**
  25. * The CallAudioProvider stretches and shrinks audio on the fly to mask issues like
  26. * packet loss or audio clock mismatches.
  27. *
  28. * Encoded audio data from the network stack is provided to the CallAudioProvider, which
  29. * stores it until it is ready to be played.
  30. *
  31. * The audio player requests raw audio data which is returned in variable sized chunks.
  32. * If too much audio is available the audio is sped up or dropped
  33. * If too little audio is available the audio is slowed down or synthesized based on the
  34. * last few packets decoded.
  35. *
  36. * @author Stuart O. Anderson
  37. */
  38. public class CallAudioProvider {
  39. private static final String TAG = "CallAudioProvider";
  40. private static final int RATE_NORMAL = 0;
  41. private static final int RATE_BIG = 1;
  42. private static final int RATE_LITTLE = 2;
  43. private static final float bigRateShift = .5f;
  44. private static final float littleRateShift = .05f;
  45. private static final int bigStart = 10; //start a quick correction if we exceed the average delay by this amount
  46. private static final float littleStartShrink = 3f;
  47. private static final float littleStartStretch = 2f;
  48. private static final int maxGap = 1;
  49. private static final int maxBuffer = 25;
  50. private PacketLogger packetLogger;
  51. private int shiftMode = RATE_NORMAL;
  52. private float playRate = 1.0f;
  53. private long streamPlayheadPosition;
  54. private long lastGoodFrame;
  55. private CallLogger callAudioLogger;
  56. private int decodeBufferLength;
  57. private int outputFrameLength;
  58. private final short decodeBuffer[] = new short[1024];
  59. private final short rateBuffer[] = new short[2048];
  60. private StatisticsWatcher frameDelayStats = new StatisticsWatcher();
  61. private StatisticsWatcher samplesPerPacketStats = new StatisticsWatcher();
  62. private StatisticsWatcher frameSizeStats = new StatisticsWatcher();
  63. private AudioCodec codec;
  64. private TreeMap<Long, EncodedAudioData> audioFrames = new TreeMap<Long, EncodedAudioData>();
  65. private DesiredCallAudioDelayChooser delayChooser;
  66. private int gapLength;
  67. private int decodedCount;
  68. CallAudioProvider(AudioCodec _codec, PacketLogger packetLogger, CallLogger callLogger, CallMonitor monitor) {
  69. delayChooser = new DesiredCallAudioDelayChooser( packetLogger );
  70. codec = _codec;
  71. this.packetLogger = packetLogger;
  72. this.callAudioLogger = callLogger;
  73. frameDelayStats.setW(1 / 20.0f);
  74. frameSizeStats.setW(1 / 20f);
  75. samplesPerPacketStats.setW(1 / 20f);
  76. monitor.addSampledMetrics("cap-latency", frameDelayStats.getSampler());
  77. monitor.addSampledMetrics("cap-samples-per-packet", samplesPerPacketStats.getSampler());
  78. monitor.addSampledMetrics("cap-frame-size", frameSizeStats.getSampler());
  79. }
  80. private void pullAudio() {
  81. EncodedAudioData ead = null;
  82. EncodedAudioData eadAtHead = null;
  83. if( audioFrames.size() != 0 ) {
  84. //see if the next sample is the one we want
  85. ead = audioFrames.get(audioFrames.firstKey());
  86. eadAtHead = audioFrames.get(streamPlayheadPosition);
  87. }
  88. if( ead != null && ead.sequenceNumber < streamPlayheadPosition - maxGap ) {
  89. //if we've played more than 'maxGap' past this first frame, reset the playhead to that frame
  90. long totalLate = streamPlayheadPosition - ead.sequenceNumber;
  91. delayChooser.notifyVeryLate( totalLate );
  92. streamPlayheadPosition = ead.sequenceNumber;
  93. packetLogger.logPacket(ead.sequenceNumber, PacketLogger.PLAYHEAD_JUMP_BACK );
  94. Log.d( "CAP", "Very Late Event" );
  95. }
  96. else if( (ead == null || ead.sequenceNumber < streamPlayheadPosition) && eadAtHead != null ) {
  97. //we've got the packet for the current playhead position, and it's not a big gap, so drop the late frames
  98. ead = eadAtHead;
  99. streamPlayheadPosition = ead.sequenceNumber;
  100. packetLogger.logPacket(ead.sequenceNumber, PacketLogger.PLAYHEAD_JUMP_FORWARD );
  101. //Log.d( "CAP", "Drop Frames to " + eadAtHead );
  102. }
  103. if( ead != null && ead.sequenceNumber == streamPlayheadPosition ) {
  104. decodeBufferLength = codec.decode( ead.data, decodeBuffer, ead.data.length );
  105. decodedCount++;
  106. packetLogger.logPacket( ead.sourceSequenceNumber, PacketLogger.PACKET_DECODED );
  107. if( gapLength < CallLogger.gapLengthCounts.length &&
  108. gapLength > 0 ) {
  109. CallLogger.gapLengthCounts[gapLength]++;
  110. }
  111. gapLength = 0;
  112. lastGoodFrame = ead.sequenceNumber;
  113. audioFrames.remove(ead.sequenceNumber);
  114. if( audioFrames.size() == 0 ) delayChooser.notifyJustInTime();
  115. return;
  116. }
  117. if( ead != null ) {
  118. //Log.d( "CAP", "PLC - PH: " + streamPlayheadPosition + " ead " + ead.sequenceNumber + " ead2 " + ead.sourceSequenceNumber );
  119. packetLogger.logPacket(streamPlayheadPosition, PacketLogger.FILLING_GAP);
  120. } else {
  121. //Log.d( "CAP", "PLCNULL" );
  122. packetLogger.logPacket(streamPlayheadPosition, PacketLogger.PLAY_BUFFER_EMPTY );
  123. }
  124. decodeBufferLength = codec.decode(null, decodeBuffer, 0 );
  125. if( gapLength % 2 == 0 ) streamPlayheadPosition--; //FIXME: Assumes 2 frames per packet here...
  126. delayChooser.notifyMissing();
  127. gapLength++;
  128. if( decodeBufferLength == 0 ) {
  129. Log.e( TAG, "zero length decode buffer returned" );
  130. decodeBufferLength = 160; //v.bad, just feed it _something_
  131. }
  132. }
  133. public void addFrame( EncodedAudioData ead ) {
  134. //Log.d( "CAP", "added: " + ead.sequenceNumber );
  135. audioFrames.put( ead.sequenceNumber, ead );
  136. delayChooser.notifyArrival(ead.sequenceNumber);
  137. }
  138. private void updatePlayRate() {
  139. long frameDelay = lastGoodFrame - streamPlayheadPosition;
  140. if( audioFrames.size() > 0 ) {
  141. frameDelay = audioFrames.lastKey() - streamPlayheadPosition;
  142. }
  143. frameDelayStats.observeValue((int)frameDelay);
  144. float desFrameDelay = delayChooser.getDesFrameDelay();
  145. if( frameDelay > desFrameDelay + bigStart &&
  146. shiftMode != RATE_BIG ) {
  147. playRate = 1-bigRateShift;
  148. shiftMode = RATE_BIG;
  149. Log.d( TAG,"Dumping Glut");
  150. }
  151. if( frameDelay <= desFrameDelay &&
  152. shiftMode == RATE_BIG ) {
  153. playRate = 1;
  154. shiftMode = RATE_NORMAL;
  155. frameDelayStats.setAvg(desFrameDelay);
  156. Log.d( TAG,"Normal Rate" );
  157. }
  158. if( frameDelayStats.getAvg() > desFrameDelay + littleStartShrink &&
  159. shiftMode == RATE_NORMAL ) {
  160. shiftMode = RATE_LITTLE;
  161. playRate = 1-littleRateShift;
  162. Log.d( TAG, "Small shrink" );
  163. }
  164. if( frameDelayStats.getAvg() < desFrameDelay &&
  165. shiftMode == RATE_LITTLE && playRate < 1 ) {
  166. playRate = 1;
  167. shiftMode = RATE_NORMAL;
  168. Log.d(TAG, "Small shrink complete" );
  169. }
  170. if( frameDelayStats.getAvg() < desFrameDelay - littleStartStretch &&
  171. shiftMode == RATE_NORMAL ) {
  172. shiftMode = RATE_LITTLE;
  173. playRate = 1+littleRateShift;
  174. Log.d(TAG, "Small stretch" );
  175. }
  176. if (frameDelayStats.getAvg() > desFrameDelay
  177. && shiftMode == RATE_LITTLE && playRate > 1) {
  178. shiftMode = RATE_NORMAL;
  179. playRate = 1;
  180. Log.d(TAG, "Small stretch complete" );
  181. }
  182. }
  183. private void discardStaleFrames() {
  184. //discard frames that happened before the last data-frame we played
  185. int sizeBeforeDiscard = audioFrames.size();
  186. audioFrames.headMap(lastGoodFrame).clear();
  187. while( audioFrames.size() > maxBuffer ) {
  188. audioFrames.remove( audioFrames.firstKey());
  189. streamPlayheadPosition = audioFrames.firstKey();
  190. }
  191. int sizeAfterDiscard = audioFrames.size();
  192. if( sizeAfterDiscard != sizeBeforeDiscard ) {
  193. Log.d( "CAP", "Discard: " + (sizeBeforeDiscard - sizeAfterDiscard ) );
  194. }
  195. delayChooser.notifyLate( sizeBeforeDiscard - sizeAfterDiscard );
  196. }
  197. private void setDebugInfo() {
  198. CallLogger.waitingFrames = audioFrames.size();
  199. CallLogger.streamPlayheadPosition = streamPlayheadPosition;
  200. CallLogger.avgDelay = frameDelayStats.getAvg();
  201. CallLogger.shiftMode = shiftMode;
  202. if( audioFrames.size() > 0 )
  203. CallLogger.largestHeldFrame = audioFrames.lastKey();
  204. }
  205. public short[] getFrame() {
  206. discardStaleFrames();
  207. setDebugInfo();
  208. pullAudio();
  209. samplesPerPacketStats.observeValue(decodeBufferLength);
  210. updatePlayRate();
  211. //model prediction frame delay offset ... is this really a good idea - confirm that it improves our estimates
  212. frameDelayStats.setAvg(frameDelayStats.getAvg() + playRate-1 );//include our actions in the buffer model
  213. if( lastGoodFrame == streamPlayheadPosition ) {
  214. outputFrameLength = PacketLossConcealer.changeSpeed(rateBuffer, decodeBuffer, decodeBufferLength, playRate);
  215. } else {
  216. outputFrameLength = PacketLossConcealer.changeSpeed(rateBuffer, decodeBuffer, decodeBufferLength, 1 );
  217. }
  218. frameSizeStats.observeValue(outputFrameLength);
  219. streamPlayheadPosition++;
  220. packetLogger.logPacket(streamPlayheadPosition, PacketLogger.PLAYHEAD);
  221. delayChooser.updateDesired();
  222. callAudioLogger.update();
  223. if( decodedCount % 500 == 1 ) {
  224. Log.d( "CAP", "Decoded: " + decodedCount );
  225. }
  226. return rateBuffer;
  227. }
  228. public int getFrameSize() {
  229. return outputFrameLength;
  230. }
  231. public void terminate() {
  232. delayChooser.terminate();
  233. }
  234. }