PageRenderTime 56ms CodeModel.GetById 29ms RepoModel.GetById 1ms app.codeStats 0ms

/src/org/thoughtcrime/redphone/audio/CallAudioProvider.java

https://github.com/RBWare/RedPhone
Java | 268 lines | 191 code | 38 blank | 39 comment | 49 complexity | 6570face6570bcdec4c5f698efb83bbd MD5 | raw file
Possible License(s): GPL-3.0
  1. /*
  2. * Copyright (C) 2011 Whisper Systems
  3. *
  4. * This program is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License as published by
  6. * the Free Software Foundation, either version 3 of the License, or
  7. * (at your option) any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. */
  17. package org.thoughtcrime.redphone.audio;
  18. import android.util.Log;
  19. import org.thoughtcrime.redphone.codec.AudioCodec;
  20. import org.thoughtcrime.redphone.profiling.PacketLogger;
  21. import org.thoughtcrime.redphone.profiling.StatisticsWatcher;
  22. import java.util.TreeMap;
  23. /**
  24. * The CallAudioProvider stretches and shrinks audio on the fly to mask issues like
  25. * packet loss or audio clock mismatches.
  26. *
  27. * Encoded audio data from the network stack is provided to the CallAudioProvider, which
  28. * stores it until it is ready to be played.
  29. *
  30. * The audio player requests raw audio data which is returned in variable sized chunks.
  31. * If too much audio is available the audio is sped up or dropped
  32. * If too little audio is available the audio is slowed down or synthesized based on the
  33. * last few packets decoded.
  34. *
  35. * @author Stuart O. Anderson
  36. */
  37. public class CallAudioProvider {
  38. private static final String TAG = "CallAudioProvider";
  39. private static final int RATE_NORMAL = 0;
  40. private static final int RATE_BIG = 1;
  41. private static final int RATE_LITTLE = 2;
  42. private static final float bigRateShift = .5f;
  43. private static final float littleRateShift = .05f;
  44. private static final int bigStart = 10; //start a quick correction if we exceed the average delay by this amount
  45. private static final float littleStartShrink = 3f;
  46. private static final float littleStartStretch = 2f;
  47. private static final int maxGap = 1;
  48. private static final int maxBuffer = 25;
  49. private PacketLogger packetLogger;
  50. private int shiftMode = RATE_NORMAL;
  51. private float playRate = 1.0f;
  52. private long streamPlayheadPosition;
  53. private long lastGoodFrame;
  54. private CallLogger callAudioLogger;
  55. private int decodeBufferLength;
  56. private int outputFrameLength;
  57. private final short decodeBuffer[] = new short[1024];
  58. private final short rateBuffer[] = new short[2048];
  59. private StatisticsWatcher averageFrameDelay = new StatisticsWatcher();
  60. private StatisticsWatcher avgSamplesPerPacket = new StatisticsWatcher();
  61. private StatisticsWatcher avgFrameObs = new StatisticsWatcher();
  62. private AudioCodec codec;
  63. private TreeMap<Long, EncodedAudioData> audioFrames = new TreeMap<Long, EncodedAudioData>();
  64. private DesiredCallAudioDelayChooser delayChooser;
  65. private int gapLength;
  66. private int decodedCount;;
  67. CallAudioProvider( AudioCodec _codec, PacketLogger packetLogger, CallLogger callLogger ) {
  68. delayChooser = new DesiredCallAudioDelayChooser( packetLogger );
  69. codec = _codec;
  70. this.packetLogger = packetLogger;
  71. this.callAudioLogger = callLogger;
  72. averageFrameDelay.setW( 1/20.0f);
  73. avgFrameObs.setW( 1/20f );
  74. avgSamplesPerPacket.setW( 1/20f );
  75. }
  76. private void pullAudio() {
  77. EncodedAudioData ead = null;
  78. EncodedAudioData eadAtHead = null;
  79. if( audioFrames.size() != 0 ) {
  80. //see if the next sample is the one we want
  81. ead = audioFrames.get(audioFrames.firstKey());
  82. eadAtHead = audioFrames.get(streamPlayheadPosition);
  83. }
  84. if( ead != null && ead.sequenceNumber < streamPlayheadPosition - maxGap ) {
  85. //if we've played more than 'maxGap' past this first frame, reset the playhead to that frame
  86. long totalLate = streamPlayheadPosition - ead.sequenceNumber;
  87. delayChooser.notifyVeryLate( totalLate );
  88. streamPlayheadPosition = ead.sequenceNumber;
  89. packetLogger.logPacket(ead.sequenceNumber, PacketLogger.PLAYHEAD_JUMP_BACK );
  90. Log.d( "CAP", "Very Late Event" );
  91. }
  92. else if( (ead == null || ead.sequenceNumber < streamPlayheadPosition) && eadAtHead != null ) {
  93. //we've got the packet for the current playhead position, and it's not a big gap, so drop the late frames
  94. ead = eadAtHead;
  95. streamPlayheadPosition = ead.sequenceNumber;
  96. packetLogger.logPacket(ead.sequenceNumber, PacketLogger.PLAYHEAD_JUMP_FORWARD );
  97. //Log.d( "CAP", "Drop Frames to " + eadAtHead );
  98. }
  99. if( ead != null && ead.sequenceNumber == streamPlayheadPosition ) {
  100. decodeBufferLength = codec.decode( ead.data, decodeBuffer, ead.data.length );
  101. decodedCount++;
  102. packetLogger.logPacket( ead.sourceSequenceNumber, PacketLogger.PACKET_DECODED );
  103. if( gapLength < CallLogger.gapLengthCounts.length &&
  104. gapLength > 0 ) {
  105. CallLogger.gapLengthCounts[gapLength]++;
  106. }
  107. gapLength = 0;
  108. lastGoodFrame = ead.sequenceNumber;
  109. audioFrames.remove(ead.sequenceNumber);
  110. if( audioFrames.size() == 0 ) delayChooser.notifyJustInTime();
  111. return;
  112. }
  113. if( ead != null ) {
  114. //Log.d( "CAP", "PLC - PH: " + streamPlayheadPosition + " ead " + ead.sequenceNumber + " ead2 " + ead.sourceSequenceNumber );
  115. packetLogger.logPacket(streamPlayheadPosition, PacketLogger.FILLING_GAP);
  116. } else {
  117. //Log.d( "CAP", "PLCNULL" );
  118. packetLogger.logPacket(streamPlayheadPosition, PacketLogger.PLAY_BUFFER_EMPTY );
  119. }
  120. decodeBufferLength = codec.decode(null, decodeBuffer, 0 );
  121. if( gapLength % 2 == 0 ) streamPlayheadPosition--; //FIXME: Assumes 2 frames per packet here...
  122. delayChooser.notifyMissing();
  123. gapLength++;
  124. if( decodeBufferLength == 0 ) {
  125. Log.e( TAG, "zero length decode buffer returned" );
  126. decodeBufferLength = 160; //v.bad, just feed it _something_
  127. }
  128. }
  129. public void addFrame( EncodedAudioData ead ) {
  130. //Log.d( "CAP", "added: " + ead.sequenceNumber );
  131. audioFrames.put( ead.sequenceNumber, ead );
  132. delayChooser.notifyArrival(ead.sequenceNumber);
  133. }
  134. private void updatePlayRate() {
  135. long frameDelay = lastGoodFrame - streamPlayheadPosition;
  136. if( audioFrames.size() > 0 ) {
  137. frameDelay = audioFrames.lastKey() - streamPlayheadPosition;
  138. }
  139. averageFrameDelay.observeValue((int)frameDelay);
  140. float desFrameDelay = delayChooser.getDesFrameDelay();
  141. if( frameDelay > desFrameDelay + bigStart &&
  142. shiftMode != RATE_BIG ) {
  143. playRate = 1-bigRateShift;
  144. shiftMode = RATE_BIG;
  145. Log.d( TAG,"Dumping Glut");
  146. }
  147. if( frameDelay <= desFrameDelay &&
  148. shiftMode == RATE_BIG ) {
  149. playRate = 1;
  150. shiftMode = RATE_NORMAL;
  151. averageFrameDelay.setAvg(desFrameDelay);
  152. Log.d( TAG,"Normal Rate" );
  153. }
  154. if( averageFrameDelay.getAvgBufferSize() > desFrameDelay + littleStartShrink &&
  155. shiftMode == RATE_NORMAL ) {
  156. shiftMode = RATE_LITTLE;
  157. playRate = 1-littleRateShift;
  158. Log.d( TAG, "Small shrink" );
  159. }
  160. if( averageFrameDelay.getAvgBufferSize() < desFrameDelay &&
  161. shiftMode == RATE_LITTLE && playRate < 1 ) {
  162. playRate = 1;
  163. shiftMode = RATE_NORMAL;
  164. Log.d(TAG, "Small shrink complete" );
  165. }
  166. if( averageFrameDelay.getAvgBufferSize() < desFrameDelay - littleStartStretch &&
  167. shiftMode == RATE_NORMAL ) {
  168. shiftMode = RATE_LITTLE;
  169. playRate = 1+littleRateShift;
  170. Log.d(TAG, "Small stretch" );
  171. }
  172. if (averageFrameDelay.getAvgBufferSize() > desFrameDelay
  173. && shiftMode == RATE_LITTLE && playRate > 1) {
  174. shiftMode = RATE_NORMAL;
  175. playRate = 1;
  176. Log.d(TAG, "Small stretch complete" );
  177. }
  178. }
  179. private void discardStaleFrames() {
  180. //discard frames that happened before the last data-frame we played
  181. int sizeBeforeDiscard = audioFrames.size();
  182. audioFrames.headMap(lastGoodFrame).clear();
  183. while( audioFrames.size() > maxBuffer ) {
  184. audioFrames.remove( audioFrames.firstKey());
  185. streamPlayheadPosition = audioFrames.firstKey();
  186. }
  187. int sizeAfterDiscard = audioFrames.size();
  188. if( sizeAfterDiscard != sizeBeforeDiscard ) {
  189. Log.d( "CAP", "Discard: " + (sizeBeforeDiscard - sizeAfterDiscard ) );
  190. }
  191. delayChooser.notifyLate( sizeBeforeDiscard - sizeAfterDiscard );
  192. }
  193. private void setDebugInfo() {
  194. CallLogger.waitingFrames = audioFrames.size();
  195. CallLogger.streamPlayheadPosition = streamPlayheadPosition;
  196. CallLogger.avgDelay = averageFrameDelay.getAvgBufferSize();
  197. CallLogger.shiftMode = shiftMode;
  198. if( audioFrames.size() > 0 )
  199. CallLogger.largestHeldFrame = audioFrames.lastKey();
  200. }
  201. public short[] getFrame() {
  202. discardStaleFrames();
  203. setDebugInfo();
  204. pullAudio();
  205. avgSamplesPerPacket.observeValue( decodeBufferLength );
  206. updatePlayRate();
  207. //model prediction frame delay offset ... is this really a good idea - confirm that it improves our estimates
  208. averageFrameDelay.setAvg(averageFrameDelay.getAvgBufferSize() + playRate-1 );//include our actions in the buffer model
  209. if( lastGoodFrame == streamPlayheadPosition ) {
  210. outputFrameLength = PacketLossConcealer.changeSpeed(rateBuffer, decodeBuffer, decodeBufferLength, playRate);
  211. } else {
  212. outputFrameLength = PacketLossConcealer.changeSpeed(rateBuffer, decodeBuffer, decodeBufferLength, 1 );
  213. }
  214. avgFrameObs.observeValue( outputFrameLength );
  215. streamPlayheadPosition++;
  216. packetLogger.logPacket(streamPlayheadPosition, PacketLogger.PLAYHEAD);
  217. delayChooser.updateDesired();
  218. callAudioLogger.update();
  219. if( decodedCount % 500 == 1 ) {
  220. Log.d( "CAP", "Decoded: " + decodedCount );
  221. }
  222. return rateBuffer;
  223. }
  224. public int getFrameSize() {
  225. return outputFrameLength;
  226. }
  227. public void terminate() {
  228. delayChooser.terminate();
  229. }
  230. }