PageRenderTime 54ms CodeModel.GetById 16ms RepoModel.GetById 1ms app.codeStats 0ms

/src/Voron/GlobalFlushingBehavior.cs

http://github.com/ayende/ravendb
C# | 316 lines | 240 code | 50 blank | 26 comment | 42 complexity | cb1e812b1dcad67f4815b03b71b1c06f MD5 | raw file
Possible License(s): GPL-3.0, MPL-2.0-no-copyleft-exception, LGPL-2.1, Apache-2.0, BSD-3-Clause, CC-BY-SA-3.0
  1. using System;
  2. using System.Collections.Concurrent;
  3. using System.Collections.Generic;
  4. using System.Runtime.ExceptionServices;
  5. using System.Threading;
  6. using Sparrow;
  7. using Sparrow.Logging;
  8. using Sparrow.Utils;
  9. using Voron.Impl.Journal;
  10. namespace Voron
  11. {
  12. public class GlobalFlushingBehavior
  13. {
  14. private const string FlushingThreadName = "Voron Global Flushing Thread";
  15. internal static readonly Lazy<GlobalFlushingBehavior> GlobalFlusher = new Lazy<GlobalFlushingBehavior>(() =>
  16. {
  17. var flusher = new GlobalFlushingBehavior();
  18. var thread = new Thread(flusher.VoronEnvironmentFlushing)
  19. {
  20. IsBackground = true,
  21. Name = FlushingThreadName
  22. };
  23. thread.Start();
  24. return flusher;
  25. });
  26. private class EnvSyncReq
  27. {
  28. public StorageEnvironment Env => Reference?.Owner;
  29. public StorageEnvironment.IndirectReference Reference;
  30. public long LastKnownSyncCounter;
  31. }
  32. private readonly ManualResetEventSlim _flushWriterEvent = new ManualResetEventSlim();
  33. private readonly int _lowNumberOfFlushingResources = Math.Max(StorageEnvironment.MaxConcurrentFlushes / 10, 3);
  34. private readonly SemaphoreSlim _concurrentFlushes = new SemaphoreSlim(StorageEnvironment.MaxConcurrentFlushes);
  35. private readonly ConcurrentQueue<EnvSyncReq> _maybeNeedToFlush = new ConcurrentQueue<EnvSyncReq>();
  36. private readonly ConcurrentQueue<EnvSyncReq> _maybeNeedToSync = new ConcurrentQueue<EnvSyncReq>();
  37. private readonly ConcurrentQueue<EnvSyncReq> _syncIsRequired = new ConcurrentQueue<EnvSyncReq>();
  38. private readonly ConcurrentDictionary<uint, MountPointInfo> _mountPoints = new ConcurrentDictionary<uint, MountPointInfo>();
  39. private readonly Logger _log = LoggingSource.Instance.GetLogger<GlobalFlushingBehavior>("Global Flusher");
  40. private class MountPointInfo
  41. {
  42. public readonly ConcurrentQueue<EnvSyncReq> StorageEnvironments = new ConcurrentQueue<EnvSyncReq>();
  43. }
  44. public bool HasLowNumberOfFlushingResources => _concurrentFlushes.CurrentCount <= _lowNumberOfFlushingResources;
  45. public void VoronEnvironmentFlushing()
  46. {
  47. NativeMemory.EnsureRegistered();
  48. // We want this to always run, even if we dispose / create new storage env, this is
  49. // static for the life time of the process, and environments will register / unregister from
  50. // it as needed
  51. try
  52. {
  53. var avoidDuplicates = new HashSet<StorageEnvironment>();
  54. while (true)
  55. {
  56. avoidDuplicates.Clear();
  57. var maybeNeedSync = _maybeNeedToSync.Count;
  58. var millisecondsTimeout = 15000 - maybeNeedSync;
  59. if (millisecondsTimeout <= 0 ||
  60. _flushWriterEvent.Wait(millisecondsTimeout) == false)
  61. {
  62. if (_maybeNeedToSync.Count == 0)
  63. continue;
  64. if (_log.IsInfoEnabled)
  65. {
  66. _log.Info($"Starting desired sync with {_maybeNeedToSync.Count:#,#} items to sync after {millisecondsTimeout:#,#} ms with no activity");
  67. }
  68. // sync after 5 seconds if no flushing occurred, or if there has been a LOT of
  69. // writes that we would like to run
  70. SyncDesiredEnvironments(avoidDuplicates);
  71. continue;
  72. }
  73. _flushWriterEvent.Reset();
  74. FlushEnvironments(avoidDuplicates);
  75. SyncRequiredEnvironments(avoidDuplicates);
  76. }
  77. }
  78. catch (Exception e)
  79. {
  80. if (_log.IsOperationsEnabled)
  81. {
  82. _log.Operations("Catastrophic failure in Voron environment flushing", e);
  83. }
  84. // wait for the message to be flushed to the logs
  85. Thread.Sleep(5000);
  86. // Note that we intentionally don't have error handling here.
  87. // If this code throws an exception that bubbles up to here, we WANT the process
  88. // to die, since we can't recover from the flusher thread dying.
  89. throw;
  90. }
  91. // ReSharper disable once FunctionNeverReturns
  92. }
  93. private void SyncDesiredEnvironments(HashSet<StorageEnvironment> avoidDuplicates)
  94. {
  95. avoidDuplicates.Clear();
  96. EnvSyncReq envToSync;
  97. var limit = _maybeNeedToSync.Count;
  98. while (
  99. // if there is high traffic into the queue, we want to abort after
  100. // we processed whatever was already in there, to avoid holding up
  101. // the rest of the operations
  102. limit-- > 0 &&
  103. _maybeNeedToSync.TryDequeue(out envToSync))
  104. {
  105. var storageEnvironment = envToSync.Env;
  106. if (storageEnvironment == null)
  107. continue;
  108. if (avoidDuplicates.Add(storageEnvironment) == false)
  109. continue; // already seen
  110. if (storageEnvironment.Disposed)
  111. continue;
  112. var mpi = _mountPoints.GetOrAdd(storageEnvironment.Options.DataPager.UniquePhysicalDriveId,
  113. _ => new MountPointInfo());
  114. mpi.StorageEnvironments.Enqueue(envToSync);
  115. }
  116. foreach (var mountPoint in _mountPoints)
  117. {
  118. int parallelSyncsPerIo = Math.Min(StorageEnvironment.NumOfConcurrentSyncsPerPhysDrive, mountPoint.Value.StorageEnvironments.Count);
  119. for (int i = 0; i < parallelSyncsPerIo; i++)
  120. {
  121. ThreadPool.QueueUserWorkItem(SyncAllEnvironmentsInMountPoint, mountPoint.Value);
  122. }
  123. }
  124. }
  125. private void SyncRequiredEnvironments(HashSet<StorageEnvironment> avoidDuplicates)
  126. {
  127. avoidDuplicates.Clear();
  128. var limit = _syncIsRequired.Count;
  129. while (
  130. // if there is high traffic into the queue, we want to abort after
  131. // we processed whatever was already in there, to avoid holding up
  132. // the rest of the operations
  133. limit-- > 0 &&
  134. _syncIsRequired.TryDequeue(out EnvSyncReq item))
  135. {
  136. var storageEnvironment = item.Env;
  137. if(storageEnvironment == null)
  138. continue;
  139. if (avoidDuplicates.Add(storageEnvironment) == false)
  140. continue; // avoid duplicates in batch
  141. ThreadPool.QueueUserWorkItem(state => SyncEnvironment((EnvSyncReq)state), item);
  142. }
  143. }
  144. private void SyncAllEnvironmentsInMountPoint(object mt)
  145. {
  146. var mountPointInfo = (MountPointInfo)mt;
  147. EnvSyncReq req;
  148. while (mountPointInfo.StorageEnvironments.TryDequeue(out req))
  149. {
  150. SyncEnvironment(req);
  151. // we have multiple threads racing for this value, no a concern, the last one wins is probably
  152. // going to be the latest, or close enough that we don't care
  153. var storageEnvironment = req.Env;
  154. if(storageEnvironment != null)
  155. Interlocked.Exchange(ref storageEnvironment.LastSyncTimeInTicks, DateTime.UtcNow.Ticks);
  156. }
  157. }
  158. private void SyncEnvironment(EnvSyncReq req)
  159. {
  160. var storageEnvironment = req.Env;
  161. if (storageEnvironment == null || storageEnvironment.Disposed)
  162. return;
  163. if (storageEnvironment.LastSyncCounter > req.LastKnownSyncCounter)
  164. return; // we already a sync after this was scheduled
  165. try
  166. {
  167. using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(storageEnvironment.Journal.Applicator))
  168. {
  169. operation.SyncDataFile();
  170. }
  171. }
  172. catch (Exception e)
  173. {
  174. if (_log.IsOperationsEnabled)
  175. _log.Operations($"Failed to sync data file for {storageEnvironment.Options.BasePath}", e);
  176. storageEnvironment.Options.SetCatastrophicFailure(ExceptionDispatchInfo.Capture(e));
  177. }
  178. }
  179. private void FlushEnvironments(HashSet<StorageEnvironment> avoidDuplicates)
  180. {
  181. avoidDuplicates.Clear();
  182. EnvSyncReq req;
  183. var limit = _maybeNeedToFlush.Count;
  184. while (
  185. // if there is high traffic into the queue, we want to abort after
  186. // we processed whatever was already in there, to avoid holding up
  187. // the rest of the operations
  188. limit-- > 0 &&
  189. _maybeNeedToFlush.TryDequeue(out req))
  190. {
  191. var envToFlush = req.Env;
  192. if (envToFlush == null)
  193. continue;
  194. if (avoidDuplicates.Add(envToFlush) == false)
  195. continue; // avoid duplicates
  196. if (envToFlush.Disposed || envToFlush.Options.ManualFlushing)
  197. continue;
  198. var sizeOfUnflushedTransactionsInJournalFile = envToFlush.SizeOfUnflushedTransactionsInJournalFile;
  199. if (sizeOfUnflushedTransactionsInJournalFile == 0)
  200. continue; // nothing to do
  201. if (sizeOfUnflushedTransactionsInJournalFile < envToFlush.Options.MaxNumberOfPagesInJournalBeforeFlush)
  202. {
  203. // we haven't reached the point where we have to flush, but we might want to, if we have enough
  204. // resources available, if we have more than half the flushing capacity, we can do it now, otherwise, we'll wait
  205. // until it is actually required.
  206. if (_concurrentFlushes.CurrentCount < StorageEnvironment.MaxConcurrentFlushes / 2)
  207. continue;
  208. // At the same time, we want to avoid excessive flushes, so we'll limit it to once in a while if we don't
  209. // have a lot to flush
  210. if ((DateTime.UtcNow - envToFlush.LastFlushTime).TotalSeconds < StorageEnvironment.TimeToSyncAfterFlashInSec)
  211. continue;
  212. }
  213. envToFlush.LastFlushTime = DateTime.UtcNow;
  214. Interlocked.Add(ref envToFlush.SizeOfUnflushedTransactionsInJournalFile, -sizeOfUnflushedTransactionsInJournalFile);
  215. _concurrentFlushes.Wait();
  216. ThreadPool.QueueUserWorkItem(env =>
  217. {
  218. var storageEnvironment = ((StorageEnvironment)env);
  219. try
  220. {
  221. if (storageEnvironment.Disposed)
  222. return;
  223. storageEnvironment.BackgroundFlushWritesToDataFile();
  224. }
  225. catch (Exception e)
  226. {
  227. if (_log.IsOperationsEnabled)
  228. _log.Operations($"Failed to flush {storageEnvironment.Options.BasePath}", e);
  229. storageEnvironment.Options.SetCatastrophicFailure(ExceptionDispatchInfo.Capture(e));
  230. }
  231. finally
  232. {
  233. _concurrentFlushes.Release();
  234. }
  235. }, envToFlush);
  236. }
  237. }
  238. public void MaybeFlushEnvironment(StorageEnvironment env)
  239. {
  240. if (env.Options.ManualFlushing)
  241. return;
  242. _maybeNeedToFlush.Enqueue(new EnvSyncReq
  243. {
  244. Reference = env.SelfReference,
  245. });
  246. _flushWriterEvent.Set();
  247. }
  248. public void MaybeSyncEnvironment(StorageEnvironment env)
  249. {
  250. _maybeNeedToSync.Enqueue(new EnvSyncReq
  251. {
  252. Reference = env.SelfReference,
  253. LastKnownSyncCounter = env.LastSyncCounter
  254. });
  255. }
  256. public void ForceFlushAndSyncEnvironment(StorageEnvironment env)
  257. {
  258. _syncIsRequired.Enqueue(new EnvSyncReq
  259. {
  260. Reference = env.SelfReference,
  261. LastKnownSyncCounter = env.LastSyncCounter
  262. });
  263. _flushWriterEvent.Set();
  264. }
  265. }
  266. }