/src/Raven.Server/Documents/TcpHandlers/SubscriptionDocumentsFetcher.cs

https://github.com/fitzchak/ravendb · C# · 281 lines · 250 code · 30 blank · 1 comment · 31 complexity · a7e06cc796766f9e8fb06134233e5815 MD5 · raw file

  1. using System;
  2. using System.Collections.Generic;
  3. using System.Diagnostics;
  4. using System.Net;
  5. using Raven.Client.Documents.Subscriptions;
  6. using Raven.Client.Exceptions.Documents.Subscriptions;
  7. using Raven.Server.Documents.Patch;
  8. using Raven.Server.Documents.Subscriptions;
  9. using Raven.Server.ServerWide.Context;
  10. using Raven.Server.Utils;
  11. using Sparrow.Json;
  12. using Sparrow.Json.Parsing;
  13. using Sparrow.Logging;
  14. namespace Raven.Server.Documents.TcpHandlers
  15. {
  16. public class SubscriptionDocumentsFetcher
  17. {
  18. private readonly DocumentDatabase _db;
  19. private readonly Logger _logger;
  20. private readonly int _maxBatchSize;
  21. private readonly long _subscriptionId;
  22. private readonly EndPoint _remoteEndpoint;
  23. private readonly string _collection;
  24. private readonly bool _revisions;
  25. private readonly SubscriptionState _subscription;
  26. private readonly SubscriptionPatchDocument _patch;
  27. public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
  28. bool revisions,
  29. SubscriptionState subscription,
  30. SubscriptionPatchDocument patch)
  31. {
  32. _db = db;
  33. _logger = LoggingSource.Instance.GetLogger<SubscriptionDocumentsFetcher>(db.Name);
  34. _maxBatchSize = maxBatchSize;
  35. _subscriptionId = subscriptionId;
  36. _remoteEndpoint = remoteEndpoint;
  37. _collection = collection;
  38. _revisions = revisions;
  39. _subscription = subscription;
  40. _patch = patch;
  41. }
  42. public IEnumerable<(Document Doc, Exception Exception)> GetDataToSend(
  43. DocumentsOperationContext docsContext,
  44. long startEtag)
  45. {
  46. if (string.IsNullOrEmpty(_collection))
  47. throw new ArgumentException("The collection name must be specified");
  48. if (_revisions)
  49. {
  50. if (_db.DocumentsStorage.RevisionsStorage.Configuration == null ||
  51. _db.DocumentsStorage.RevisionsStorage.GetRevisionsConfiguration(_collection).Disabled)
  52. throw new SubscriptionInvalidStateException($"Cannot use a revisions subscription, database {_db.Name} does not have revisions configuration.");
  53. return GetRevisionsToSend(docsContext, startEtag);
  54. }
  55. return GetDocumentsToSend(docsContext, startEtag);
  56. }
  57. private IEnumerable<(Document Doc, Exception Exception)> GetDocumentsToSend(DocumentsOperationContext docsContext,
  58. long startEtag)
  59. {
  60. int size = 0;
  61. using (_db.Scripts.GetScriptRunner(_patch,true, out var run))
  62. {
  63. foreach (var doc in _db.DocumentsStorage.GetDocumentsFrom(
  64. docsContext,
  65. _collection,
  66. startEtag + 1,
  67. 0,
  68. int.MaxValue))
  69. {
  70. using (doc.Data)
  71. {
  72. if (ShouldSendDocument(_subscription, run, _patch, docsContext, doc, out BlittableJsonReaderObject transformResult, out var exception) == false)
  73. {
  74. if (exception != null)
  75. {
  76. yield return (doc, exception);
  77. if (++size >= _maxBatchSize)
  78. yield break;
  79. }
  80. else
  81. {
  82. doc.Data = null;
  83. yield return (doc, null);
  84. }
  85. doc.Data = null;
  86. }
  87. else
  88. {
  89. using (transformResult)
  90. {
  91. if (transformResult == null)
  92. {
  93. yield return (doc, null);
  94. }
  95. else
  96. {
  97. yield return (new Document
  98. {
  99. Id = doc.Id,
  100. Etag = doc.Etag,
  101. Data = transformResult,
  102. LowerId = doc.LowerId,
  103. ChangeVector = doc.ChangeVector,
  104. LastModified = doc.LastModified,
  105. }, null);
  106. }
  107. }
  108. if (++size >= _maxBatchSize)
  109. yield break;
  110. }
  111. }
  112. }
  113. }
  114. }
  115. private IEnumerable<(Document Doc, Exception Exception)> GetRevisionsToSend(
  116. DocumentsOperationContext docsContext,
  117. long startEtag)
  118. {
  119. int size = 0;
  120. var collectionName = new CollectionName(_collection);
  121. using (_db.Scripts.GetScriptRunner(_patch, true, out var run))
  122. {
  123. foreach (var revisionTuple in _db.DocumentsStorage.RevisionsStorage.GetRevisionsFrom(docsContext, collectionName, startEtag + 1, int.MaxValue))
  124. {
  125. var item = (revisionTuple.current ?? revisionTuple.previous);
  126. Debug.Assert(item != null);
  127. if (ShouldSendDocumentWithRevisions(_subscription, run, _patch, docsContext, item, revisionTuple, out var transformResult, out var exception) == false)
  128. {
  129. if (exception != null)
  130. {
  131. yield return (item, exception);
  132. if (++size >= _maxBatchSize)
  133. yield break;
  134. }
  135. else
  136. {
  137. // make sure that if we read a lot of irrelevant documents, we send keep alive over the network
  138. yield return (new Document
  139. {
  140. Data = null,
  141. ChangeVector = item.ChangeVector,
  142. Etag = item.Etag,
  143. LastModified = item.LastModified,
  144. }, null);
  145. }
  146. }
  147. else
  148. {
  149. using (transformResult)
  150. {
  151. if (transformResult == null)
  152. {
  153. yield return (revisionTuple.current, null);
  154. }
  155. else
  156. {
  157. yield return (new Document
  158. {
  159. Id = item.Id,
  160. Etag = item.Etag,
  161. Data = transformResult,
  162. LowerId = item.LowerId,
  163. ChangeVector = item.ChangeVector,
  164. LastModified = item.LastModified,
  165. }, null);
  166. }
  167. if (++size >= _maxBatchSize)
  168. yield break;
  169. }
  170. }
  171. }
  172. }
  173. }
  174. private bool ShouldSendDocument(SubscriptionState subscriptionState,
  175. ScriptRunner.SingleRun run,
  176. SubscriptionPatchDocument patch,
  177. DocumentsOperationContext dbContext,
  178. Document doc,
  179. out BlittableJsonReaderObject transformResult,
  180. out Exception exception)
  181. {
  182. transformResult = null;
  183. exception = null;
  184. var conflictStatus = ChangeVectorUtils.GetConflictStatus(
  185. remoteAsString: doc.ChangeVector,
  186. localAsString: subscriptionState.ChangeVectorForNextBatchStartingPoint);
  187. if (conflictStatus == ConflictStatus.AlreadyMerged)
  188. return false;
  189. if (patch == null)
  190. return true;
  191. try
  192. {
  193. return patch.MatchCriteria(run, dbContext, doc, ref transformResult);
  194. }
  195. catch (Exception ex)
  196. {
  197. if (_logger.IsInfoEnabled)
  198. {
  199. _logger.Info(
  200. $"Criteria script threw exception for subscription {_subscriptionId} connected to {_remoteEndpoint} for document id {doc.Id}",
  201. ex);
  202. }
  203. exception = ex;
  204. return false;
  205. }
  206. }
  207. private bool ShouldSendDocumentWithRevisions(SubscriptionState subscriptionState,
  208. ScriptRunner.SingleRun run,
  209. SubscriptionPatchDocument patch,
  210. DocumentsOperationContext dbContext,
  211. Document item,
  212. (Document Previous, Document Current) revision,
  213. out BlittableJsonReaderObject transformResult,
  214. out Exception exception)
  215. {
  216. exception = null;
  217. transformResult = null;
  218. var conflictStatus = ChangeVectorUtils.GetConflictStatus(
  219. remoteAsString: item.ChangeVector,
  220. localAsString: subscriptionState.ChangeVectorForNextBatchStartingPoint);
  221. if (conflictStatus == ConflictStatus.AlreadyMerged)
  222. return false;
  223. revision.Current?.EnsureMetadata();
  224. revision.Previous?.EnsureMetadata();
  225. transformResult = dbContext.ReadObject(new DynamicJsonValue
  226. {
  227. ["Current"] = revision.Current?.Data,
  228. ["Previous"] = revision.Previous?.Data
  229. }, item.Id);
  230. if (patch == null)
  231. return true;
  232. revision.Current?.ResetModifications();
  233. revision.Previous?.ResetModifications();
  234. try
  235. {
  236. return patch.MatchCriteria(run, dbContext, transformResult, ref transformResult);
  237. }
  238. catch (Exception ex)
  239. {
  240. if (_logger.IsInfoEnabled)
  241. {
  242. _logger.Info(
  243. $"Criteria script threw exception for subscription {_subscriptionId} connected to {_remoteEndpoint} for document id {item.Id}",
  244. ex);
  245. }
  246. exception = ex;
  247. return false;
  248. }
  249. }
  250. }
  251. }