PageRenderTime 57ms CodeModel.GetById 26ms RepoModel.GetById 0ms app.codeStats 1ms

/ToMigrate/Raven.Database/Server/Controllers/StreamsController.cs

http://github.com/ayende/ravendb
C# | 605 lines | 529 code | 67 blank | 9 comment | 40 complexity | f5bf8d93f9d60e0bad0a0a179adfb1c1 MD5 | raw file
Possible License(s): GPL-3.0, MPL-2.0-no-copyleft-exception, LGPL-2.1, Apache-2.0, BSD-3-Clause, CC-BY-SA-3.0
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Collections.Specialized;
  4. using System.Diagnostics;
  5. using System.Globalization;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Net;
  9. using System.Net.Http;
  10. using System.Net.Http.Headers;
  11. using System.Security.Principal;
  12. using System.Text;
  13. using System.Threading;
  14. using System.Threading.Tasks;
  15. using System.Web.Http;
  16. using Raven.Abstractions;
  17. using Raven.Abstractions.Data;
  18. using Raven.Abstractions.Extensions;
  19. using Raven.Abstractions.Indexing;
  20. using Raven.Abstractions.Linq;
  21. using Raven.Abstractions.Util;
  22. using Raven.Abstractions.Util.Encryptors;
  23. using Raven.Database.Actions;
  24. using Raven.Database.Extensions;
  25. using Raven.Database.Impl;
  26. using Raven.Database.Server.WebApi.Attributes;
  27. using Raven.Database.Storage;
  28. using Raven.Imports.Newtonsoft.Json;
  29. using Raven.Imports.Newtonsoft.Json.Linq;
  30. using Raven.Json.Linq;
  31. using Sparrow;
  32. namespace Raven.Database.Server.Controllers
  33. {
  34. public class StreamsController : ClusterAwareRavenDbApiController
  35. {
  36. [HttpHead]
  37. [RavenRoute("streams/docs")]
  38. [RavenRoute("databases/{databaseName}/streams/docs")]
  39. public HttpResponseMessage StreamDocsHead()
  40. {
  41. return GetEmptyMessage();
  42. }
  43. [HttpGet]
  44. [RavenRoute("streams/docs")]
  45. [RavenRoute("databases/{databaseName}/streams/docs")]
  46. public HttpResponseMessage StreamDocsGet()
  47. {
  48. var start = GetStart();
  49. var etag = GetEtagFromQueryString();
  50. var startsWith = GetQueryStringValue("startsWith");
  51. var pageSize = GetPageSize(int.MaxValue);
  52. var matches = GetQueryStringValue("matches");
  53. var nextPageStart = GetNextPageStart();
  54. if (string.IsNullOrEmpty(GetQueryStringValue("pageSize")))
  55. pageSize = int.MaxValue;
  56. var skipAfter = GetQueryStringValue("skipAfter");
  57. var transformer = GetQueryStringValue("transformer");
  58. var transformerParameters = ExtractTransformerParameters();
  59. var headers = CurrentOperationContext.Headers.Value;
  60. var user = CurrentOperationContext.User.Value;
  61. return new HttpResponseMessage(HttpStatusCode.OK)
  62. {
  63. Content = new PushStreamContent((stream, content, transportContext) =>
  64. StreamToClient(stream, startsWith, start, pageSize, etag, matches, nextPageStart, skipAfter, transformer, transformerParameters, headers, user))
  65. {
  66. Headers =
  67. {
  68. ContentType = new MediaTypeHeaderValue("application/json") { CharSet = "utf-8" }
  69. }
  70. }
  71. };
  72. }
  73. private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter, string transformer, Dictionary<string, RavenJToken> transformerParameters,
  74. Lazy<NameValueCollection> headers, IPrincipal user)
  75. {
  76. var old = CurrentOperationContext.Headers.Value;
  77. var oldUser = CurrentOperationContext.User.Value;
  78. try
  79. {
  80. CurrentOperationContext.Headers.Value = headers;
  81. CurrentOperationContext.User.Value = user;
  82. var bufferStream = new BufferedStream(stream, 1024 * 64);
  83. using (var cts = new CancellationTokenSource())
  84. using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan))
  85. using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
  86. {
  87. writer.WriteStartObject();
  88. writer.WritePropertyName("Results");
  89. writer.WriteStartArray();
  90. Action<JsonDocument> addDocument = doc =>
  91. {
  92. timeout.Delay();
  93. if (doc == null)
  94. {
  95. // we only have this heartbit when the streaming has gone on for a long time
  96. // and we haven't send anything to the user in a while (because of filtering, skipping, etc).
  97. writer.WriteRaw(Environment.NewLine);
  98. writer.Flush();
  99. return;
  100. }
  101. doc.ToJson().WriteTo(writer);
  102. writer.WriteRaw(Environment.NewLine);
  103. };
  104. Database.TransactionalStorage.Batch(accessor =>
  105. {
  106. // we may be sending a LOT of documents to the user, and most
  107. // of them aren't going to be relevant for other ops, so we are going to skip
  108. // the cache for that, to avoid filling it up very quickly
  109. using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
  110. {
  111. if (string.IsNullOrEmpty(startsWith))
  112. {
  113. Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, doc => { addDocument(doc); return true; }, transformer, transformerParameters);
  114. }
  115. else
  116. {
  117. var nextPageStartInternal = nextPageStart;
  118. Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, addDocument, transformer, transformerParameters, skipAfter);
  119. nextPageStart = nextPageStartInternal;
  120. }
  121. }
  122. });
  123. writer.WriteEndArray();
  124. writer.WritePropertyName("NextPageStart");
  125. writer.WriteValue(nextPageStart);
  126. writer.WriteEndObject();
  127. writer.Flush();
  128. bufferStream.Flush();
  129. }
  130. }
  131. finally
  132. {
  133. CurrentOperationContext.Headers.Value = old;
  134. CurrentOperationContext.User.Value = oldUser;
  135. }
  136. }
  137. [HttpHead]
  138. [RavenRoute("streams/query/{*id}")]
  139. [RavenRoute("databases/{databaseName}/streams/query/{*id}")]
  140. public HttpResponseMessage SteamQueryHead(string id)
  141. {
  142. return GetEmptyMessage();
  143. }
  144. [HttpGet]
  145. [RavenRoute("streams/query/{*id}")]
  146. [RavenRoute("databases/{databaseName}/streams/query/{*id}")]
  147. public HttpResponseMessage SteamQueryGet(string id)
  148. {
  149. var cts = new CancellationTokenSource();
  150. var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan);
  151. var msg = GetEmptyMessage();
  152. var index = id;
  153. var query = GetIndexQuery(int.MaxValue);
  154. if (string.IsNullOrEmpty(GetQueryStringValue("pageSize"))) query.PageSize = int.MaxValue;
  155. var isHeadRequest = InnerRequest.Method == HttpMethods.Head;
  156. if (isHeadRequest) query.PageSize = 0;
  157. var accessor = Database.TransactionalStorage.CreateAccessor(); //accessor will be disposed in the StreamQueryContent.SerializeToStreamAsync!
  158. try
  159. {
  160. var queryOp = new QueryActions.DatabaseQueryOperation(Database, index, query, accessor, cts);
  161. queryOp.Init();
  162. msg.Content = new StreamQueryContent(InnerRequest, queryOp, accessor, timeout, mediaType => msg.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType) {CharSet = "utf-8"});
  163. msg.Headers.Add("Raven-Result-Etag", queryOp.Header.ResultEtag.ToString());
  164. msg.Headers.Add("Raven-Index-Etag", queryOp.Header.IndexEtag.ToString());
  165. msg.Headers.Add("Raven-Is-Stale", queryOp.Header.IsStale ? "true" : "false");
  166. msg.Headers.Add("Raven-Index", queryOp.Header.Index);
  167. msg.Headers.Add("Raven-Total-Results", queryOp.Header.TotalResults.ToString(CultureInfo.InvariantCulture));
  168. msg.Headers.Add("Raven-Index-Timestamp", queryOp.Header.IndexTimestamp.GetDefaultRavenFormat());
  169. if (IsCsvDownloadRequest(InnerRequest))
  170. {
  171. msg.Content.Headers.Add("Content-Disposition", "attachment; filename=export.csv");
  172. }
  173. }
  174. catch (OperationCanceledException e)
  175. {
  176. accessor.Dispose();
  177. throw new TimeoutException(string.Format("The query did not produce results in {0}", DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan), e);
  178. }
  179. catch (Exception)
  180. {
  181. accessor.Dispose();
  182. throw;
  183. }
  184. return msg;
  185. }
  186. [HttpGet]
  187. [RavenRoute("streams/exploration")]
  188. [RavenRoute("databases/{databaseName}/streams/exploration")]
  189. public Task<HttpResponseMessage> Exploration()
  190. {
  191. var linq = GetQueryStringValue("linq");
  192. var collection = GetQueryStringValue("collection");
  193. int timeoutSeconds;
  194. if (int.TryParse(GetQueryStringValue("timeoutSeconds"), out timeoutSeconds) == false)
  195. timeoutSeconds = 60;
  196. int pageSize;
  197. if (int.TryParse(GetQueryStringValue("pageSize"), out pageSize) == false)
  198. pageSize = 100000;
  199. var hash = Hashing.XXHash64.CalculateRaw(linq);
  200. var sourceHashed = hash.ToString("X");
  201. var transformerName = Constants.TemporaryTransformerPrefix + sourceHashed;
  202. var transformerDefinition = Database.IndexDefinitionStorage.GetTransformerDefinition(transformerName);
  203. if (transformerDefinition == null)
  204. {
  205. transformerDefinition = new TransformerDefinition
  206. {
  207. Name = transformerName,
  208. Temporary = true,
  209. TransformResults = linq
  210. };
  211. Database.Transformers.PutTransform(transformerName, transformerDefinition);
  212. }
  213. var msg = GetEmptyMessage();
  214. using (var cts = new CancellationTokenSource())
  215. {
  216. var timeout = cts.TimeoutAfter(TimeSpan.FromSeconds(timeoutSeconds));
  217. var indexQuery = new IndexQuery
  218. {
  219. PageSize = pageSize,
  220. Start = 0,
  221. Query = "Tag:" + collection,
  222. ResultsTransformer = transformerName
  223. };
  224. var accessor = Database.TransactionalStorage.CreateAccessor(); //accessor will be disposed in the StreamQueryContent.SerializeToStreamAsync!
  225. try
  226. {
  227. var queryOp = new QueryActions.DatabaseQueryOperation(Database, "Raven/DocumentsByEntityName", indexQuery, accessor, cts);
  228. queryOp.Init();
  229. msg.Content = new StreamQueryContent(InnerRequest, queryOp, accessor, timeout, mediaType => msg.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType) { CharSet = "utf-8" },
  230. o =>
  231. {
  232. if (o.Count == 2 &&
  233. o.ContainsKey(Constants.DocumentIdFieldName) &&
  234. o.ContainsKey(Constants.Metadata))
  235. {
  236. // this is the raw value out of the server, we don't want to get that
  237. var doc = queryOp.DocRetriever.Load(o.Value<string>(Constants.DocumentIdFieldName));
  238. var djo = doc as IDynamicJsonObject;
  239. if (djo != null)
  240. return djo.Inner;
  241. }
  242. return o;
  243. });
  244. msg.Headers.Add("Raven-Result-Etag", queryOp.Header.ResultEtag.ToString());
  245. msg.Headers.Add("Raven-Index-Etag", queryOp.Header.IndexEtag.ToString());
  246. msg.Headers.Add("Raven-Is-Stale", queryOp.Header.IsStale ? "true" : "false");
  247. msg.Headers.Add("Raven-Index", queryOp.Header.Index);
  248. msg.Headers.Add("Raven-Total-Results", queryOp.Header.TotalResults.ToString(CultureInfo.InvariantCulture));
  249. msg.Headers.Add("Raven-Index-Timestamp", queryOp.Header.IndexTimestamp.GetDefaultRavenFormat());
  250. if (IsCsvDownloadRequest(InnerRequest))
  251. {
  252. msg.Content.Headers.Add("Content-Disposition", "attachment; filename=export.csv");
  253. }
  254. }
  255. catch (Exception)
  256. {
  257. accessor.Dispose();
  258. throw;
  259. }
  260. return new CompletedTask<HttpResponseMessage>(msg);
  261. }
  262. }
  263. [HttpPost]
  264. [RavenRoute("streams/query/{*id}")]
  265. [RavenRoute("databases/{databaseName}/streams/query/{*id}")]
  266. public async Task<HttpResponseMessage> SteamQueryPost(string id)
  267. {
  268. var postedQuery = await ReadStringAsync().ConfigureAwait(false);
  269. SetPostRequestQuery(postedQuery);
  270. return SteamQueryGet(id);
  271. }
  272. public class StreamQueryContent : HttpContent
  273. {
  274. private readonly HttpRequestMessage req;
  275. private readonly QueryActions.DatabaseQueryOperation queryOp;
  276. private readonly IStorageActionsAccessor accessor;
  277. private readonly CancellationTimeout _timeout;
  278. private readonly Action<string> outputContentTypeSetter;
  279. private Lazy<NameValueCollection> headers;
  280. private IPrincipal user;
  281. private readonly Func<RavenJObject, RavenJObject> modifyDocument;
  282. [CLSCompliant(false)]
  283. public StreamQueryContent(HttpRequestMessage req, QueryActions.DatabaseQueryOperation queryOp, IStorageActionsAccessor accessor,
  284. CancellationTimeout timeout,
  285. Action<string> contentTypeSetter,
  286. Func<RavenJObject,RavenJObject> modifyDocument = null)
  287. {
  288. headers = CurrentOperationContext.Headers.Value;
  289. user = CurrentOperationContext.User.Value;
  290. this.req = req;
  291. this.queryOp = queryOp;
  292. this.accessor = accessor;
  293. _timeout = timeout;
  294. outputContentTypeSetter = contentTypeSetter;
  295. this.modifyDocument = modifyDocument;
  296. }
  297. protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
  298. {
  299. var old = CurrentOperationContext.Headers.Value;
  300. var oldUser = CurrentOperationContext.User.Value;
  301. try
  302. {
  303. CurrentOperationContext.User.Value = user;
  304. CurrentOperationContext.Headers.Value = headers;
  305. var bufferSize = queryOp.Header.TotalResults > 1024 ? 1024 * 64 : 1024 * 8;
  306. using (var bufferedStream = new BufferedStream(stream, bufferSize))
  307. using (queryOp)
  308. using (accessor)
  309. using (_timeout)
  310. using (var writer = GetOutputWriter(req, bufferedStream))
  311. // we may be sending a LOT of documents to the user, and most
  312. // of them aren't going to be relevant for other ops, so we are going to skip
  313. // the cache for that, to avoid filling it up very quickly
  314. using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
  315. {
  316. outputContentTypeSetter(writer.ContentType);
  317. writer.WriteHeader();
  318. try
  319. {
  320. queryOp.Execute(o =>
  321. {
  322. _timeout.Delay();
  323. if (modifyDocument != null)
  324. o = modifyDocument(o);
  325. writer.Write(o);
  326. });
  327. }
  328. catch (Exception e)
  329. {
  330. writer.WriteError(e);
  331. }
  332. }
  333. return Task.FromResult(true);
  334. }
  335. finally
  336. {
  337. CurrentOperationContext.Headers.Value = old;
  338. CurrentOperationContext.User.Value = oldUser;
  339. }
  340. }
  341. protected override bool TryComputeLength(out long length)
  342. {
  343. length = -1;
  344. return false;
  345. }
  346. }
  347. private static IOutputWriter GetOutputWriter(HttpRequestMessage req, Stream stream)
  348. {
  349. var useExcelFormat = "excel".Equals(GetQueryStringValue(req, "format"), StringComparison.InvariantCultureIgnoreCase);
  350. return useExcelFormat ? (IOutputWriter)new ExcelOutputWriter(stream) : new JsonOutputWriter(stream);
  351. }
  352. private static Boolean IsCsvDownloadRequest(HttpRequestMessage req)
  353. {
  354. return "true".Equals(GetQueryStringValue(req, "download"), StringComparison.InvariantCultureIgnoreCase)
  355. && "excel".Equals(GetQueryStringValue(req, "format"), StringComparison.InvariantCultureIgnoreCase);
  356. }
  357. public interface IOutputWriter : IDisposable
  358. {
  359. string ContentType { get; }
  360. void WriteHeader();
  361. void Write(RavenJObject result);
  362. void WriteError(Exception exception);
  363. }
  364. private class ExcelOutputWriter : IOutputWriter
  365. {
  366. private const string CsvContentType = "text/csv";
  367. private readonly Stream stream;
  368. private StreamWriter writer;
  369. private bool doIncludeId;
  370. public ExcelOutputWriter(Stream stream)
  371. {
  372. this.stream = stream;
  373. }
  374. public string ContentType
  375. {
  376. get { return CsvContentType; }
  377. }
  378. public void Dispose()
  379. {
  380. if (writer == null)
  381. return;
  382. writer.Flush();
  383. stream.Flush();
  384. writer.Close();
  385. }
  386. public void WriteHeader()
  387. {
  388. writer = new StreamWriter(stream, Encoding.UTF8);
  389. }
  390. public void Write(RavenJObject result)
  391. {
  392. if (properties == null)
  393. {
  394. GetPropertiesAndWriteCsvHeader(result, out doIncludeId);
  395. Debug.Assert(properties != null);
  396. }
  397. if (doIncludeId)
  398. {
  399. RavenJToken token;
  400. if (result.TryGetValue("@metadata", out token))
  401. {
  402. var metadata = token as RavenJObject;
  403. if (metadata != null)
  404. {
  405. if (metadata.TryGetValue("@id", out token))
  406. {
  407. OutputCsvValue(token.Value<string>());
  408. }
  409. writer.Write(',');
  410. }
  411. }
  412. }
  413. foreach (var property in properties)
  414. {
  415. var token = result.SelectToken(property);
  416. if (token != null)
  417. {
  418. switch (token.Type)
  419. {
  420. case JTokenType.Null:
  421. break;
  422. case JTokenType.Array:
  423. case JTokenType.Object:
  424. OutputCsvValue(token.ToString(Formatting.None));
  425. break;
  426. default:
  427. OutputCsvValue(token.Value<string>());
  428. break;
  429. }
  430. }
  431. writer.Write(',');
  432. }
  433. writer.WriteLine();
  434. }
  435. public void WriteError(Exception exception)
  436. {
  437. writer.WriteLine();
  438. writer.WriteLine();
  439. writer.WriteLine(exception.ToString());
  440. }
  441. private void GetPropertiesAndWriteCsvHeader(RavenJObject result, out bool includeId)
  442. {
  443. includeId = false;
  444. properties = DocumentHelpers.GetPropertiesFromJObject(result,
  445. parentPropertyPath: "",
  446. includeNestedProperties: true,
  447. includeMetadata: false,
  448. excludeParentPropertyNames: true).ToList();
  449. RavenJToken token;
  450. if (result.TryGetValue("@metadata", out token))
  451. {
  452. var metadata = token as RavenJObject;
  453. if (metadata != null)
  454. {
  455. if (metadata.TryGetValue("@id", out token))
  456. {
  457. OutputCsvValue("@id");
  458. writer.Write(',');
  459. includeId = true;
  460. }
  461. }
  462. }
  463. foreach (var property in properties)
  464. {
  465. OutputCsvValue(property);
  466. writer.Write(',');
  467. }
  468. writer.WriteLine();
  469. }
  470. private static readonly char[] RequireQuotesChars = { ',', '\r', '\n', '"' };
  471. private IEnumerable<string> properties;
  472. private void OutputCsvValue(string val)
  473. {
  474. var needsQuoutes = val.IndexOfAny(RequireQuotesChars) != -1;
  475. if (needsQuoutes)
  476. writer.Write('"');
  477. writer.Write(needsQuoutes ? val.Replace("\"", "\"\"") : val);
  478. if (needsQuoutes)
  479. writer.Write('"');
  480. }
  481. }
  482. public class JsonOutputWriter : IOutputWriter
  483. {
  484. private const string JsonContentType = "application/json";
  485. private readonly Stream stream;
  486. private JsonWriter writer;
  487. private bool closedArray = false;
  488. public JsonOutputWriter(Stream stream)
  489. {
  490. this.stream = stream;
  491. }
  492. public string ContentType
  493. {
  494. get { return JsonContentType; }
  495. }
  496. public void WriteHeader()
  497. {
  498. writer = new JsonTextWriter(new StreamWriter(stream));
  499. writer.WriteStartObject();
  500. writer.WritePropertyName("Results");
  501. writer.WriteStartArray();
  502. }
  503. public void Dispose()
  504. {
  505. if (writer == null)
  506. return;
  507. if (closedArray == false)
  508. writer.WriteEndArray();
  509. writer.WriteEndObject();
  510. writer.Flush();
  511. stream.Flush();
  512. writer.Close();
  513. }
  514. public void Write(RavenJObject result)
  515. {
  516. result.WriteTo(writer, Default.Converters);
  517. writer.WriteRaw(Environment.NewLine);
  518. }
  519. public void WriteError(Exception exception)
  520. {
  521. closedArray = true;
  522. writer.WriteEndArray();
  523. writer.WritePropertyName("Error");
  524. writer.WriteValue(exception.ToString());
  525. }
  526. }
  527. }
  528. }