PageRenderTime 399ms CodeModel.GetById 161ms app.highlight 84ms RepoModel.GetById 147ms app.codeStats 0ms

/ToMigrate/Raven.Database/Server/Controllers/StreamsController.cs

http://github.com/ayende/ravendb
C# | 605 lines | 529 code | 67 blank | 9 comment | 40 complexity | f5bf8d93f9d60e0bad0a0a179adfb1c1 MD5 | raw file
  1using System;
  2using System.Collections.Generic;
  3using System.Collections.Specialized;
  4using System.Diagnostics;
  5using System.Globalization;
  6using System.IO;
  7using System.Linq;
  8using System.Net;
  9using System.Net.Http;
 10using System.Net.Http.Headers;
 11using System.Security.Principal;
 12using System.Text;
 13using System.Threading;
 14using System.Threading.Tasks;
 15using System.Web.Http;
 16
 17using Raven.Abstractions;
 18using Raven.Abstractions.Data;
 19using Raven.Abstractions.Extensions;
 20using Raven.Abstractions.Indexing;
 21using Raven.Abstractions.Linq;
 22using Raven.Abstractions.Util;
 23using Raven.Abstractions.Util.Encryptors;
 24using Raven.Database.Actions;
 25using Raven.Database.Extensions;
 26using Raven.Database.Impl;
 27using Raven.Database.Server.WebApi.Attributes;
 28using Raven.Database.Storage;
 29using Raven.Imports.Newtonsoft.Json;
 30using Raven.Imports.Newtonsoft.Json.Linq;
 31using Raven.Json.Linq;
 32
 33using Sparrow;
 34
 35namespace Raven.Database.Server.Controllers
 36{
 37    public class StreamsController : ClusterAwareRavenDbApiController
 38    {
 39        [HttpHead]
 40        [RavenRoute("streams/docs")]
 41        [RavenRoute("databases/{databaseName}/streams/docs")]
 42        public HttpResponseMessage StreamDocsHead()
 43        {
 44            return GetEmptyMessage();
 45        }
 46
 47        [HttpGet]
 48        [RavenRoute("streams/docs")]
 49        [RavenRoute("databases/{databaseName}/streams/docs")]
 50        public HttpResponseMessage StreamDocsGet()
 51        {
 52            var start = GetStart();
 53            var etag = GetEtagFromQueryString();
 54            var startsWith = GetQueryStringValue("startsWith");
 55            var pageSize = GetPageSize(int.MaxValue);
 56            var matches = GetQueryStringValue("matches");
 57            var nextPageStart = GetNextPageStart();
 58            if (string.IsNullOrEmpty(GetQueryStringValue("pageSize")))
 59                pageSize = int.MaxValue;
 60
 61            var skipAfter = GetQueryStringValue("skipAfter");
 62            var transformer = GetQueryStringValue("transformer");
 63            var transformerParameters = ExtractTransformerParameters();
 64
 65            var headers = CurrentOperationContext.Headers.Value;
 66            var user = CurrentOperationContext.User.Value;
 67            return new HttpResponseMessage(HttpStatusCode.OK)
 68            {
 69                Content = new PushStreamContent((stream, content, transportContext) =>
 70                    StreamToClient(stream, startsWith, start, pageSize, etag, matches, nextPageStart, skipAfter, transformer, transformerParameters, headers, user))
 71                {
 72                    Headers =
 73                    {
 74                        ContentType = new MediaTypeHeaderValue("application/json") { CharSet = "utf-8" }
 75                    }
 76                }
 77            };
 78        }
 79
 80        private void StreamToClient(Stream stream, string startsWith, int start, int pageSize, Etag etag, string matches, int nextPageStart, string skipAfter, string transformer, Dictionary<string, RavenJToken> transformerParameters,
 81            Lazy<NameValueCollection> headers, IPrincipal user)
 82        {
 83            var old = CurrentOperationContext.Headers.Value;
 84            var oldUser = CurrentOperationContext.User.Value;
 85            try
 86            {
 87                CurrentOperationContext.Headers.Value = headers;
 88                CurrentOperationContext.User.Value = user;
 89
 90
 91            var bufferStream = new BufferedStream(stream, 1024 * 64);
 92            using (var cts = new CancellationTokenSource())
 93            using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan))
 94            using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
 95            {
 96                writer.WriteStartObject();
 97                writer.WritePropertyName("Results");
 98                writer.WriteStartArray();
 99
100                Action<JsonDocument> addDocument = doc =>
101                {
102                    timeout.Delay();
103                    if (doc == null)
104                    {
105                        // we only have this heartbit when the streaming has gone on for a long time
106                        // and we haven't send anything to the user in a while (because of filtering, skipping, etc).
107                        writer.WriteRaw(Environment.NewLine);
108                        writer.Flush();
109                        return;
110                    }
111                    doc.ToJson().WriteTo(writer);
112                    writer.WriteRaw(Environment.NewLine);
113                };
114
115                Database.TransactionalStorage.Batch(accessor =>
116                {
117                    // we may be sending a LOT of documents to the user, and most 
118                    // of them aren't going to be relevant for other ops, so we are going to skip
119                    // the cache for that, to avoid filling it up very quickly
120                    using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
121                    {
122                        if (string.IsNullOrEmpty(startsWith))
123                        {
124                            Database.Documents.GetDocuments(start, pageSize, etag, cts.Token, doc => { addDocument(doc); return true; }, transformer, transformerParameters);
125                        }
126                        else
127                        {
128                            var nextPageStartInternal = nextPageStart;
129
130                            Database.Documents.GetDocumentsWithIdStartingWith(startsWith, matches, null, start, pageSize, cts.Token, ref nextPageStartInternal, addDocument, transformer, transformerParameters, skipAfter);
131
132                            nextPageStart = nextPageStartInternal;
133                        }
134                    }
135                });
136
137                writer.WriteEndArray();
138                writer.WritePropertyName("NextPageStart");
139                writer.WriteValue(nextPageStart);
140                writer.WriteEndObject();
141                writer.Flush();
142                bufferStream.Flush();
143            }
144        }
145            finally
146            {
147                CurrentOperationContext.Headers.Value = old;
148                CurrentOperationContext.User.Value = oldUser;
149            }
150        }
151
152        [HttpHead]
153        [RavenRoute("streams/query/{*id}")]
154        [RavenRoute("databases/{databaseName}/streams/query/{*id}")]
155        public HttpResponseMessage SteamQueryHead(string id)
156        {
157            return GetEmptyMessage();
158        }
159
160        [HttpGet]
161        [RavenRoute("streams/query/{*id}")]
162        [RavenRoute("databases/{databaseName}/streams/query/{*id}")]
163        public HttpResponseMessage SteamQueryGet(string id)
164        {
165            var cts = new CancellationTokenSource();
166            var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan);
167            var msg = GetEmptyMessage();
168
169            var index = id;
170            var query = GetIndexQuery(int.MaxValue);
171            if (string.IsNullOrEmpty(GetQueryStringValue("pageSize"))) query.PageSize = int.MaxValue;
172            var isHeadRequest = InnerRequest.Method == HttpMethods.Head;
173            if (isHeadRequest) query.PageSize = 0;
174
175            var accessor = Database.TransactionalStorage.CreateAccessor(); //accessor will be disposed in the StreamQueryContent.SerializeToStreamAsync!
176
177            try
178            {
179                var queryOp = new QueryActions.DatabaseQueryOperation(Database, index, query, accessor, cts);
180                queryOp.Init();
181
182                msg.Content = new StreamQueryContent(InnerRequest, queryOp, accessor, timeout, mediaType => msg.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType) {CharSet = "utf-8"});
183                msg.Headers.Add("Raven-Result-Etag", queryOp.Header.ResultEtag.ToString());
184                msg.Headers.Add("Raven-Index-Etag", queryOp.Header.IndexEtag.ToString());
185                msg.Headers.Add("Raven-Is-Stale", queryOp.Header.IsStale ? "true" : "false");
186                msg.Headers.Add("Raven-Index", queryOp.Header.Index);
187                msg.Headers.Add("Raven-Total-Results", queryOp.Header.TotalResults.ToString(CultureInfo.InvariantCulture));
188                msg.Headers.Add("Raven-Index-Timestamp", queryOp.Header.IndexTimestamp.GetDefaultRavenFormat());
189
190                if (IsCsvDownloadRequest(InnerRequest))
191                {
192                    msg.Content.Headers.Add("Content-Disposition", "attachment; filename=export.csv");
193                }
194            }
195            catch (OperationCanceledException e)
196            {
197                accessor.Dispose();
198                throw new TimeoutException(string.Format("The query did not produce results in {0}", DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan), e);
199            }
200            catch (Exception)
201            {
202                accessor.Dispose();
203                throw;
204            }
205
206            return msg;
207        }
208
209        [HttpGet]
210        [RavenRoute("streams/exploration")]
211        [RavenRoute("databases/{databaseName}/streams/exploration")]
212        public Task<HttpResponseMessage> Exploration()
213        {
214            var linq = GetQueryStringValue("linq");
215            var collection = GetQueryStringValue("collection");
216            int timeoutSeconds;
217            if (int.TryParse(GetQueryStringValue("timeoutSeconds"), out timeoutSeconds) == false)
218                timeoutSeconds = 60;
219            int pageSize;
220            if (int.TryParse(GetQueryStringValue("pageSize"), out pageSize) == false)
221                pageSize = 100000;
222
223
224            var hash = Hashing.XXHash64.CalculateRaw(linq);
225            var sourceHashed = hash.ToString("X");
226            var transformerName = Constants.TemporaryTransformerPrefix + sourceHashed;
227
228            var transformerDefinition = Database.IndexDefinitionStorage.GetTransformerDefinition(transformerName);
229            if (transformerDefinition == null)
230            {
231                transformerDefinition = new TransformerDefinition
232                {
233                    Name = transformerName,
234                    Temporary = true,
235                    TransformResults = linq
236                };
237                Database.Transformers.PutTransform(transformerName, transformerDefinition);
238            }
239
240            var msg = GetEmptyMessage();
241
242            using (var cts = new CancellationTokenSource())
243            {
244                var timeout = cts.TimeoutAfter(TimeSpan.FromSeconds(timeoutSeconds));
245                var indexQuery = new IndexQuery
246                {
247                    PageSize = pageSize,
248                    Start = 0,
249                    Query = "Tag:" + collection,
250                    ResultsTransformer = transformerName
251                };
252
253                var accessor = Database.TransactionalStorage.CreateAccessor(); //accessor will be disposed in the StreamQueryContent.SerializeToStreamAsync!
254
255                try
256                {
257                    var queryOp = new QueryActions.DatabaseQueryOperation(Database, "Raven/DocumentsByEntityName", indexQuery, accessor, cts);
258                    queryOp.Init();
259
260                    msg.Content = new StreamQueryContent(InnerRequest, queryOp, accessor, timeout, mediaType => msg.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType) { CharSet = "utf-8" },
261                        o =>
262                        {
263                            if (o.Count == 2 &&
264                                o.ContainsKey(Constants.DocumentIdFieldName) &&
265                                    o.ContainsKey(Constants.Metadata))
266                            {
267                                // this is the raw value out of the server, we don't want to get that
268                                var doc = queryOp.DocRetriever.Load(o.Value<string>(Constants.DocumentIdFieldName));
269                                var djo = doc as IDynamicJsonObject;
270                                if (djo != null)
271                                    return djo.Inner;
272                            }
273                            return o;
274                        });
275                    msg.Headers.Add("Raven-Result-Etag", queryOp.Header.ResultEtag.ToString());
276                    msg.Headers.Add("Raven-Index-Etag", queryOp.Header.IndexEtag.ToString());
277                    msg.Headers.Add("Raven-Is-Stale", queryOp.Header.IsStale ? "true" : "false");
278                    msg.Headers.Add("Raven-Index", queryOp.Header.Index);
279                    msg.Headers.Add("Raven-Total-Results", queryOp.Header.TotalResults.ToString(CultureInfo.InvariantCulture));
280                    msg.Headers.Add("Raven-Index-Timestamp", queryOp.Header.IndexTimestamp.GetDefaultRavenFormat());
281
282                    if (IsCsvDownloadRequest(InnerRequest))
283                    {
284                        msg.Content.Headers.Add("Content-Disposition", "attachment; filename=export.csv");
285                    }
286                }
287                catch (Exception)
288                {
289                    accessor.Dispose();
290                    throw;
291                }
292
293                return new CompletedTask<HttpResponseMessage>(msg);
294            }
295        }
296
297        [HttpPost]
298        [RavenRoute("streams/query/{*id}")]
299        [RavenRoute("databases/{databaseName}/streams/query/{*id}")]
300        public async Task<HttpResponseMessage> SteamQueryPost(string id)
301        {
302            var postedQuery = await ReadStringAsync().ConfigureAwait(false);
303
304            SetPostRequestQuery(postedQuery);
305
306            return SteamQueryGet(id);
307        }
308
309        public class StreamQueryContent : HttpContent
310        {
311            private readonly HttpRequestMessage req;
312            private readonly QueryActions.DatabaseQueryOperation queryOp;
313            private readonly IStorageActionsAccessor accessor;
314            private readonly CancellationTimeout _timeout;
315            private readonly Action<string> outputContentTypeSetter;
316            private Lazy<NameValueCollection> headers;
317            private IPrincipal user;
318            private readonly Func<RavenJObject, RavenJObject> modifyDocument;
319
320            [CLSCompliant(false)]
321            public StreamQueryContent(HttpRequestMessage req, QueryActions.DatabaseQueryOperation queryOp, IStorageActionsAccessor accessor,
322                CancellationTimeout timeout,
323                Action<string> contentTypeSetter,
324                Func<RavenJObject,RavenJObject> modifyDocument = null)
325            {
326                headers = CurrentOperationContext.Headers.Value;
327                user = CurrentOperationContext.User.Value;
328                this.req = req;
329                this.queryOp = queryOp;
330                this.accessor = accessor;
331                _timeout = timeout;
332                outputContentTypeSetter = contentTypeSetter;
333                this.modifyDocument = modifyDocument;
334            }
335
336            protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
337            {
338                var old = CurrentOperationContext.Headers.Value;
339                var oldUser = CurrentOperationContext.User.Value;
340                try
341                {
342                    CurrentOperationContext.User.Value = user;
343                    CurrentOperationContext.Headers.Value = headers;
344                var bufferSize = queryOp.Header.TotalResults > 1024 ? 1024 * 64 : 1024 * 8;
345                using (var bufferedStream = new BufferedStream(stream, bufferSize))
346                using (queryOp)
347                using (accessor)
348                using (_timeout)
349                using (var writer = GetOutputWriter(req, bufferedStream))
350                // we may be sending a LOT of documents to the user, and most 
351                // of them aren't going to be relevant for other ops, so we are going to skip
352                // the cache for that, to avoid filling it up very quickly
353                using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
354                {
355                    outputContentTypeSetter(writer.ContentType);
356
357                    writer.WriteHeader();
358                    try
359                    {
360                        queryOp.Execute(o =>
361                        {
362                            _timeout.Delay();
363                            if (modifyDocument != null)
364                                o = modifyDocument(o);
365                            writer.Write(o);
366                        });
367                    }
368                    catch (Exception e)
369                    {
370                        writer.WriteError(e);
371                    }
372                }
373                return Task.FromResult(true);
374            }
375                finally
376                {
377                    CurrentOperationContext.Headers.Value = old;
378                    CurrentOperationContext.User.Value = oldUser;
379                }
380            }
381
382            protected override bool TryComputeLength(out long length)
383            {
384                length = -1;
385                return false;
386            }
387        }
388
389        private static IOutputWriter GetOutputWriter(HttpRequestMessage req, Stream stream)
390        {
391            var useExcelFormat = "excel".Equals(GetQueryStringValue(req, "format"), StringComparison.InvariantCultureIgnoreCase);
392            return useExcelFormat ? (IOutputWriter)new ExcelOutputWriter(stream) : new JsonOutputWriter(stream);
393        }
394
395        private static Boolean IsCsvDownloadRequest(HttpRequestMessage req)
396        {
397            return "true".Equals(GetQueryStringValue(req, "download"), StringComparison.InvariantCultureIgnoreCase)
398                && "excel".Equals(GetQueryStringValue(req, "format"), StringComparison.InvariantCultureIgnoreCase);
399        }
400
401
402        public interface IOutputWriter : IDisposable
403        {
404            string ContentType { get; }
405
406            void WriteHeader();
407            void Write(RavenJObject result);
408            void WriteError(Exception exception);
409        }
410
411        private class ExcelOutputWriter : IOutputWriter
412        {
413            private const string CsvContentType = "text/csv";
414
415            private readonly Stream stream;
416            private StreamWriter writer;
417            private bool doIncludeId;
418
419            public ExcelOutputWriter(Stream stream)
420            {
421                this.stream = stream;
422            }
423
424            public string ContentType
425            {
426                get { return CsvContentType; }
427            }
428
429            public void Dispose()
430            {
431                if (writer == null)
432                    return;
433
434                writer.Flush();
435                stream.Flush();
436                writer.Close();
437            }
438
439            public void WriteHeader()
440            {
441                writer = new StreamWriter(stream, Encoding.UTF8);
442            }
443
444            public void Write(RavenJObject result)
445            {
446                if (properties == null)
447                {
448                    GetPropertiesAndWriteCsvHeader(result, out doIncludeId);
449                    Debug.Assert(properties != null);
450                }
451
452                if (doIncludeId)
453                {
454                    RavenJToken token;
455                    if (result.TryGetValue("@metadata", out token))
456                    {
457                        var metadata = token as RavenJObject;
458                        if (metadata != null)
459                        {
460                            if (metadata.TryGetValue("@id", out token))
461                            {
462                                OutputCsvValue(token.Value<string>());
463                            }
464                            writer.Write(',');
465                        }
466                    }
467                }
468
469                foreach (var property in properties)
470                {
471                    var token = result.SelectToken(property);
472                    if (token != null)
473                    {
474                        switch (token.Type)
475                        {
476                            case JTokenType.Null:
477                                break;
478
479                            case JTokenType.Array:
480                            case JTokenType.Object:
481                                OutputCsvValue(token.ToString(Formatting.None));
482                                break;
483
484                            default:
485                                OutputCsvValue(token.Value<string>());
486                                break;
487                        }
488                    }
489
490                    writer.Write(',');
491                }
492
493                writer.WriteLine();
494            }
495
496            public void WriteError(Exception exception)
497            {
498                writer.WriteLine();
499                writer.WriteLine();
500                writer.WriteLine(exception.ToString());
501            }
502
503            private void GetPropertiesAndWriteCsvHeader(RavenJObject result, out bool includeId)
504            {
505                includeId = false;
506                properties = DocumentHelpers.GetPropertiesFromJObject(result,
507                    parentPropertyPath: "",
508                    includeNestedProperties: true,
509                    includeMetadata: false,
510                    excludeParentPropertyNames: true).ToList();
511
512                RavenJToken token;
513                if (result.TryGetValue("@metadata", out token))
514                {
515                    var metadata = token as RavenJObject;
516                    if (metadata != null)
517                    {
518                        if (metadata.TryGetValue("@id", out token))
519                        {
520                            OutputCsvValue("@id");
521                            writer.Write(',');
522
523                            includeId = true;
524                        }
525                    }
526                }
527
528                foreach (var property in properties)
529                {
530                    OutputCsvValue(property);
531                    writer.Write(',');
532                }
533                writer.WriteLine();
534            }
535
536            private static readonly char[] RequireQuotesChars = { ',', '\r', '\n', '"' };
537            private IEnumerable<string> properties;
538
539            private void OutputCsvValue(string val)
540            {
541                var needsQuoutes = val.IndexOfAny(RequireQuotesChars) != -1;
542                if (needsQuoutes)
543                    writer.Write('"');
544
545                writer.Write(needsQuoutes ? val.Replace("\"", "\"\"") : val);
546                if (needsQuoutes)
547                    writer.Write('"');
548            }
549
550        }
551
552        public class JsonOutputWriter : IOutputWriter
553        {
554            private const string JsonContentType = "application/json";
555            private readonly Stream stream;
556            private JsonWriter writer;
557            private bool closedArray = false;
558
559            public JsonOutputWriter(Stream stream)
560            {
561                this.stream = stream;
562            }
563
564            public string ContentType
565            {
566                get { return JsonContentType; }
567            }
568
569            public void WriteHeader()
570            {
571                writer = new JsonTextWriter(new StreamWriter(stream));
572                writer.WriteStartObject();
573                writer.WritePropertyName("Results");
574                writer.WriteStartArray();
575            }
576
577            public void Dispose()
578            {
579                if (writer == null)
580                    return;
581                if (closedArray == false)
582                    writer.WriteEndArray();
583                writer.WriteEndObject();
584
585                writer.Flush();
586                stream.Flush();
587                writer.Close();
588            }
589
590            public void Write(RavenJObject result)
591            {
592                result.WriteTo(writer, Default.Converters);
593                writer.WriteRaw(Environment.NewLine);
594            }
595
596            public void WriteError(Exception exception)
597            {
598                closedArray = true;
599                writer.WriteEndArray();
600                writer.WritePropertyName("Error");
601                writer.WriteValue(exception.ToString());
602            }
603        }
604    }
605}