PageRenderTime 22ms CodeModel.GetById 2ms app.highlight 15ms RepoModel.GetById 1ms app.codeStats 0ms

/ToMigrate/Raven.Database/Actions/QueryActions.cs

Relevant Search: With Applications for Solr and Elasticsearch

'Chapter 4. Taming tokens'. If you want to know how to extract ideas rather than words this book is for you. Learn concepts of precision and recall, making trade-offs between them and controlling the specificity of matches. Amazon Affiliate Link
http://github.com/ayende/ravendb
C# | 414 lines | 355 code | 53 blank | 6 comment | 77 complexity | 9c7fe721b1fbc4aec08bb9f85e77e892 MD5 | raw file
  1// -----------------------------------------------------------------------
  2//  <copyright file="QueryActions.cs" company="Hibernating Rhinos LTD">
  3//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
  4//  </copyright>
  5// -----------------------------------------------------------------------
  6using System;
  7using System.Collections.Generic;
  8using System.Diagnostics;
  9using System.Linq;
 10using System.Threading;
 11using Raven.Abstractions;
 12using Raven.Abstractions.Data;
 13using Raven.Abstractions.Exceptions;
 14using Raven.Abstractions.Extensions;
 15using Raven.Abstractions.Logging;
 16using Raven.Database.Data;
 17using Raven.Database.Impl;
 18using Raven.Database.Indexing;
 19using Raven.Database.Linq;
 20using Raven.Database.FileSystem.Extensions;
 21using Raven.Database.Storage;
 22using Raven.Database.Util;
 23using Raven.Json.Linq;
 24using Sparrow.Collections;
 25
 26namespace Raven.Database.Actions
 27{
 28    public class QueryActions : ActionsBase
 29    {
 30        public QueryActions(DocumentDatabase database, SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo> recentTouches, IUuidGenerator uuidGenerator, ILog log)
 31            : base(database, recentTouches, uuidGenerator, log)
 32        {
 33        }
 34
 35        public HashSet<string> QueryDocumentIds(string index, IndexQuery query, CancellationTokenSource tokenSource, out bool stale)
 36        {
 37            var queryStat = AddToCurrentlyRunningQueryList(index, query, tokenSource);
 38            try
 39            {
 40                bool isStale = false;
 41                HashSet<string> loadedIds = null;
 42                TransactionalStorage.Batch(
 43                    actions =>
 44                    {
 45                        var definition = IndexDefinitionStorage.GetIndexDefinition(index);
 46                        if (definition == null)
 47                            throw new ArgumentException("specified index definition was not found", "index");
 48
 49                        isStale = actions.Staleness.IsIndexStale(definition.IndexId, query.Cutoff, null);
 50
 51                        if (isStale == false && query.Cutoff == null)
 52                        {
 53                            var indexInstance = Database.IndexStorage.GetIndexInstance(index);
 54                            isStale = isStale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
 55                        }
 56
 57                        if (isStale && actions.Staleness.IsIndexStaleByTask(definition.IndexId, query.Cutoff) == false &&
 58                            actions.Staleness.IsReduceStale(definition.IndexId) == false)
 59                        {
 60                            var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
 61                            if (viewGenerator == null)
 62                                throw new ArgumentException("specified index definition was not found", "index");
 63
 64                            var forEntityNames = viewGenerator.ForEntityNames.ToList();
 65                            var lastIndexedEtag = actions.Indexing.GetIndexStats(definition.IndexId).LastIndexedEtag;
 66
 67                            if (Database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
 68                                isStale = false;
 69                        }
 70
 71                        var indexFailureInformation = actions.Indexing.GetFailureRate(definition.IndexId);
 72
 73                        if (indexFailureInformation.IsInvalidIndex)
 74                        {
 75                            throw new IndexDisabledException(indexFailureInformation);
 76                        }
 77                        loadedIds = new HashSet<string>(from queryResult in Database.IndexStorage.Query(index, query, result => true, new FieldsToFetch(null, false, Constants.DocumentIdFieldName), Database.IndexQueryTriggers, tokenSource.Token)
 78                                                        select queryResult.Key);
 79                    });
 80                stale = isStale;
 81                return loadedIds;
 82            }
 83            finally
 84            {
 85                RemoveFromCurrentlyRunningQueryList(index, queryStat);
 86            }
 87        }
 88
 89
 90        public QueryResultWithIncludes Query(string index, IndexQuery query, CancellationToken externalCancellationToken)
 91        {
 92            DateTime? showTimingByDefaultUntil = WorkContext.ShowTimingByDefaultUntil;
 93            if (showTimingByDefaultUntil != null)
 94            {
 95                if (showTimingByDefaultUntil < SystemTime.UtcNow) // expired, reset
 96                {
 97                    WorkContext.ShowTimingByDefaultUntil = null;
 98                }
 99                else
100                {
101                    query.ShowTimings = true;
102                }
103            }
104
105            QueryResultWithIncludes result = null;
106            using (var cts = CancellationTokenSource.CreateLinkedTokenSource(externalCancellationToken, WorkContext.CancellationToken))
107            {
108                TransactionalStorage.Batch(
109                    accessor =>
110                    {
111                        using (var op = new DatabaseQueryOperation(Database, index, query, accessor, cts)
112                        {
113                            ShouldSkipDuplicateChecking = query.SkipDuplicateChecking
114                        })
115                        {
116                            var list = new List<RavenJObject>();
117                            op.Init();
118                            op.Execute(list.Add);
119                            op.Result.Results = list;
120                            result = op.Result;
121                        }
122                    });
123            }
124
125            return result;
126        }
127
128        public class DatabaseQueryOperation : IDisposable
129        {
130            public bool ShouldSkipDuplicateChecking = false;
131            private readonly DocumentDatabase database;
132            private readonly string indexName;
133            private readonly IndexQuery query;
134            private readonly IStorageActionsAccessor actions;
135
136            private readonly CancellationToken cancellationToken;
137
138            private readonly ExecutingQueryInfo queryStat;
139            public QueryResultWithIncludes Result = new QueryResultWithIncludes();
140            public QueryHeaderInformation Header;
141            private bool stale;
142            private IEnumerable<RavenJObject> results;
143            private DocumentRetriever docRetriever;
144            internal DocumentRetriever DocRetriever
145            {
146                get { return docRetriever; }
147            }
148            private Stopwatch duration;
149            private List<string> transformerErrors;
150            private bool nonAuthoritativeInformation;
151            private Etag resultEtag;
152            private Tuple<DateTime, Etag> indexTimestamp;
153            private Dictionary<string, Dictionary<string, string[]>> highlightings;
154            private Dictionary<string, string> scoreExplanations;
155            private HashSet<string> idsToLoad;
156
157            private readonly ILog logger = LogManager.GetCurrentClassLogger();
158
159            private readonly Dictionary<QueryTimings, double> executionTimes = new Dictionary<QueryTimings, double>();
160
161            public DocumentDatabase Database
162            {
163                get { return database; }
164            }
165            public DatabaseQueryOperation(DocumentDatabase database, string indexName, IndexQuery query, IStorageActionsAccessor actions, CancellationTokenSource cancellationTokenSource)
166            {
167                this.database = database;
168                this.indexName = indexName != null ? indexName.Trim() : null;
169                this.query = query;
170                this.actions = actions;
171                cancellationToken = cancellationTokenSource.Token;
172                queryStat = database.Queries.AddToCurrentlyRunningQueryList(indexName, query, cancellationTokenSource);
173
174                if (query.ShowTimings)
175                {
176                    executionTimes[QueryTimings.Lucene] = 0;
177                    executionTimes[QueryTimings.LoadDocuments] = 0;
178                    executionTimes[QueryTimings.TransformResults] = 0;
179                }
180            }
181
182            public void Init()
183            {
184                highlightings = new Dictionary<string, Dictionary<string, string[]>>();
185                scoreExplanations = new Dictionary<string, string>();
186                Func<IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult =>
187                {
188                  
189                    if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null))
190                        highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings);
191                    if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null)
192                        scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation);
193                    return null;
194                };
195                stale = false;
196                indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
197                resultEtag = Etag.Empty;
198                nonAuthoritativeInformation = false;
199
200                if (string.IsNullOrEmpty(query.ResultsTransformer) == false &&
201                    (query.FieldsToFetch == null || query.FieldsToFetch.Length == 0))
202                {
203                    query.FieldsToFetch = new[] { Constants.AllFields };
204                }
205
206                duration = Stopwatch.StartNew();
207                idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
208
209                var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
210                var index = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
211                if (viewGenerator == null)
212                    throw new IndexDoesNotExistsException("Could not find index named: " + indexName);
213
214                resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer);
215
216                stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag);
217
218                if (stale == false && query.Cutoff == null && query.CutoffEtag == null)
219                {
220                    var indexInstance = database.IndexStorage.GetIndexInstance(indexName);
221                    stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
222                }
223
224                if (stale &&
225                    actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false &&
226                    actions.Staleness.IsReduceStale(index.IndexId) == false)
227                {
228                    var forEntityNames = viewGenerator.ForEntityNames.ToList();
229                    var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag;
230
231                    if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
232                        stale = false;
233                }
234
235                indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId);
236                var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId);
237                if (indexFailureInformation.IsInvalidIndex)
238                {
239                    throw new IndexDisabledException(indexFailureInformation);
240                }
241                docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad);
242                var fieldsToFetch = new FieldsToFetch(query,
243                    viewGenerator.ReduceDefinition == null
244                        ? Constants.DocumentIdFieldName
245                        : Constants.ReduceKeyFieldName);
246                Func<IndexQueryResult, bool> shouldIncludeInResults =
247                    result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking);
248                var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults,
249                    fieldsToFetch, database.IndexQueryTriggers, cancellationToken, (query.ShowTimings ? (Action<double>)(time => executionTimes[QueryTimings.Parse] = time) : null));
250                if (query.ShowTimings)
251                {
252                    indexQueryResults = new TimedEnumerable<IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] += timeInMilliseconds);
253                } 
254                indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);
255               
256
257                var docs = from queryResult in indexQueryResults
258                           let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking)
259                           where doc != null
260                           let __ = tryRecordHighlightingAndScoreExplanation(queryResult)
261                           select doc;
262
263                transformerErrors = new List<string>();
264                results = database
265                    .Queries
266                    .GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors,
267                        timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments] = timeInMilliseconds,
268                        timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds,
269                        query.ShowTimings, cancellationToken);
270
271                Header = new QueryHeaderInformation
272                {
273                    Index = indexName,
274                    IsStale = stale,
275                    ResultEtag = resultEtag,
276                    IndexTimestamp = indexTimestamp.Item1,
277                    IndexEtag = indexTimestamp.Item2,
278                    TotalResults = query.TotalSize.Value
279                };
280            }
281
282            public void Execute(Action<RavenJObject> onResult)
283            {
284                using (new CurrentTransformationScope(database, docRetriever))
285                {
286                    foreach (var result in results)
287                    {
288                        cancellationToken.ThrowIfCancellationRequested();
289                        database.WorkContext.UpdateFoundWork();
290
291                        onResult(result);
292                    }
293                    if (transformerErrors.Count > 0)
294                    {
295                        logger.Error("The transform results function failed.\r\n{0}", string.Join("\r\n", transformerErrors));
296                        throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
297                    }
298                }
299
300                Result = new QueryResultWithIncludes
301                {
302                    IndexName = indexName,
303                    IsStale = stale,
304                    SkippedResults = query.SkippedResults.Value,
305                    TotalResults = query.TotalSize.Value,
306                    IndexTimestamp = indexTimestamp.Item1,
307                    IndexEtag = indexTimestamp.Item2,
308                    ResultEtag = resultEtag,
309                    IdsToInclude = idsToLoad,
310                    LastQueryTime = SystemTime.UtcNow,
311                    Highlightings = highlightings,
312                    DurationMilliseconds = duration.ElapsedMilliseconds,
313                    ScoreExplanations = scoreExplanations,
314                    TimingsInMilliseconds = NormalizeTimings()
315                };
316            }
317
318            private Dictionary<string, double> NormalizeTimings()
319            {
320                if (query.ShowTimings)
321                {
322                    var luceneTime = executionTimes[QueryTimings.Lucene];
323                    var loadDocumentsTime = executionTimes[QueryTimings.LoadDocuments];
324                    var transformResultsTime = executionTimes[QueryTimings.TransformResults];
325
326                    executionTimes[QueryTimings.LoadDocuments] -= loadDocumentsTime > 0 ? luceneTime : 0;
327                    executionTimes[QueryTimings.TransformResults] -= transformResultsTime > 0 ? loadDocumentsTime + luceneTime : 0;
328                }
329
330                return executionTimes.ToDictionary(x => x.Key.GetDescription(), x => x.Value >= 0 ? Math.Round(x.Value) : 0);
331            }
332
333            public void Dispose()
334            {
335                database.Queries.RemoveFromCurrentlyRunningQueryList(indexName, queryStat);
336                var resultsAsDisposable = results as IDisposable;
337                if (resultsAsDisposable != null)
338                    resultsAsDisposable.Dispose();
339            }
340        }
341
342        private void RemoveFromCurrentlyRunningQueryList(string index, ExecutingQueryInfo queryStat)
343        {
344            ConcurrentSet<ExecutingQueryInfo> set;
345            if (WorkContext.CurrentlyRunningQueries.TryGetValue(index, out set) == false)
346                return;
347
348            set.TryRemove(queryStat);
349        }
350
351        private ExecutingQueryInfo AddToCurrentlyRunningQueryList(string index, IndexQuery query, CancellationTokenSource externalTokenSource)
352        {
353            var set = WorkContext.CurrentlyRunningQueries.GetOrAdd(index, x => new ConcurrentSet<ExecutingQueryInfo>());
354            var queryStartTime = DateTime.UtcNow;
355            var queryId = WorkContext.GetNextQueryId();
356            var executingQueryInfo = new ExecutingQueryInfo(queryStartTime, query, queryId, externalTokenSource);
357            set.Add(executingQueryInfo);
358            return executingQueryInfo;
359        }
360
361        private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors, Action<double> loadingDocumentsFinish, Action<double> transformerFinish, bool showTimings, CancellationToken token)
362        {
363            if (query.PageSize <= 0) // maybe they just want the stats? 
364            {
365                return Enumerable.Empty<RavenJObject>();
366            }
367
368            IndexingFunc transformFunc = null;
369
370            // Check an explicitly declared one first
371            if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
372            {
373                var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
374
375                if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
376                    transformFunc = transformGenerator.TransformResultsDefinition;
377                else
378                    throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
379            }
380
381            if (transformFunc == null)
382            {
383                var resultsWithoutTransformer = results.Select(x =>
384                {
385                    var ravenJObject = x.ToJson();
386                    if (query.IsDistinct)
387                    {
388                        ravenJObject[Constants.DocumentIdFieldName] = x.Key;
389                    }
390                    return ravenJObject;
391                });
392                return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer;
393            }
394
395            var dynamicJsonObjects = results.Select(x =>
396            {
397                var ravenJObject = x.ToJson();
398                if (query.IsDistinct)
399                {
400                    ravenJObject[Constants.DocumentIdFieldName] = x.Key;
401                }
402                return new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject);
403            });
404            var robustEnumerator = new RobustEnumerator(token, 100, 
405                onError: (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)));
406
407            var resultsWithTransformer = robustEnumerator
408                .RobustEnumeration(dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc)
409                .Select(JsonExtensions.ToJObject);
410
411            return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer;
412        }
413    }
414}