PageRenderTime 317ms CodeModel.GetById 141ms app.highlight 76ms RepoModel.GetById 95ms app.codeStats 0ms

/ToMigrate/Raven.Database/Impl/DocumentRetriever.cs

http://github.com/ayende/ravendb
C# | 407 lines | 346 code | 48 blank | 13 comment | 88 complexity | 2a754969a26e60b2654e976b3a8fc9a9 MD5 | raw file
  1//-----------------------------------------------------------------------
  2// <copyright file="DocumentRetriever.cs" company="Hibernating Rhinos LTD">
  3//     Copyright (c) Hibernating Rhinos LTD. All rights reserved.
  4// </copyright>
  5//-----------------------------------------------------------------------
  6using System;
  7using System.Collections;
  8using System.Collections.Generic;
  9using Raven.Abstractions.Exceptions;
 10using Raven.Abstractions.Logging;
 11using Raven.Database.Config;
 12using Raven.Imports.Newtonsoft.Json.Linq;
 13using Raven.Abstractions.Data;
 14using Raven.Abstractions.Indexing;
 15using Raven.Abstractions.Linq;
 16using Raven.Abstractions.MEF;
 17using Raven.Database.Data;
 18using Raven.Database.Indexing;
 19using Raven.Database.Linq;
 20using Raven.Database.Plugins;
 21using Raven.Database.Storage;
 22using Raven.Abstractions.Json;
 23using System.Linq;
 24using Raven.Json.Linq;
 25
 26namespace Raven.Database.Impl
 27{
 28    internal class DocumentRetriever : ITranslatorDatabaseAccessor
 29    {
 30        private static readonly ILog log = LogManager.GetCurrentClassLogger();
 31
 32        private readonly IDictionary<string, JsonDocument> cache = new Dictionary<string, JsonDocument>(StringComparer.OrdinalIgnoreCase);
 33        private readonly HashSet<string> loadedIdsForRetrieval = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
 34        private readonly HashSet<string> loadedIdsForFilter = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
 35        private readonly RavenConfiguration configuration;
 36        private readonly IStorageActionsAccessor actions;
 37        private readonly OrderedPartCollection<AbstractReadTrigger> triggers;
 38        private readonly Dictionary<string, RavenJToken> transformerParameters;
 39        private readonly HashSet<string> itemsToInclude;
 40        private bool disableCache;
 41
 42        public Etag Etag = Etag.Empty;
 43
 44        public DocumentRetriever(RavenConfiguration configuration, IStorageActionsAccessor actions, OrderedPartCollection<AbstractReadTrigger> triggers,
 45            Dictionary<string, RavenJToken> transformerParameters = null,
 46            HashSet<string> itemsToInclude = null)
 47        {
 48            this.configuration = configuration;
 49            this.actions = actions;
 50            this.triggers = triggers;
 51            this.transformerParameters = transformerParameters ?? new Dictionary<string, RavenJToken>();
 52            this.itemsToInclude = itemsToInclude ?? new HashSet<string>();
 53        }
 54
 55        public JsonDocument RetrieveDocumentForQuery(IndexQueryResult queryResult, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch, bool skipDuplicateCheck)
 56        {
 57            return ExecuteReadTriggers(
 58                        ProcessReadVetoes( 
 59                            RetrieveDocumentInternal(queryResult, loadedIdsForRetrieval, fieldsToFetch, indexDefinition, skipDuplicateCheck), ReadOperation.Query), ReadOperation.Query);
 60        }
 61
 62
 63        public JsonDocument ExecuteReadTriggers(JsonDocument document, ReadOperation operation)
 64        {
 65            return ExecuteReadTriggersOnRead(ProcessReadVetoes(document, operation), operation);
 66        }
 67
 68        private JsonDocument ExecuteReadTriggersOnRead(JsonDocument resultingDocument, ReadOperation operation)
 69        {
 70            if (resultingDocument == null)
 71                return null;
 72
 73            var doc = new JsonDocument
 74            {
 75                Key = resultingDocument.Key,
 76                Etag = resultingDocument.Etag,
 77                LastModified = resultingDocument.LastModified,
 78                SerializedSizeOnDisk = resultingDocument.SerializedSizeOnDisk,
 79                SkipDeleteFromIndex = resultingDocument.SkipDeleteFromIndex,  
 80                TempIndexScore = resultingDocument.TempIndexScore,
 81                DataAsJson = resultingDocument.DataAsJson.IsSnapshot
 82                                    ? (RavenJObject)resultingDocument.DataAsJson.CreateSnapshot()
 83                                    : resultingDocument.DataAsJson,
 84                Metadata = resultingDocument.Metadata.IsSnapshot
 85                                ? (RavenJObject)resultingDocument.Metadata.CreateSnapshot()
 86                                : resultingDocument.Metadata,
 87            };
 88
 89            triggers.Apply(
 90                trigger =>
 91                trigger.OnRead(doc.Key, doc.DataAsJson, doc.Metadata, operation));
 92
 93            return doc;
 94        }
 95
 96        private JsonDocument RetrieveDocumentInternal(
 97            IndexQueryResult queryResult,
 98            HashSet<string> loadedIds,
 99            FieldsToFetch fieldsToFetch,
100            IndexDefinition indexDefinition,
101            bool skipDuplicateCheck)
102        {
103            var queryScore = queryResult.Score;
104
105            if (float.IsNaN(queryScore))
106                queryScore = 0f;
107
108            if (queryResult.Projection == null)
109            {
110                // duplicate document, filter it out
111                if (skipDuplicateCheck == false && loadedIds.Add(queryResult.Key) == false)
112                    return null;
113                var document = GetDocumentWithCaching(queryResult);
114                if (document == null)
115                    return null;
116
117                document.Metadata = GetMetadata(document);
118
119                if (skipDuplicateCheck == false)
120                    document.Metadata[Constants.TemporaryScoreValue] = queryScore;
121
122                return document;
123            }
124
125            JsonDocument doc = null;
126
127            if (fieldsToFetch.IsProjection)
128            {
129                if (indexDefinition.IsMapReduce == false)
130                {
131                    bool hasStoredFields = false;
132                    FieldStorage value;
133                    if (indexDefinition.Stores.TryGetValue(Constants.AllFields, out value))
134                    {
135                        hasStoredFields = value != FieldStorage.No;
136                    }
137                    foreach (var fieldToFetch in fieldsToFetch.Fields)
138                    {
139                        if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false && value != FieldStorage.No) continue;
140                        hasStoredFields = true;
141                    }
142                    if (hasStoredFields == false)
143                    {
144                        // duplicate document, filter it out
145                        if (loadedIds.Add(queryResult.Key) == false) return null;
146                    }
147                }
148
149                // We have to load the document if user explicitly asked for the id, since 
150                // we normalize the casing for the document id on the index, and we need to return
151                // the id to the user with the same casing they gave us.
152                var fetchingId = fieldsToFetch.HasField(Constants.DocumentIdFieldName);
153                var fieldsToFetchFromDocument = fieldsToFetch.Fields.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null).ToArray();
154                if (fieldsToFetchFromDocument.Length > 0 || fetchingId)
155                {
156                    switch (configuration.Core.ImplicitFetchFieldsFromDocumentMode)
157                    {
158                        case ImplicitFetchFieldsMode.Enabled:
159                            doc = GetDocumentWithCaching(queryResult);
160                            if (doc != null)
161                            {
162                                if (fetchingId)
163                                {
164                                    queryResult.Projection[Constants.DocumentIdFieldName] = doc.Key;
165                                }
166
167                                var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
168                                foreach (var property in result)
169                                {
170                                    if (property.Value == null ) continue;
171
172                                    queryResult.Projection[property.Key] = property.Value;
173                                }
174                            }
175                            break;
176                        case ImplicitFetchFieldsMode.DoNothing:
177                            break;
178                        case ImplicitFetchFieldsMode.Exception:
179                            string message = string.Format("Implicit fetching of fields from the document is disabled." + Environment.NewLine +
180                                                  "Check your index ({0}) to make sure that all fields you want to project are stored in the index." + Environment.NewLine +
181                                                  "You can control this behavior using the Raven/ImplicitFetchFieldsFromDocumentMode setting." + Environment.NewLine +
182                                                  "Fields to fetch from document are: {1}" + Environment.NewLine +
183                                                  "Fetching id: {2}", indexDefinition.Name, string.Join(", ", fieldsToFetchFromDocument), fetchingId);
184                            throw new ImplicitFetchFieldsFromDocumentNotAllowedException(message);
185                        default:
186                            throw new ArgumentOutOfRangeException(configuration.Core.ImplicitFetchFieldsFromDocumentMode.ToString());
187                    }
188                }
189            }
190            else if (fieldsToFetch.FetchAllStoredFields && string.IsNullOrEmpty(queryResult.Key) == false
191                && (fieldsToFetch.Query == null || fieldsToFetch.Query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer == false)
192                )
193            {
194                // duplicate document, filter it out
195                if (loadedIds.Add(queryResult.Key) == false)
196                    return null;
197
198                doc = GetDocumentWithCaching(queryResult);
199            }
200
201            var metadata = GetMetadata(doc);
202            metadata.Remove("@id");
203            metadata[Constants.TemporaryScoreValue] = queryScore;
204            return new JsonDocument
205            {
206                Key = queryResult.Key,
207                DataAsJson = queryResult.Projection,
208                Metadata = metadata
209            };
210        }
211
212        private static RavenJObject GetMetadata(JsonDocument doc)
213        {
214            if (doc == null)
215                return new RavenJObject();
216
217            if (doc.Metadata.IsSnapshot)
218                return (RavenJObject)doc.Metadata.CreateSnapshot();
219
220            return doc.Metadata;
221        }
222
223        private JsonDocument GetDocumentWithCaching(IndexQueryResult iqr)
224        {
225            if (iqr.DocumentLoaded)
226                return iqr.Document;
227
228            iqr.DocumentLoaded = true;
229            iqr.Document = GetDocumentWithCaching(iqr.Key);
230            if (iqr.Document != null)
231                iqr.Key = iqr.Document.Key;// to get the actual document id in the right case sensitive manner
232            return iqr.Document;
233        }
234
235        private JsonDocument GetDocumentWithCaching(string key)
236        {
237            if (key == null)
238                return null;
239            JsonDocument doc;
240            if (disableCache == false && cache.TryGetValue(key, out doc))
241                return doc;
242
243            doc = actions.Documents.DocumentByKey(key);
244            JsonDocument.EnsureIdInMetadata(doc);
245
246            if (doc != null && doc.Metadata != null)
247                doc.Metadata.EnsureCannotBeChangeAndEnableSnapshotting();
248
249
250            if (disableCache == false)
251                cache[key] = doc;
252
253            if (cache.Count > 2048)
254            {
255                // we are probably doing a stream here, no point in trying to cache things, we might be
256                // going through the entire db here!
257                disableCache = true;
258                cache.Clear();
259            }
260            return doc;
261        }
262
263        public bool ShouldIncludeResultInQuery(IndexQueryResult arg, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch, bool skipDuplicateCheck)
264        {
265            JsonDocument doc;
266            if (arg.DocumentLoaded)
267            {
268                doc = arg.Document;
269            }
270            else
271            {
272                doc = RetrieveDocumentInternal(arg, loadedIdsForFilter, fieldsToFetch, indexDefinition, skipDuplicateCheck);
273                arg.Document = doc;
274                arg.DocumentLoaded = true;
275            }
276
277            if (doc == null)
278                return false;
279            doc = ProcessReadVetoes(doc, ReadOperation.Query);
280            return doc != null;
281        }
282
283        public T ProcessReadVetoes<T>(T document, ReadOperation operation)
284            where T : class, IJsonDocumentMetadata, new()
285        {
286            if (document == null)
287                return null;
288            foreach (var readTrigger in triggers)
289            {
290                var readVetoResult = readTrigger.Value.AllowRead(document.Key, document.Metadata, operation);
291                switch (readVetoResult.Veto)
292                {
293                    case ReadVetoResult.ReadAllow.Allow:
294                        break;
295                    case ReadVetoResult.ReadAllow.Deny:
296                        return new T
297                        {
298                            Etag = Etag.Empty,
299                            LastModified = DateTime.MinValue,
300                            Key = document.Key,
301                            Metadata = new RavenJObject
302                                                      {
303                                                          {
304                                                              "Raven-Read-Veto", new RavenJObject
305                                                                                     {
306                                                                                         {"Reason", readVetoResult.Reason},
307                                                                                         {"Trigger", readTrigger.ToString()}
308                                                                                     }
309                                                              }
310                                                      }
311                        };
312                    case ReadVetoResult.ReadAllow.Ignore:
313                        if (log.IsDebugEnabled)
314                            log.Debug("Trigger {0} asked us to ignore {1}", readTrigger.Value, document.Key);
315                        return null;
316                    default:
317                        throw new ArgumentOutOfRangeException(readVetoResult.Veto.ToString());
318                }
319            }
320
321            return document;
322        }
323
324        public dynamic Include(object maybeId)
325        {
326            if (maybeId == null || maybeId is DynamicNullObject)
327                return new DynamicNullObject();
328            var id = maybeId as string;
329            if (id != null)
330                return Include(id);
331            var jId = maybeId as RavenJValue;
332            if (jId != null)
333                return Include(jId.Value.ToString());
334
335            var items = new List<dynamic>();
336            foreach (var itemId in (IEnumerable)maybeId)
337            {
338                var include = Include(itemId);// this is where the real work happens
339                items.Add(include);
340            }
341            return new DynamicList(items);
342
343        }
344        public dynamic Include(string id)
345        {
346            ItemsToInclude.Add(id);
347            return Load(id);
348        }
349
350        public dynamic Include(IEnumerable<string> ids)
351        {
352            var items = new List<dynamic>();
353            foreach (var itemId in ids)
354            {
355                var include = Include(itemId);// this is where the real work happens
356                items.Add(include);
357            }
358            return new DynamicList(items);
359        }
360
361        public dynamic Load(string id)
362        {
363            var document = GetDocumentWithCaching(id);
364            document = ProcessReadVetoes(document,  ReadOperation.Load);
365            if (document == null)
366            {
367                Etag = Etag.HashWith(Etag.Empty);
368                return new DynamicNullObject();
369            }
370            Etag = Etag.HashWith(document.Etag);
371            if (document.Metadata.ContainsKey("Raven-Read-Veto"))
372            {
373                return new DynamicNullObject();
374            }
375
376            return new DynamicJsonObject(document.ToJson());
377        }
378
379        public dynamic Load(object maybeId)
380        {
381            if (maybeId == null || maybeId is DynamicNullObject)
382            {
383                Etag = Etag.HashWith(Etag.Empty);
384                return new DynamicNullObject();
385            }
386            var id = maybeId as string;
387            if (id != null)
388                return Load(id);
389            var jId = maybeId as RavenJValue;
390            if (jId != null)
391                return Load(jId.Value.ToString());
392
393            var items = new List<dynamic>();
394            foreach (var itemId in (IEnumerable)maybeId)
395            {
396                items.Add(Load(itemId));
397            }
398            return new DynamicList(items.Select(x => (object)x).ToArray());
399        }
400
401        public Dictionary<string, RavenJToken> TransformerParameters { get { return this.transformerParameters; } }
402        public HashSet<string> ItemsToInclude
403        {
404            get { return itemsToInclude; }
405        }
406    }
407}