PageRenderTime 5ms CodeModel.GetById 2ms app.highlight 27ms RepoModel.GetById 1ms app.codeStats 1ms

/ToMigrate/Raven.Database/Actions/DocumentActions.cs

http://github.com/ayende/ravendb
C# | 846 lines | 674 code | 159 blank | 13 comment | 134 complexity | e0eeb0347cf50dc9a23ac0409ff4e9f5 MD5 | raw file
  1// -----------------------------------------------------------------------
  2//  <copyright file="DocumentActions.cs" company="Hibernating Rhinos LTD">
  3//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
  4//  </copyright>
  5// -----------------------------------------------------------------------
  6using System;
  7using System.Collections.Generic;
  8using System.Linq;
  9using System.Threading;
 10
 11using Raven.Abstractions.Data;
 12using Raven.Abstractions.Exceptions;
 13using Raven.Abstractions.Extensions;
 14using Raven.Abstractions.Linq;
 15using Raven.Abstractions.Logging;
 16using Raven.Database.Data;
 17using Raven.Database.Extensions;
 18using Raven.Database.Impl;
 19using Raven.Database.Indexing;
 20using Raven.Database.Linq;
 21using Raven.Database.Plugins;
 22using Raven.Database.Storage;
 23using Raven.Database.Tasks;
 24using Raven.Database.Util;
 25using Raven.Json.Linq;
 26
 27namespace Raven.Database.Actions
 28{
 29    public class DocumentActions : ActionsBase
 30    {
 31        public DocumentActions(DocumentDatabase database, SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo> recentTouches, IUuidGenerator uuidGenerator, ILog log)
 32            : base(database, recentTouches, uuidGenerator, log)
 33        {
 34        }
 35
 36        public long GetNextIdentityValueWithoutOverwritingOnExistingDocuments(string key,
 37    IStorageActionsAccessor actions)
 38        {
 39            int tries;
 40            return GetNextIdentityValueWithoutOverwritingOnExistingDocuments(key, actions, out tries);
 41        }
 42
 43        public long GetNextIdentityValueWithoutOverwritingOnExistingDocuments(string key,
 44            IStorageActionsAccessor actions,
 45            out int tries)
 46        {
 47            long nextIdentityValue = actions.General.GetNextIdentityValue(key);
 48
 49            if (actions.Documents.DocumentMetadataByKey(key + nextIdentityValue) == null)
 50            {
 51                tries = 1;
 52                return nextIdentityValue;
 53            }
 54            tries = 1;
 55            // there is already a document with this id, this means that we probably need to search
 56            // for an opening in potentially large data set. 
 57            var lastKnownBusy = nextIdentityValue;
 58            var maybeFree = nextIdentityValue * 2;
 59            var lastKnownFree = long.MaxValue;
 60            while (true)
 61            {
 62                tries++;
 63                if (actions.Documents.DocumentMetadataByKey(key + maybeFree) == null)
 64                {
 65                    if (lastKnownBusy + 1 == maybeFree)
 66                    {
 67                        actions.General.SetIdentityValue(key, maybeFree);
 68                        return maybeFree;
 69                    }
 70                    lastKnownFree = maybeFree;
 71                    maybeFree = Math.Max(maybeFree - (maybeFree - lastKnownBusy) / 2, lastKnownBusy + 1);
 72
 73                }
 74                else
 75                {
 76                    lastKnownBusy = maybeFree;
 77                    maybeFree = Math.Min(lastKnownFree, maybeFree * 2);
 78                }
 79            }
 80        }
 81
 82
 83        private void AssertPutOperationNotVetoed(string key, RavenJObject metadata, RavenJObject document)
 84        {
 85            var vetoResult = Database.PutTriggers
 86                .Select(trigger => new { Trigger = trigger, VetoResult = trigger.AllowPut(key, document, metadata) })
 87                .FirstOrDefault(x => x.VetoResult.IsAllowed == false);
 88            if (vetoResult != null)
 89            {
 90                throw new OperationVetoedException("PUT vetoed on document " + key + " by " + vetoResult.Trigger + " because: " + vetoResult.VetoResult.Reason);
 91            }
 92        }
 93
 94        public RavenJArray GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start,
 95                                                          int pageSize, CancellationToken token, ref int nextStart,
 96                                                          string transformer = null, Dictionary<string, RavenJToken> transformerParameters = null,
 97                                                          string skipAfter = null)
 98        {
 99            var list = new RavenJArray();
100            GetDocumentsWithIdStartingWith(idPrefix, matches, exclude, start, pageSize, token, ref nextStart, 
101                doc => { if (doc != null) list.Add(doc.ToJson()); }, transformer, transformerParameters, skipAfter);
102
103            return list;
104        }
105
106        public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
107                                                   CancellationToken token, ref int nextStart, Action<JsonDocument> addDoc,
108                                                   string transformer = null, Dictionary<string, RavenJToken> transformerParameters = null,
109                                                   string skipAfter = null)
110        {
111            if (idPrefix == null)
112                throw new ArgumentNullException("idPrefix");
113            idPrefix = idPrefix.Trim();
114
115            var canPerformRapidPagination = nextStart > 0 && start == nextStart;
116            var actualStart = canPerformRapidPagination ? start : 0;
117            var addedDocs = 0;
118            var docCountOnLastAdd = 0;
119            var matchedDocs = 0;
120
121            TransactionalStorage.Batch(
122                actions =>
123                {
124                    var docsToSkip = canPerformRapidPagination ? 0 : start;
125                    int docCount;
126
127                    AbstractTransformer storedTransformer = null;
128                    if (transformer != null)
129                    {
130                        storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
131                        if (storedTransformer == null)
132                            throw new InvalidOperationException("No transformer with the name: " + transformer);
133                    }
134
135                    do
136                    {
137                        Database.WorkContext.UpdateFoundWork();
138
139                        docCount = 0;
140                        var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
141                        var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
142
143                        foreach (var doc in docs)
144                        {
145                            token.ThrowIfCancellationRequested();
146                            docCount++;
147                            if (docCount - docCountOnLastAdd > 1000)
148                            {
149                                addDoc(null); // heartbeat
150                            }
151
152                            var keyTest = doc.Key.Substring(idPrefix.Length);
153
154                            if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
155                                continue;
156
157                            JsonDocument.EnsureIdInMetadata(doc);
158
159                            var document = documentRetriever.ExecuteReadTriggers(doc, ReadOperation.Load);
160                            if (document == null)
161                                continue;
162
163                            matchedDocs++;
164
165                            if (matchedDocs <= docsToSkip)
166                                continue;
167
168                            token.ThrowIfCancellationRequested();
169
170                            document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
171                            addDoc(document);
172
173                            addedDocs++;
174                            docCountOnLastAdd = docCount;
175
176                            if (addedDocs >= pageSize)
177                                break;
178                        }
179
180                        actualStart += pageSize;
181                    }
182                    while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
183                });
184
185            if (addedDocs != pageSize)
186                nextStart = start; // will mark as last page
187            else if (canPerformRapidPagination)
188                nextStart = start + matchedDocs;
189            else
190                nextStart = actualStart;
191        }
192
193        private JsonDocument TransformDocumentIfNeeded(JsonDocument document, AbstractTransformer storedTransformer, DocumentRetriever documentRetriever)
194        {
195            if (storedTransformer == null)
196                return document;
197
198            using (new CurrentTransformationScope(Database, documentRetriever))
199            {
200                var transformed = storedTransformer
201                    .TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
202                    .Select<dynamic, dynamic>(x => JsonExtensions.ToJObject((object)x))
203                    .ToArray();
204
205                RavenJObject ravenJObject;
206                switch (transformed.Length)
207                {
208                    case 0:
209                        throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
210                    case 1:
211                        ravenJObject = transformed[0];
212                        break;
213                    default:
214                        ravenJObject = new RavenJObject { { "$values", new RavenJArray(transformed) } };
215                        break;
216                }
217
218                return new JsonDocument
219                {
220                    Etag = document.Etag.CombineHashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
221                    LastModified = document.LastModified,
222                    DataAsJson = ravenJObject
223                };
224            }
225        }
226
227        private void RemoveMetadataReservedProperties(RavenJObject metadata)
228        {
229            RemoveReservedProperties(metadata);
230            metadata.Remove("Raven-Last-Modified");
231            metadata.Remove("Last-Modified");
232        }
233
234        private void RemoveReservedProperties(RavenJObject document)
235        {
236            document.Remove(string.Empty);
237            var toRemove = document.Keys.Where(propertyName => propertyName.StartsWith("@") || HeadersToIgnoreServer.Contains(propertyName) || Database.Configuration.Core.HeadersToIgnore.Contains(propertyName)).ToList();
238            foreach (var propertyName in toRemove)
239            {
240                document.Remove(propertyName);
241            }
242        }
243
244        private void AssertDeleteOperationNotVetoed(string key)
245        {
246            var vetoResult = Database.DeleteTriggers
247                .Select(trigger => new { Trigger = trigger, VetoResult = trigger.AllowDelete(key) })
248                .FirstOrDefault(x => x.VetoResult.IsAllowed == false);
249            if (vetoResult != null)
250            {
251                throw new OperationVetoedException("DELETE vetoed on document " + key + " by " + vetoResult.Trigger +
252                                                   " because: " + vetoResult.VetoResult.Reason);
253            }
254        }
255
256        public int BulkInsert(BulkInsertOptions options, IEnumerable<IEnumerable<JsonDocument>> docBatches, Guid operationId, CancellationToken token, CancellationTimeout timeout = null)
257        {
258            var documents = 0;
259
260            Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
261            {
262                OperationId = operationId,
263                Type = DocumentChangeTypes.BulkInsertStarted
264            });
265            using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, WorkContext.CancellationToken))
266            {
267                foreach (var docs in docBatches)
268                {
269                    cts.Token.ThrowIfCancellationRequested();
270
271                    var docsToInsert = docs.ToArray();
272                    var batch = 0;
273                    var keys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
274                    var collectionsAndEtags = new Dictionary<string, Etag>(StringComparer.OrdinalIgnoreCase);
275
276                    if (timeout != null)
277                        timeout.Pause();
278                    using (Database.DocumentLock.Lock())
279                    {
280                        if (timeout != null)
281                            timeout.Resume();
282
283                        TransactionalStorage.Batch(accessor =>
284                        {
285                            var inserts = 0;
286                            
287                            foreach (var doc in docsToInsert)
288                            {
289                                try
290                                {
291                                    if (string.IsNullOrEmpty(doc.Key))
292                                        throw new InvalidOperationException("Cannot try to bulk insert a document without a key");
293
294                                    RemoveReservedProperties(doc.DataAsJson);
295                                    RemoveMetadataReservedProperties(doc.Metadata);
296
297                                    if (options.CheckReferencesInIndexes)
298                                        keys.Add(doc.Key);
299                                    documents++;
300                                    batch++;
301                                    AssertPutOperationNotVetoed(doc.Key, doc.Metadata, doc.DataAsJson);
302
303                                    if (options.OverwriteExisting && options.SkipOverwriteIfUnchanged)
304                                    {
305                                        var existingDoc = accessor.Documents.DocumentByKey(doc.Key);
306
307                                        if (IsTheSameDocument(doc, existingDoc))
308                                            continue;
309                                    }
310
311                                    foreach (var trigger in Database.PutTriggers)
312                                    {
313                                        trigger.Value.OnPut(doc.Key, doc.DataAsJson, doc.Metadata);
314                                    }
315
316                                    var result = accessor.Documents.InsertDocument(doc.Key, doc.DataAsJson, doc.Metadata, options.OverwriteExisting);
317                                    if (result.Updated == false)
318                                        inserts++;
319
320                                    doc.Etag = result.Etag;
321
322                                    doc.Metadata.EnsureSnapshot(
323                                        "Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
324                                    doc.DataAsJson.EnsureSnapshot(
325                                        "Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
326
327                                    var entityName = doc.Metadata.Value<string>(Constants.RavenEntityName);
328
329                                    Etag highestEtagInCollection;
330                                    if (string.IsNullOrEmpty(entityName) == false && (collectionsAndEtags.TryGetValue(entityName, out highestEtagInCollection) == false ||
331                                                                                      result.Etag.CompareTo(highestEtagInCollection) > 0))
332                                    {
333                                        collectionsAndEtags[entityName] = result.Etag;
334                                    }
335
336                                    foreach (var trigger in Database.PutTriggers)
337                                    {
338                                        trigger.Value.AfterPut(doc.Key, doc.DataAsJson, doc.Metadata, result.Etag);
339                                    }
340
341                                    Database.WorkContext.UpdateFoundWork();
342                                }
343                                catch (Exception e)
344                                {
345                                    Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
346                                    {
347                                        OperationId = operationId,
348                                        Message = e.Message,
349                                        Etag = doc.Etag,
350                                        Id = doc.Key,
351                                        Type = DocumentChangeTypes.BulkInsertError
352                                    });
353
354                                    throw;
355                                }
356                            }
357
358                            if (options.CheckReferencesInIndexes)
359                            {
360                                foreach (var key in keys)
361                                {
362                                    Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, accessor);
363                                }
364                            }
365
366                            accessor.Documents.IncrementDocumentCount(inserts);
367                        });
368
369                        foreach (var collectionEtagPair in collectionsAndEtags)
370                        {
371                            Database.LastCollectionEtags.Update(collectionEtagPair.Key, collectionEtagPair.Value);
372                        }
373
374                        WorkContext.ShouldNotifyAboutWork(() => "BulkInsert batch of " + batch + " docs");
375                        WorkContext.NotifyAboutWork(); // forcing notification so we would start indexing right away
376                        WorkContext.UpdateFoundWork();
377                    }
378                }
379            }
380
381            Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification
382            {
383                OperationId = operationId,
384                Type = DocumentChangeTypes.BulkInsertEnded
385            });
386
387            if (documents > 0)
388                WorkContext.ShouldNotifyAboutWork(() => "BulkInsert of " + documents + " docs");
389
390            return documents;
391        }
392
393        private bool IsTheSameDocument(JsonDocument doc, JsonDocument existingDoc)
394        {
395            if (existingDoc == null)
396                return false;
397
398            if (RavenJToken.DeepEquals(doc.DataAsJson, existingDoc.DataAsJson) == false)
399                return false;
400
401            var existingMetadata = (RavenJObject) existingDoc.Metadata.CloneToken();
402            var newMetadata = (RavenJObject) doc.Metadata.CloneToken();
403            // in order to compare metadata we need to remove metadata records created by triggers
404            foreach (var trigger in Database.PutTriggers)
405            {
406                var metadataToIgnore = trigger.Value.GeneratedMetadataNames;
407
408                if (metadataToIgnore == null)
409                    continue;
410
411                foreach (var toIgnore in metadataToIgnore)
412                {
413                    existingMetadata.Remove(toIgnore);
414                    newMetadata.Remove(toIgnore);
415                }
416            }
417
418            return RavenJToken.DeepEquals(newMetadata, existingMetadata);
419        }
420
421        public TouchedDocumentInfo GetRecentTouchesFor(string key)
422        {
423            TouchedDocumentInfo info;
424            RecentTouches.TryGetValue(key, out info);
425            return info;
426        }
427
428        public RavenJArray GetDocumentsAsJson(int start, int pageSize, Etag etag, CancellationToken token)
429        {
430            var list = new RavenJArray();
431            GetDocuments(start, pageSize, etag, token, doc =>
432            {
433                if (doc != null) list.Add(doc.ToJson());
434                return true;
435            });
436            return list;
437        }
438
439        public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func<JsonDocument, bool> addDocument, string transformer = null, Dictionary<string, RavenJToken> transformerParameters = null)
440        {
441            Etag lastDocumentReadEtag = null;
442
443            TransactionalStorage.Batch(actions =>
444            {
445                AbstractTransformer storedTransformer = null;
446                if (transformer != null)
447                {
448                    storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
449                    if (storedTransformer == null)
450                        throw new InvalidOperationException("No transformer with the name: " + transformer);
451                }
452
453                bool returnedDocs = false;
454                while (true)
455                {
456                    var documents = etag == null
457                                        ? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
458                                        : actions.Documents.GetDocumentsAfter(etag, pageSize, token);
459
460                    var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
461                    int docCount = 0;
462                    int docCountOnLastAdd = 0;
463                    foreach (var doc in documents)
464                    {
465                        docCount++;
466
467                        token.ThrowIfCancellationRequested();
468
469                        if (docCount - docCountOnLastAdd > 1000)
470                        {
471                            addDocument(null); // heartbeat
472                        }
473
474                        if (etag != null)
475                            etag = doc.Etag;
476
477                        JsonDocument.EnsureIdInMetadata(doc);
478
479                        var document = documentRetriever.ExecuteReadTriggers(doc, ReadOperation.Load);
480                        if (document == null)
481                            continue;
482                        
483                        returnedDocs = true;
484                        Database.WorkContext.UpdateFoundWork();
485                        
486                        document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
487
488                        bool canContinue = addDocument(document);
489                        if (!canContinue)
490                            break;
491
492                        lastDocumentReadEtag = etag;
493
494                        docCountOnLastAdd = docCount;
495                    }
496
497                    if (returnedDocs || docCount == 0)
498                        break;
499
500                    // No document was found that matches the requested criteria
501                    // If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
502                    if (lastDocumentReadEtag != null)
503                        etag = lastDocumentReadEtag;
504
505                    start += docCount;
506                }
507            });
508
509            return lastDocumentReadEtag;
510        }
511
512        public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Func<JsonDocument, bool> addDocument)
513        {
514            Etag lastDocumentReadEtag = null;
515
516            TransactionalStorage.Batch(actions =>
517            {
518                bool returnedDocs = false;
519                while (true)
520                {
521                    var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2), lastProcessedDocument: x => lastDocumentReadEtag = x );
522                    var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers);
523                    
524                    int docCount = 0;
525                    int docCountOnLastAdd = 0;
526                    foreach (var doc in documents)
527                    {
528                        docCount++;                        
529                        if (docCount - docCountOnLastAdd > 1000)
530                        {
531                            addDocument(null); // heartbeat
532                        }
533
534                        token.ThrowIfCancellationRequested();
535                        
536                        etag = doc.Etag;
537                        
538                        JsonDocument.EnsureIdInMetadata(doc);
539                        
540                        var document = documentRetriever.ExecuteReadTriggers(doc, ReadOperation.Load);
541                        if (document == null)
542                            continue;
543
544                        returnedDocs = true;
545                        Database.WorkContext.UpdateFoundWork();
546
547                        bool canContinue = addDocument(document);                                                                          
548
549                        docCountOnLastAdd = docCount;
550
551                        if (!canContinue)
552                            break;
553                    }
554
555                    if (returnedDocs)
556                        break;
557
558                    // No document was found that matches the requested criteria
559                    if ( docCount == 0 )
560                    {
561                        // If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
562                        if (lastDocumentReadEtag != null)
563                            etag = lastDocumentReadEtag;
564
565                        break;
566                    }
567                }
568            });
569
570            return etag;
571        }
572
573        public JsonDocument Get(string key)
574        {
575            if (key == null)
576                throw new ArgumentNullException("key");
577
578            key = key.Trim();
579
580            JsonDocument document = null;
581            TransactionalStorage.Batch(actions =>
582            {
583                document = actions.Documents.DocumentByKey(key);
584            });
585
586            JsonDocument.EnsureIdInMetadata(document);
587
588            return new DocumentRetriever(null, null, Database.ReadTriggers)
589                            .ExecuteReadTriggers(document, ReadOperation.Load);
590        }
591
592        public JsonDocumentMetadata GetDocumentMetadata(string key)
593        {
594            if (key == null)
595                throw new ArgumentNullException("key");
596
597            key = key.Trim();
598
599            JsonDocumentMetadata document = null;
600                TransactionalStorage.Batch(actions =>
601                {
602                    document = actions.Documents.DocumentMetadataByKey(key);
603                });
604
605            JsonDocument.EnsureIdInMetadata(document);
606
607            return new DocumentRetriever(null, null, Database.ReadTriggers)
608                            .ProcessReadVetoes(document, ReadOperation.Load);
609        }
610
611        public Etag GetLastEtagForCollection(string collectionName)
612        {
613            Etag value = Etag.Empty;
614            TransactionalStorage.Batch(accessor =>
615            {
616                var dbvalue = accessor.Lists.Read("Raven/Collection/Etag", collectionName);
617                if (dbvalue != null)
618                {
619                    value = Etag.Parse(dbvalue.Data.Value<Byte[]>("Etag"));
620                }
621            });
622            return value;
623        }
624
625
626        public JsonDocument GetWithTransformer(string key, string transformer, Dictionary<string, RavenJToken> transformerParameters, out HashSet<string> itemsToInclude)
627        {
628            JsonDocument result = null;
629            DocumentRetriever docRetriever = null;
630            TransactionalStorage.Batch(
631            actions =>
632            {
633                docRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
634                using (new CurrentTransformationScope(Database, docRetriever))
635                {
636                    var document = Get(key);
637                    if (document == null)
638                        return;
639
640                    if (document.Metadata.ContainsKey("Raven-Read-Veto") || document.Metadata.ContainsKey(Constants.RavenReplicationConflict))
641                    {
642                        result = document;
643                        return;
644                    }
645
646                    var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
647                    if (storedTransformer == null)
648                        throw new InvalidOperationException("No transformer with the name: " + transformer);
649
650                    var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
651                                     .Select(x => JsonExtensions.ToJObject(x))
652                                     .ToArray();
653                   
654                    if (transformed.Length == 0)
655                        return;
656
657                    result = new JsonDocument
658                    {
659                        Etag = document.Etag.CombineHashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag),
660                        LastModified = document.LastModified,
661                        DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed.Cast<Object>().ToArray()) } },
662                    };
663                }
664            });
665            itemsToInclude = docRetriever.ItemsToInclude;
666            return result;
667        }
668
669
670
671        public PutResult Put(string key, Etag etag, RavenJObject document, RavenJObject metadata, string[] participatingIds = null)
672        {
673            WorkContext.MetricsCounters.DocsPerSecond.Mark();
674            key = string.IsNullOrWhiteSpace(key) ? Guid.NewGuid().ToString() : key.Trim();
675            RemoveReservedProperties(document);
676            RemoveMetadataReservedProperties(metadata);
677            var newEtag = Etag.Empty;
678
679            using (Database.DocumentLock.Lock())
680            {
681                TransactionalStorage.Batch(actions =>
682                {
683                    if (key.EndsWith("/"))
684                    {
685                        key += GetNextIdentityValueWithoutOverwritingOnExistingDocuments(key, actions);
686                    }
687
688                    AssertPutOperationNotVetoed(key, metadata, document);
689
690                    Database.PutTriggers.Apply(trigger => trigger.OnPut(key, document, metadata));
691
692                        var addDocumentResult = actions.Documents.AddDocument(key, etag, document, metadata);
693                        newEtag = addDocumentResult.Etag;
694
695                        Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions, participatingIds);
696                        metadata[Constants.LastModified] = addDocumentResult.SavedAt;
697
698                    metadata.EnsureSnapshot("Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
699                    document.EnsureSnapshot("Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
700
701                        actions.AfterStorageCommitBeforeWorkNotifications(new JsonDocument
702                        {
703                            Metadata = metadata,
704                            Key = key,
705                            DataAsJson = document,
706                            Etag = newEtag,
707                            LastModified = addDocumentResult.SavedAt,
708                            SkipDeleteFromIndex = addDocumentResult.Updated == false
709                        }, documents =>
710                        {
711                            if(Database.IndexDefinitionStorage.IndexesCount == 0 || Database.WorkContext.RunIndexing == false)
712                                return;
713
714                            Database.Prefetcher.AfterStorageCommitBeforeWorkNotifications(PrefetchingUser.Indexer, documents);
715                        });
716
717                    Database.PutTriggers.Apply(trigger => trigger.AfterPut(key, document, metadata, newEtag));
718
719                        TransactionalStorage
720                            .ExecuteImmediatelyOrRegisterForSynchronization(() =>
721                            {
722                                Database.PutTriggers.Apply(trigger => trigger.AfterCommit(key, document, metadata, newEtag));
723                                
724                                var newDocumentChangeNotification =
725                                    new DocumentChangeNotification
726                                    {
727                                        Id = key,
728                                        Type = DocumentChangeTypes.Put,
729                                        TypeName = metadata.Value<string>(Constants.RavenClrType),
730                                        CollectionName = metadata.Value<string>(Constants.RavenEntityName),
731                                        Etag = newEtag
732                                    };
733                                
734                                Database.Notifications.RaiseNotifications(newDocumentChangeNotification, metadata);
735                            });
736
737                        WorkContext.ShouldNotifyAboutWork(() => "PUT " + key);
738                });
739
740                if (Log.IsDebugEnabled)
741                    Log.Debug("Put document {0} with etag {1}", key, newEtag);
742
743                return new PutResult
744                {
745                    Key = key,
746                    ETag = newEtag
747                };
748            }
749        }
750
751        public bool Delete(string key, Etag etag, string[] participatingIds = null)
752        {
753            RavenJObject metadata;
754            return Delete(key, etag, out metadata, participatingIds);
755        }
756
757        public bool Delete(string key, Etag etag,  out RavenJObject metadata, string[] participatingIds = null)
758        {
759            if (key == null)
760                throw new ArgumentNullException("key");
761
762            key = key.Trim();
763
764            var deleted = false;
765
766            if (Log.IsDebugEnabled)
767                Log.Debug("Delete a document with key: {0} and etag {1}", key, etag);
768
769            RavenJObject metadataVar = null;
770            using (Database.DocumentLock.Lock())
771            {
772                TransactionalStorage.Batch(actions =>
773                {
774                    AssertDeleteOperationNotVetoed(key);
775
776                    Database.DeleteTriggers.Apply(trigger => trigger.OnDelete(key));
777
778                        string collection = null;
779                        Etag deletedETag;
780                        if (actions.Documents.DeleteDocument(key, etag, out metadataVar, out deletedETag))
781                        {
782                            deleted = true;
783                            actions.Indexing.RemoveAllDocumentReferencesFrom(key);
784                            WorkContext.MarkDeleted(key);
785
786                            Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions, participatingIds);
787
788                            collection = metadataVar.Value<string>(Constants.RavenEntityName);
789                            
790                            DeleteDocumentFromIndexesForCollection(key, collection, actions);
791                            if (deletedETag != null)
792                                Database.Prefetcher.AfterDelete(key, deletedETag);
793                        Database.DeleteTriggers.Apply(trigger => trigger.AfterDelete(key));
794                        }
795
796                        TransactionalStorage
797                            .ExecuteImmediatelyOrRegisterForSynchronization(() =>
798                            {
799                                Database.DeleteTriggers.Apply(trigger => trigger.AfterCommit(key));
800                                if (string.IsNullOrEmpty(collection) == false)
801                                    Database.LastCollectionEtags.Update(collection);
802
803                                Database.Notifications.RaiseNotifications(new DocumentChangeNotification
804                                {
805                                    Id = key,
806                                    Type = DocumentChangeTypes.Delete,
807                                    TypeName = (metadataVar != null) ? metadataVar.Value<string>(Constants.RavenClrType) : null,
808                                    CollectionName = (metadataVar != null) ? metadataVar.Value<string>(Constants.RavenEntityName) : null
809
810                                }, metadataVar);
811                            });
812
813                    WorkContext.ShouldNotifyAboutWork(() => "DEL " + key);
814                });
815
816                metadata = metadataVar;
817                return deleted;
818            }
819        }
820
821        internal void DeleteDocumentFromIndexesForCollection(string key, string collection, IStorageActionsAccessor actions)
822        {
823            foreach (var indexName in IndexDefinitionStorage.IndexNames)
824            {
825                AbstractViewGenerator abstractViewGenerator =
826                    IndexDefinitionStorage.GetViewGenerator(indexName);
827                if (abstractViewGenerator == null)
828                    continue;
829
830
831                if (collection != null && // the document has a entity name
832                    abstractViewGenerator.ForEntityNames.Count > 0)
833                    // the index operations on specific entities
834                {
835                    if (abstractViewGenerator.ForEntityNames.Contains(collection) == false)
836                        continue;
837                }
838
839                var instance = IndexDefinitionStorage.GetIndexDefinition(indexName);
840                var task = actions.GetTask(x => x.Index == instance.IndexId, 
841                    new RemoveFromIndexTask(instance.IndexId));
842                task.AddKey(key);
843            }
844        }
845    }
846}