PageRenderTime 73ms CodeModel.GetById 14ms app.highlight 51ms RepoModel.GetById 2ms app.codeStats 0ms

/Raven.Database/Storage/Voron/StorageActions/DocumentsStorageActions.cs

https://github.com/nwendel/ravendb
C# | 695 lines | 548 code | 145 blank | 2 comment | 119 complexity | 6de3c7ab17dad266510349a1985baadb MD5 | raw file
  1using Raven.Abstractions;
  2using Raven.Abstractions.Data;
  3using Raven.Abstractions.Exceptions;
  4using Raven.Abstractions.Extensions;
  5using Raven.Abstractions.Logging;
  6using Raven.Abstractions.MEF;
  7using Raven.Abstractions.Util;
  8using Raven.Abstractions.Util.Streams;
  9using Raven.Database.Impl;
 10using Raven.Database.Plugins;
 11using Raven.Database.Storage.Voron.Impl;
 12using Raven.Json.Linq;
 13using System;
 14using System.Collections.Generic;
 15using System.Diagnostics;
 16using System.IO;
 17using System.Linq;
 18using System.Text;
 19using System.Threading;
 20using Voron;
 21using Voron.Impl;
 22using Constants = Raven.Abstractions.Data.Constants;
 23
 24namespace Raven.Database.Storage.Voron.StorageActions
 25{
 26	public class DocumentsStorageActions : StorageActionsBase, IDocumentStorageActions
 27	{
 28		private readonly Reference<WriteBatch> writeBatch;
 29
 30		private readonly IUuidGenerator uuidGenerator;
 31		private readonly OrderedPartCollection<AbstractDocumentCodec> documentCodecs;
 32		private readonly IDocumentCacher documentCacher;
 33
 34		private static readonly ILog logger = LogManager.GetCurrentClassLogger();
 35        private readonly Dictionary<Etag, Etag> etagTouches = new Dictionary<Etag, Etag>();
 36		private readonly TableStorage tableStorage;
 37
 38		private readonly Index metadataIndex;
 39
 40		public DocumentsStorageActions(IUuidGenerator uuidGenerator,
 41			OrderedPartCollection<AbstractDocumentCodec> documentCodecs,
 42			IDocumentCacher documentCacher,
 43			Reference<WriteBatch> writeBatch,
 44			Reference<SnapshotReader> snapshot,
 45            TableStorage tableStorage, 
 46            IBufferPool bufferPool)
 47			: base(snapshot, bufferPool)
 48		{
 49			this.uuidGenerator = uuidGenerator;
 50			this.documentCodecs = documentCodecs;
 51			this.documentCacher = documentCacher;
 52			this.writeBatch = writeBatch;
 53			this.tableStorage = tableStorage;
 54
 55			metadataIndex = tableStorage.Documents.GetIndex(Tables.Documents.Indices.Metadata);
 56		}
 57
 58		public IEnumerable<JsonDocument> GetDocumentsByReverseUpdateOrder(int start, int take)
 59		{
 60			if (start < 0)
 61				throw new ArgumentException("must have zero or positive value", "start");
 62			if (take < 0)
 63				throw new ArgumentException("must have zero or positive value", "take");
 64			if (take == 0) yield break;
 65
 66			using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
 67											.Iterate(Snapshot, writeBatch.Value))
 68			{
 69				int fetchedDocumentCount = 0;
 70				if (!iterator.Seek(Slice.AfterAllKeys))
 71					yield break;
 72
 73				if (!iterator.Skip(-start))
 74					yield break;
 75				do
 76				{
 77					if (iterator.CurrentKey == null || iterator.CurrentKey.Equals(Slice.Empty))
 78						yield break;
 79
 80					var key = GetKeyFromCurrent(iterator);
 81
 82					var document = DocumentByKey(key, null);
 83					if (document == null) //precaution - should never be true
 84					{
 85						throw new InvalidDataException(string.Format("Possible data corruption - the key = '{0}' was found in the documents indice, but matching document was not found.", key));
 86					}
 87
 88					yield return document;
 89
 90					fetchedDocumentCount++;
 91				} while (iterator.MovePrev() && fetchedDocumentCount < take);
 92			}
 93		}
 94
 95		public IEnumerable<KeyValuePair<string, Etag>> GetDocumentEtagsFromKeyByEtagIndice()
 96		{
 97			using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
 98				.Iterate(Snapshot, writeBatch.Value))
 99			{
100				if (!iterator.Seek(Slice.AfterAllKeys))
101					yield break;
102				do
103				{
104					if (iterator.CurrentKey == null || iterator.CurrentKey.Equals(Slice.Empty))
105						continue;
106
107					var key = GetKeyFromCurrent(iterator);
108					var etag = Etag.Parse(iterator.CurrentKey.ToString());
109
110					yield return new KeyValuePair<string, Etag>(key,etag);
111				} while (iterator.MovePrev());
112			}
113		}
114
115		public IEnumerable<KeyValuePair<string, Etag>> GetDocumentEtagsFromMetadata()
116		{			
117			using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
118				.Iterate(Snapshot, writeBatch.Value))
119			{
120				if (!iterator.Seek(Slice.AfterAllKeys))
121					yield break;
122				do
123				{
124					if (iterator.CurrentKey == null || iterator.CurrentKey.Equals(Slice.Empty))
125						continue;
126
127					var key = GetKeyFromCurrent(iterator);
128					var documentMetadata = DocumentMetadataByKey(key, null);
129					yield return new KeyValuePair<string, Etag>(key, documentMetadata.Etag);
130
131				} while (iterator.MovePrev());
132			}
133		}
134
135
136		public IEnumerable<JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long? maxSize = null, Etag untilEtag = null, TimeSpan? timeout = null)
137		{
138			if (take < 0)
139				throw new ArgumentException("must have zero or positive value", "take");
140			if (take == 0) yield break;
141
142			if (string.IsNullOrEmpty(etag))
143				throw new ArgumentNullException("etag");
144
145			Stopwatch duration = null;
146			if (timeout != null)
147				duration = Stopwatch.StartNew();
148
149			using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
150											.Iterate(Snapshot, writeBatch.Value))
151			{
152				if (!iterator.Seek(Slice.BeforeAllKeys))
153				{
154					yield break;
155				}
156
157				long fetchedDocumentTotalSize = 0;
158				int fetchedDocumentCount = 0;
159
160				do
161				{
162					cancellationToken.ThrowIfCancellationRequested();
163
164					if (iterator.CurrentKey == null || iterator.CurrentKey.Equals(Slice.Empty))
165						yield break;
166
167					var docEtag = Etag.Parse(iterator.CurrentKey.ToString());
168
169					if (!EtagUtil.IsGreaterThan(docEtag, etag)) continue;
170
171					if (untilEtag != null && fetchedDocumentCount > 0)
172					{
173						if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
174							yield break;
175					}
176
177					var key = GetKeyFromCurrent(iterator);
178
179					var document = DocumentByKey(key, null);
180					if (document == null) //precaution - should never be true
181					{
182						throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents indice, but matching document was not found", key));
183					}
184
185					if (!document.Etag.Equals(docEtag))
186					{
187						throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its indice",key));
188					}
189
190					fetchedDocumentTotalSize += document.SerializedSizeOnDisk;
191					fetchedDocumentCount++;
192
193					if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize)
194					{
195						yield return document;
196						yield break;
197					}
198
199					yield return document;
200
201					if (timeout != null)
202					{
203						if (duration.Elapsed > timeout.Value)
204							yield break;
205					}
206				} while (iterator.MoveNext() && fetchedDocumentCount < take);
207			}
208		}
209
210		private static string GetKeyFromCurrent(global::Voron.Trees.IIterator iterator)
211		{
212			string key;
213			using (var currentDataStream = iterator.CreateReaderForCurrent().AsStream())
214			{
215				var keyBytes = currentDataStream.ReadData();
216				key = Encoding.UTF8.GetString(keyBytes);
217			}
218			return key;
219		}
220
221		public IEnumerable<JsonDocument> GetDocumentsWithIdStartingWith(string idPrefix, int start, int take, string skipAfter)
222		{
223			if (string.IsNullOrEmpty(idPrefix))
224				throw new ArgumentNullException("idPrefix");
225			if (start < 0)
226				throw new ArgumentException("must have zero or positive value", "start");
227			if (take < 0)
228				throw new ArgumentException("must have zero or positive value", "take");
229
230			if (take == 0)
231				yield break;
232
233			using (var iterator = tableStorage.Documents.Iterate(Snapshot, writeBatch.Value))
234			{
235				iterator.RequiredPrefix = idPrefix.ToLowerInvariant();
236				var seekStart = skipAfter == null ? iterator.RequiredPrefix : skipAfter.ToLowerInvariant();
237				if (iterator.Seek(seekStart) == false || !iterator.Skip(start))
238					yield break;
239
240				if (skipAfter != null && !iterator.MoveNext())
241					yield break; // move to the _next_ one
242					
243
244				var fetchedDocumentCount = 0;
245				do
246				{
247					var key = iterator.CurrentKey.ToString();
248
249					var fetchedDocument = DocumentByKey(key, null);
250					if (fetchedDocument == null) continue;
251
252					fetchedDocumentCount++;
253					yield return fetchedDocument;
254				} while (iterator.MoveNext() && fetchedDocumentCount < take);
255			}
256		}
257
258		public long GetDocumentsCount()
259		{
260			return tableStorage.GetEntriesCount(tableStorage.Documents);
261		}
262
263		public JsonDocument DocumentByKey(string key, TransactionInformation transactionInformation)
264		{
265			if (string.IsNullOrEmpty(key))
266			{
267				logger.Debug("Document with empty key was not found");
268				return null;
269			}
270
271			var lowerKey = CreateKey(key);
272			if (!tableStorage.Documents.Contains(Snapshot, lowerKey, writeBatch.Value))
273			{
274				logger.Debug("Document with key='{0}' was not found", key);
275				return null;
276			}
277
278			var metadataDocument = ReadDocumentMetadata(key);
279			if (metadataDocument == null)
280			{
281				logger.Warn(string.Format("Metadata of document with key='{0} was not found, but the document itself exists.", key));
282				return null;
283			}
284
285			var documentData = ReadDocumentData(key, metadataDocument.Etag, metadataDocument.Metadata);
286
287			logger.Debug("DocumentByKey() by key ='{0}'", key);
288
289			var docSize = tableStorage.Documents.GetDataSize(Snapshot, lowerKey);
290
291			var metadataSize = metadataIndex.GetDataSize(Snapshot, lowerKey);
292
293			return new JsonDocument
294			{
295				DataAsJson = documentData,
296				Etag = metadataDocument.Etag,
297				Key = metadataDocument.Key, //original key - with user specified casing, etc.
298				Metadata = metadataDocument.Metadata,
299				SerializedSizeOnDisk = docSize + metadataSize,
300				LastModified = metadataDocument.LastModified
301			};
302		}
303
304		public JsonDocumentMetadata DocumentMetadataByKey(string key, TransactionInformation transactionInformation)
305		{
306			if (string.IsNullOrEmpty(key))
307				throw new ArgumentNullException("key");
308
309			var lowerKey = CreateKey(key);
310
311			if (tableStorage.Documents.Contains(Snapshot, lowerKey, writeBatch.Value))
312				return ReadDocumentMetadata(key);
313
314			logger.Debug("Document with key='{0}' was not found", key);
315			return null;
316		}
317
318		public bool DeleteDocument(string key, Etag etag, out RavenJObject metadata, out Etag deletedETag)
319		{
320			if (string.IsNullOrEmpty(key))
321				throw new ArgumentNullException("key");
322
323			var loweredKey = CreateKey(key);
324			
325			if(etag != null)
326				EnsureDocumentEtagMatch(loweredKey, etag, "DELETE");
327
328			ushort? existingVersion;
329			if (!tableStorage.Documents.Contains(Snapshot, loweredKey, writeBatch.Value, out existingVersion))
330			{
331				logger.Debug("Document with key '{0}' was not found, and considered deleted", key);
332				metadata = null;
333				deletedETag = null;
334				return false;
335			}
336
337			if (!metadataIndex.Contains(Snapshot, loweredKey, writeBatch.Value)) //data exists, but metadata is not --> precaution, should never be true
338			{
339				var errorString = string.Format("Document with key '{0}' was found, but its metadata wasn't found --> possible data corruption", key);
340				throw new InvalidDataException(errorString);
341			}
342
343			var existingEtag = EnsureDocumentEtagMatch(key, etag, "DELETE");
344			var documentMetadata = ReadDocumentMetadata(key);
345			metadata = documentMetadata.Metadata;
346
347			deletedETag = etag != null ? existingEtag : documentMetadata.Etag;
348
349			tableStorage.Documents.Delete(writeBatch.Value, loweredKey, existingVersion);
350			metadataIndex.Delete(writeBatch.Value, loweredKey);
351
352			tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
353						  .Delete(writeBatch.Value, deletedETag);
354
355			documentCacher.RemoveCachedDocument(loweredKey, etag);
356
357			logger.Debug("Deleted document with key = '{0}'", key);
358
359			return true;
360		}
361
362		public AddDocumentResult AddDocument(string key, Etag etag, RavenJObject data, RavenJObject metadata)
363		{
364			if (string.IsNullOrEmpty(key))
365				throw new ArgumentNullException("key");
366
367			if (key != null && Encoding.UTF8.GetByteCount(key) >= UInt16.MaxValue)
368				throw new ArgumentException(string.Format("The dataKey must be a maximum of {0} bytes in Unicode, key is: '{1}'", UInt16.MaxValue, key), "key");
369
370			Etag existingEtag;
371			Etag newEtag;
372
373			DateTime savedAt;
374			var isUpdate = WriteDocumentData(key, etag, data, metadata, out newEtag, out existingEtag, out savedAt);
375
376			logger.Debug("AddDocument() - {0} document with key = '{1}'", isUpdate ? "Updated" : "Added", key);
377
378			return new AddDocumentResult
379			{
380				Etag = newEtag,
381				PrevEtag = existingEtag,
382				SavedAt = savedAt,
383				Updated = isUpdate
384			};
385		}
386
387		private bool PutDocumentMetadataInternal(string key, RavenJObject metadata, Etag newEtag, DateTime savedAt)
388		{
389			return WriteDocumentMetadata(new JsonDocumentMetadata
390			{
391				Key = key,
392				Etag = newEtag,
393				Metadata = metadata,
394				LastModified = savedAt
395			});
396		}
397
398		public void IncrementDocumentCount(int value)
399		{
400			//nothing to do here			
401		}
402
403		public AddDocumentResult InsertDocument(string key, RavenJObject data, RavenJObject metadata, bool overwriteExisting)
404		{
405			if (string.IsNullOrEmpty(key))
406				throw new ArgumentNullException("key");
407
408			if (!overwriteExisting && tableStorage.Documents.Contains(Snapshot, CreateKey(key), writeBatch.Value))
409			{
410				throw new ConcurrencyException(string.Format("InsertDocument() - overwriteExisting is false and document with key = '{0}' already exists", key));
411			}
412
413			return AddDocument(key, null, data, metadata);
414		}
415
416		public void TouchDocument(string key, out Etag preTouchEtag, out Etag afterTouchEtag)
417		{
418			if (string.IsNullOrEmpty(key))
419				throw new ArgumentNullException("key");
420
421			var lowerKey = CreateKey(key);
422
423			if (!tableStorage.Documents.Contains(Snapshot, lowerKey, writeBatch.Value))
424			{
425				logger.Debug("Document with dataKey='{0}' was not found", key);
426				preTouchEtag = null;
427				afterTouchEtag = null;
428				return;
429			}
430
431			var metadata = ReadDocumentMetadata(key);
432
433			var newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
434			afterTouchEtag = newEtag;
435			preTouchEtag = metadata.Etag;
436			metadata.Etag = newEtag;
437
438			WriteDocumentMetadata(metadata,shouldIgnoreConcurrencyExceptions:true);
439
440			var keyByEtagIndex = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag);
441
442			keyByEtagIndex.Delete(writeBatch.Value, preTouchEtag);
443			keyByEtagIndex.Add(writeBatch.Value, newEtag, lowerKey);
444            etagTouches.Add(preTouchEtag, afterTouchEtag);
445
446			logger.Debug("TouchDocument() - document with key = '{0}'", key);
447		}
448
449		public Etag GetBestNextDocumentEtag(Etag etag)
450		{
451			if (etag == null) throw new ArgumentNullException("etag");
452
453			using (var iter = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
454													.Iterate(Snapshot, writeBatch.Value))
455			{
456				if (!iter.Seek(etag.ToString()) &&
457					!iter.Seek(Slice.BeforeAllKeys)) //if parameter etag not found, scan from beginning. if empty --> return original etag
458					return etag;
459
460				do
461				{
462					var docEtag = Etag.Parse(iter.CurrentKey.ToString());
463					
464					if (EtagUtil.IsGreaterThan(docEtag, etag))
465						return docEtag;
466				} while (iter.MoveNext());
467			}
468
469			return etag; //if not found, return the original etag
470		}
471
472		private Etag EnsureDocumentEtagMatch(string key, Etag etag, string method)
473		{
474			var metadata = ReadDocumentMetadata(key);
475
476			if (metadata == null)
477				return Etag.InvalidEtag;
478
479			var existingEtag = metadata.Etag;
480
481           
482			if (etag != null)
483			{
484                Etag next;
485                while (etagTouches.TryGetValue(etag, out next))
486                {
487                    etag = next;
488                }
489
490				if (existingEtag != etag)
491				{
492					if (etag == Etag.Empty)
493					{
494						if (metadata.Metadata.ContainsKey(Constants.RavenDeleteMarker) &&
495							metadata.Metadata.Value<bool>(Constants.RavenDeleteMarker))
496						{
497							return existingEtag;
498						}
499					}
500
501					throw new ConcurrencyException(method + " attempted on document '" + key +
502												   "' using a non current etag")
503					{
504						ActualETag = existingEtag,
505						ExpectedETag = etag
506					};
507				}
508			}
509
510			return existingEtag;
511		}
512
513		//returns true if it was update operation
514		private bool WriteDocumentMetadata(JsonDocumentMetadata metadata,bool shouldIgnoreConcurrencyExceptions = false)
515		{
516            var metadataStream = CreateStream();
517
518			metadataStream.Write(metadata.Etag);
519			metadataStream.Write(metadata.Key);
520
521			if (metadata.LastModified.HasValue)
522				metadataStream.Write(metadata.LastModified.Value.ToBinary());
523			else
524				metadataStream.Write((long)0);
525
526			metadata.Metadata.WriteTo(metadataStream);
527
528			metadataStream.Position = 0;
529
530			var loweredKey = CreateKey(metadata.Key);
531
532			ushort? existingVersion;
533			var isUpdate = metadataIndex.Contains(Snapshot, loweredKey, writeBatch.Value, out existingVersion);
534			metadataIndex.Add(writeBatch.Value, loweredKey, metadataStream, existingVersion, shouldIgnoreConcurrencyExceptions);
535
536			return isUpdate;
537		}
538
539		private JsonDocumentMetadata ReadDocumentMetadata(string key)
540		{
541			var loweredKey = CreateKey(key);
542
543			var metadataReadResult = metadataIndex.Read(Snapshot, loweredKey, writeBatch.Value);
544			if (metadataReadResult == null)
545				return null;
546
547			using (var stream = metadataReadResult.Reader.AsStream())
548			{
549				stream.Position = 0;
550				var etag = stream.ReadEtag();
551				var originalKey = stream.ReadString();
552				var lastModifiedDateTimeBinary = stream.ReadInt64();
553
554				var existingCachedDocument = documentCacher.GetCachedDocument(loweredKey, etag);
555
556				var metadata = existingCachedDocument != null ? existingCachedDocument.Metadata : stream.ToJObject();
557			    var lastModified = DateTime.FromBinary(lastModifiedDateTimeBinary);
558
559				return new JsonDocumentMetadata
560				{
561					Key = originalKey,
562					Etag = etag,
563					Metadata = metadata,
564					LastModified = lastModified
565				};
566			}
567		}
568
569		private bool WriteDocumentData(string key, Etag etag, RavenJObject data, RavenJObject metadata, out Etag newEtag, out Etag existingEtag, out DateTime savedAt)
570		{
571			var keyByEtagDocumentIndex = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag);
572			var loweredKey = CreateKey(key);
573
574			ushort? existingVersion;
575			var isUpdate = tableStorage.Documents.Contains(Snapshot, loweredKey, writeBatch.Value, out existingVersion);
576			existingEtag = null;
577
578			if (isUpdate)
579			{
580				existingEtag = EnsureDocumentEtagMatch(loweredKey, etag, "PUT");
581				keyByEtagDocumentIndex.Delete(writeBatch.Value, existingEtag);
582			}
583			else if (etag != null && etag != Etag.Empty)
584			{
585				throw new ConcurrencyException("PUT attempted on document '" + key +
586													   "' using a non current etag (document deleted)")
587				{
588					ExpectedETag = etag
589				};
590			}
591
592            var dataStream = CreateStream();
593
594			using (var finalDataStream = documentCodecs.Aggregate((Stream) new UndisposableStream(dataStream),
595				(current, codec) => codec.Encode(loweredKey, data, metadata, current)))
596			{
597				data.WriteTo(finalDataStream);
598				finalDataStream.Flush();
599			}
600 
601			dataStream.Position = 0;
602			tableStorage.Documents.Add(writeBatch.Value, loweredKey, dataStream, existingVersion ?? 0); 
603
604			newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
605			savedAt = SystemTime.UtcNow;
606
607			var isUpdated = PutDocumentMetadataInternal(key, metadata, newEtag, savedAt);
608
609			keyByEtagDocumentIndex.Add(writeBatch.Value, newEtag, loweredKey);
610
611			return isUpdated;
612		}
613
614		private RavenJObject ReadDocumentData(string key, Etag existingEtag, RavenJObject metadata)
615		{
616			var loweredKey = CreateKey(key);
617
618			var existingCachedDocument = documentCacher.GetCachedDocument(loweredKey, existingEtag);
619			if (existingCachedDocument != null)
620				return existingCachedDocument.Document;
621
622			var documentReadResult = tableStorage.Documents.Read(Snapshot, loweredKey, writeBatch.Value);
623			if (documentReadResult == null) //non existing document
624				return null;
625
626			using (var stream = documentReadResult.Reader.AsStream())
627			{
628				using (var decodedDocumentStream = documentCodecs.Aggregate(stream,
629						(current, codec) => codec.Value.Decode(loweredKey, metadata, current)))
630				{
631					var documentData = decodedDocumentStream.ToJObject();
632
633					documentCacher.SetCachedDocument(loweredKey, existingEtag, documentData, metadata, (int)stream.Length);
634
635					return documentData;	
636				}
637			}
638		}
639
640		public DebugDocumentStats GetDocumentStatsVerySlowly()
641		{
642			var sp = Stopwatch.StartNew();
643			var stat = new DebugDocumentStats { Total = GetDocumentsCount() };
644
645			var documentsByEtag = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag);
646			using (var iterator = documentsByEtag.Iterate(Snapshot, writeBatch.Value))
647			{
648				if (!iterator.Seek(Slice.BeforeAllKeys))
649				{
650					stat.TimeToGenerate = sp.Elapsed;
651					return stat;
652				}
653
654				do
655				{
656					var key = GetKeyFromCurrent(iterator);
657                    var doc = DocumentByKey(key, null);
658                    var size = doc.SerializedSizeOnDisk;
659				    stat.TotalSize += size;
660				    if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase))
661				    {
662                        stat.System++;
663				        stat.SystemSize += size;
664				    }
665						
666
667					var metadata = ReadDocumentMetadata(key);
668
669					var entityName = metadata.Metadata.Value<string>(Constants.RavenEntityName);
670				    if (string.IsNullOrEmpty(entityName))
671				    {
672                        stat.NoCollection++;
673				        stat.NoCollectionSize += size;
674				    }
675				        
676				    else
677				    {
678  
679                        stat.IncrementCollection(entityName, size);
680 				    }
681						
682
683					if (metadata.Metadata.ContainsKey(Constants.RavenDeleteMarker))
684						stat.Tombstones++;
685
686				}
687				while (iterator.MoveNext());
688                var sortedStat = stat.Collections.OrderByDescending(x => x.Value.Size).ToDictionary(x => x.Key, x => x.Value);
689                stat.TimeToGenerate = sp.Elapsed;
690			    stat.Collections = sortedStat;
691                return stat;
692			}
693		}
694	}
695}