/Raven.Database/DocumentDatabase.cs

Large files files are truncated, but you can click here to view the full file

//-----------------------------------------------------------------------

// <copyright file="DocumentDatabase.cs" company="Hibernating Rhinos LTD">

//     Copyright (c) Hibernating Rhinos LTD. All rights reserved.

// </copyright>

//-----------------------------------------------------------------------

using System;

using System.Collections.Concurrent;

using System.Collections.Generic;

using System.ComponentModel.Composition;

using System.Diagnostics;

using System.Globalization;

using System.IO;

using System.Linq;

using System.Runtime.CompilerServices;

using System.Security.Cryptography;

using System.Text;

using System.Threading;

using System.Threading.Tasks;

using Lucene.Net.Search;

using Raven.Abstractions.Logging;

using Raven.Abstractions.Util;

using Raven.Database.Commercial;

using Raven.Database.Impl.DTC;

using Raven.Database.Prefetching;

using Raven.Database.Queries;

using Raven.Database.Server;

using Raven.Database.Server.Connections;

using Raven.Database.Server.Responders.Debugging;

using Raven.Database.Util;

using Raven.Abstractions;

using Raven.Abstractions.Commands;

using Raven.Abstractions.Data;

using Raven.Abstractions.Exceptions;

using Raven.Abstractions.Extensions;

using Raven.Abstractions.Indexing;

using Raven.Abstractions.Linq;

using Raven.Abstractions.MEF;

using Raven.Database.Config;

using Raven.Database.Data;

using Raven.Database.Extensions;

using Raven.Database.Impl;

using Raven.Database.Indexing;

using Raven.Database.Json;

using Raven.Database.Linq;

using Raven.Database.Plugins;

using Raven.Database.Storage;

using Raven.Database.Tasks;

using Constants = Raven.Abstractions.Data.Constants;

using Raven.Json.Linq;

using BitConverter = System.BitConverter;

using Index = Raven.Database.Indexing.Index;

using Task = System.Threading.Tasks.Task;

using TransactionInformation = Raven.Abstractions.Data.TransactionInformation;



namespace Raven.Database

{

    public class DocumentDatabase : IDisposable

    {

        private readonly InMemoryRavenConfiguration configuration;



        [ImportMany]

        public OrderedPartCollection<AbstractRequestResponder> RequestResponders { get; set; }



        [ImportMany]

        public OrderedPartCollection<IStartupTask> StartupTasks { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractAttachmentPutTrigger> AttachmentPutTriggers { get; set; }

        public InFlightTransactionalState InFlightTransactionalState

        {

            get { return inFlightTransactionalState; }

        }



        [ImportMany]

        public OrderedPartCollection<AbstractIndexQueryTrigger> IndexQueryTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractAttachmentDeleteTrigger> AttachmentDeleteTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractAttachmentReadTrigger> AttachmentReadTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractPutTrigger> PutTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractDeleteTrigger> DeleteTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractIndexUpdateTrigger> IndexUpdateTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractReadTrigger> ReadTriggers { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractDynamicCompilationExtension> Extensions { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractIndexCodec> IndexCodecs { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractDocumentCodec> DocumentCodecs { get; set; }



        [ImportMany]

        public OrderedPartCollection<AbstractIndexReaderWarmer> IndexReaderWarmers { get; set; }



        private readonly List<IDisposable> toDispose = new List<IDisposable>();



        private long pendingTaskCounter;

        private readonly ConcurrentDictionary<long, PendingTaskAndState> pendingTasks = new ConcurrentDictionary<long, PendingTaskAndState>();



	    private readonly InFlightTransactionalState inFlightTransactionalState;



        private class PendingTaskAndState

        {

            public Task Task;

            public object State;

        }



        /// <summary>

        /// The name of the database.

        /// Defaults to null for the root database (or embedded database), or the name of the database if this db is a tenant database

        /// </summary>

        public string Name { get; private set; }



        private readonly WorkContext workContext;

        private readonly IndexingExecuter indexingExecuter;

        public IndexingExecuter IndexingExecuter

        {

            get { return indexingExecuter; }

        }



		private readonly Prefetcher prefetcher;

		public Prefetcher Prefetcher

		{

			get { return prefetcher; }

		}



        /// <summary>

        /// Requires to avoid having serialize writes to the same attachments

        /// </summary>

        private readonly ConcurrentDictionary<string, object> putAttachmentSerialLock = new ConcurrentDictionary<string, object>(StringComparer.OrdinalIgnoreCase);



        internal PutSerialLock DocumentLock { get; private set; }



        /// <summary>

        /// This is used to hold state associated with this instance by external extensions

        /// </summary>

        public AtomicDictionary<object> ExtensionsState { get; private set; }



        public TaskScheduler BackgroundTaskScheduler { get { return backgroundTaskScheduler; } }



        private readonly ThreadLocal<bool> disableAllTriggers = new ThreadLocal<bool>(() => false);

        private System.Threading.Tasks.Task indexingBackgroundTask;

        private System.Threading.Tasks.Task reducingBackgroundTask;

        private readonly TaskScheduler backgroundTaskScheduler;

        private readonly object idleLocker = new object();



        private static readonly ILog log = LogManager.GetCurrentClassLogger();



        private readonly SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo> recentTouches;



        public DocumentDatabase(InMemoryRavenConfiguration configuration, TransportState transportState = null)

        {

            DocumentLock = new PutSerialLock();

            this.configuration = configuration;

            this.transportState = transportState ?? new TransportState();

            

            using (LogManager.OpenMappedContext("database", configuration.DatabaseName ?? Constants.SystemDatabase))

            {

                log.Debug("Start loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase);



                if (configuration.IsTenantDatabase == false)

                {

                    validateLicense = new ValidateLicense();

                    validateLicense.Execute(configuration);

                }

                AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit;

                AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit;



                Name = configuration.DatabaseName;

                backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Default;

                

                ExtensionsState = new AtomicDictionary<object>();

                Configuration = configuration;



                ExecuteAlterConfiguration();



                recentTouches = new SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo>(configuration.MaxRecentTouchesToRemember, StringComparer.OrdinalIgnoreCase);



                configuration.Container.SatisfyImportsOnce(this);



                workContext = new WorkContext

                {

                    Database = this,

                    DatabaseName = Name,

                    IndexUpdateTriggers = IndexUpdateTriggers,

                    ReadTriggers = ReadTriggers,

                    RaiseIndexChangeNotification = RaiseNotifications,

                    TaskScheduler = backgroundTaskScheduler,

                    Configuration = configuration,

                    IndexReaderWarmers = IndexReaderWarmers

                };



                TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications);



                try

                {

                    sequentialUuidGenerator = new SequentialUuidGenerator();

                    TransactionalStorage.Initialize(sequentialUuidGenerator, DocumentCodecs);

                }

                catch (Exception)

                {

                    TransactionalStorage.Dispose();

                    throw;

                }



                try

                {



	                inFlightTransactionalState = TransactionalStorage.GetInFlightTransactionalState(Put, Delete);



                    TransactionalStorage.Batch(actions =>

                        sequentialUuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag"));



                    // Index codecs must be initialized before we try to read an index

                    InitializeIndexCodecTriggers();



                    IndexDefinitionStorage = new IndexDefinitionStorage(

                        configuration,

                        TransactionalStorage,

                        configuration.DataDirectory,

                        configuration.Container.GetExportedValues<AbstractViewGenerator>(),

                        Extensions);

                    IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this);



                    CompleteWorkContextSetup();



					prefetcher = new Prefetcher(workContext);

                    indexingExecuter = new IndexingExecuter(workContext, prefetcher);



                    InitializeTriggersExceptIndexCodecs();

                    SecondStageInitialization();



                    ExecuteStartupTasks();

                    log.Debug("Finish loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase);

                }

                catch (Exception)

                {

                    Dispose();

                    throw;

                }

            }

        }



        private void SecondStageInitialization()

        {

            DocumentCodecs.OfType<IRequiresDocumentDatabaseInitialization>()

                .Concat(PutTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(DeleteTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(IndexCodecs.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(IndexQueryTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(AttachmentPutTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(AttachmentDeleteTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(AttachmentReadTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

                .Concat(IndexUpdateTriggers.OfType<IRequiresDocumentDatabaseInitialization>())

            .Apply(initialization => initialization.SecondStageInit());

        }



        private void CompleteWorkContextSetup()

        {

            workContext.IndexStorage = IndexStorage;

            workContext.TransactionalStorage = TransactionalStorage;

            workContext.IndexDefinitionStorage = IndexDefinitionStorage;



            workContext.Init(Name);

        }



        private void DomainUnloadOrProcessExit(object sender, EventArgs eventArgs)

        {

            Dispose();

        }



        private void InitializeTriggersExceptIndexCodecs()

        {

            DocumentCodecs

                //.Init(disableAllTriggers) // Document codecs should always be activated (RavenDB-576)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            PutTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            DeleteTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            ReadTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            IndexQueryTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            AttachmentPutTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            AttachmentDeleteTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            AttachmentReadTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));



            IndexUpdateTriggers

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));

        }



        private void InitializeIndexCodecTriggers()

        {

            IndexCodecs

                .Init(disableAllTriggers)

                .OfType<IRequiresDocumentDatabaseInitialization>().Apply(initialization => initialization.Initialize(this));

        }



        private void ExecuteAlterConfiguration()

        {

            foreach (var alterConfiguration in Configuration.Container.GetExportedValues<IAlterConfiguration>())

            {

                alterConfiguration.AlterConfiguration(Configuration);

            }

        }



        private void ExecuteStartupTasks()

        {

            using (LogContext.WithDatabase(Name))

            {

                foreach (var task in StartupTasks)

                {

                    var disposable = task.Value as IDisposable;

                    if (disposable != null)

                        toDispose.Add(disposable);

                    task.Value.Execute(this);

                }

            }

        }



        public DatabaseStatistics Statistics

        {

            get

            {

                var result = new DatabaseStatistics

                {

                    CurrentNumberOfItemsToIndexInSingleBatch = workContext.CurrentNumberOfItemsToIndexInSingleBatch,

                    CurrentNumberOfItemsToReduceInSingleBatch = workContext.CurrentNumberOfItemsToReduceInSingleBatch,

                    IndexingBatchInfo = workContext.LastActualIndexingBatchInfo.ToArray(),

                    InMemoryIndexingQueueSize = prefetcher.GetInMemoryIndexingQueueSize(PrefetchingUser.Indexer),

                    Prefetches = workContext.FutureBatchStats.OrderBy(x => x.Timestamp).ToArray(),

                    CountOfIndexes = IndexStorage.Indexes.Length,

                    DatabaseTransactionVersionSizeInMB = ConvertBytesToMBs(workContext.TransactionalStorage.GetDatabaseTransactionVersionSizeInBytes()),

                    Errors = workContext.Errors,

                    DatabaseId = TransactionalStorage.Id,

                    Triggers = PutTriggers.Select(x => new DatabaseStatistics.TriggerInfo { Name = x.ToString(), Type = "Put" })

                        .Concat(DeleteTriggers.Select(x => new DatabaseStatistics.TriggerInfo { Name = x.ToString(), Type = "Delete" }))

                        .Concat(ReadTriggers.Select(x => new DatabaseStatistics.TriggerInfo { Name = x.ToString(), Type = "Read" }))

                        .Concat(IndexUpdateTriggers.Select(x => new DatabaseStatistics.TriggerInfo { Name = x.ToString(), Type = "Index Update" }))

                        .ToArray(),

                    Extensions = Configuration.ReportExtensions(

                        typeof(IStartupTask),

                        typeof(AbstractReadTrigger),

                        typeof(AbstractDeleteTrigger),

                        typeof(AbstractPutTrigger),

                        typeof(AbstractDocumentCodec),

                        typeof(AbstractIndexCodec),

                        typeof(AbstractDynamicCompilationExtension),

                        typeof(AbstractIndexQueryTrigger),

                        typeof(AbstractIndexUpdateTrigger),

                        typeof(AbstractAnalyzerGenerator),

                        typeof(AbstractAttachmentDeleteTrigger),

                        typeof(AbstractAttachmentPutTrigger),

                        typeof(AbstractAttachmentReadTrigger),

                        typeof(AbstractBackgroundTask),

                        typeof(IAlterConfiguration)

                        ),

                };

                TransactionalStorage.Batch(actions =>

                {

                    result.LastDocEtag = actions.Staleness.GetMostRecentDocumentEtag();

                    result.LastAttachmentEtag = actions.Staleness.GetMostRecentAttachmentEtag();



                    result.ApproximateTaskCount = actions.Tasks.ApproximateTaskCount;

                    result.CountOfDocuments = actions.Documents.GetDocumentsCount();

                    result.StaleIndexes = IndexStorage.Indexes

                        .Where(s =>

                        {

							var indexInstance = IndexStorage.GetIndexInstance(s);

	                        return (indexInstance != null && indexInstance.IsMapIndexingInProgress) ||

	                               actions.Staleness.IsIndexStale(s, null, null);

                        }).ToArray();

					result.Indexes = actions.Indexing.GetIndexesStats().Where(x => x != null).ToArray();

                });



                if (result.Indexes != null)

                {

                    foreach (var index in result.Indexes)

                    {

	                    try

	                    {

		                    index.LastQueryTimestamp = IndexStorage.GetLastQueryTime(index.Name);

		                    index.Performance = IndexStorage.GetIndexingPerformance(index.Name);

		                    index.IsOnRam = IndexStorage.IndexOnRam(index.Name);

		                    var indexDefinition = IndexDefinitionStorage.GetIndexDefinition(index.Name);

		                    if (indexDefinition != null)

			                    index.LockMode = indexDefinition.LockMode;

		                    index.ForEntityName = IndexDefinitionStorage.GetViewGenerator(index.Name).ForEntityNames.ToList();

		                    IndexSearcher searcher;

		                    using (IndexStorage.GetCurrentIndexSearcher(index.Name, out searcher))

		                    {

			                    index.DocsCount = searcher.IndexReader.NumDocs();

		                    }

		                   

	                    }

	                    catch (Exception)

	                    {

		                    // might happen if the index was deleted mid operation

							// we don't really care for that, so we ignore this

	                    }

                    }

                }



                return result;

            }

        }





        private decimal ConvertBytesToMBs(long bytes)

        {

            return Math.Round(bytes / 1024.0m / 1024.0m, 2);

        }



        public InMemoryRavenConfiguration Configuration

        {

            get;

            private set;

        }



        public ITransactionalStorage TransactionalStorage { get; private set; }



        public IndexDefinitionStorage IndexDefinitionStorage { get; private set; }



        public IndexStorage IndexStorage { get; private set; }



        public event EventHandler Disposing;



        public void Dispose()

        {

            if (disposed)

                return;



            log.Debug("Start shutdown the following database: {0}", Name ?? Constants.SystemDatabase);



            var onDisposing = Disposing;

            if (onDisposing != null)

            {

                try

                {

                    onDisposing(this, EventArgs.Empty);

                }

                catch (Exception e)

                {

                    log.WarnException("Error when notifying about db disposal, ignoring error and continuing with disposal", e);

                }

            }



            var exceptionAggregator = new ExceptionAggregator(log, "Could not properly dispose of DatabaseDocument");



            exceptionAggregator.Execute(() =>

            {

                AppDomain.CurrentDomain.DomainUnload -= DomainUnloadOrProcessExit;

                AppDomain.CurrentDomain.ProcessExit -= DomainUnloadOrProcessExit;

                disposed = true;



                if (workContext != null)

                    workContext.StopWorkRude();

            });



            if (validateLicense != null)

                exceptionAggregator.Execute(validateLicense.Dispose);



            exceptionAggregator.Execute(() =>

            {

                if (ExtensionsState == null)

                    return;



                foreach (var value in ExtensionsState.Values.OfType<IDisposable>())

                {

                    exceptionAggregator.Execute(value.Dispose);

                }

            });



            exceptionAggregator.Execute(() =>

            {

                if (toDispose == null)

                    return;

                foreach (var shouldDispose in toDispose)

                {

                    exceptionAggregator.Execute(shouldDispose.Dispose);

                }

            });





            exceptionAggregator.Execute(() =>

            {

                foreach (var shouldDispose in pendingTasks)

                {

					var pendingTaskAndState = shouldDispose.Value;

					exceptionAggregator.Execute(() =>

                    {

	                    try

	                    {

		                    pendingTaskAndState.Task.Wait();

	                    }

	                    catch (Exception)

	                    {

		                    // we explictly don't care about this during shutdown

	                    }

                    });

                }

                pendingTasks.Clear();

            });



            exceptionAggregator.Execute(() =>

            {

                if (indexingBackgroundTask != null)

                    indexingBackgroundTask.Wait();

            });

            exceptionAggregator.Execute(() =>

            {

                if (reducingBackgroundTask != null)

                    reducingBackgroundTask.Wait();

            });



            exceptionAggregator.Execute(() =>

            {

                var disposable = backgroundTaskScheduler as IDisposable;

                if (disposable != null)

                    disposable.Dispose();

            });



            if (TransactionalStorage != null)

                exceptionAggregator.Execute(TransactionalStorage.Dispose);

            if (IndexStorage != null)

                exceptionAggregator.Execute(IndexStorage.Dispose);



            if (Configuration != null)

                exceptionAggregator.Execute(Configuration.Dispose);



            exceptionAggregator.Execute(disableAllTriggers.Dispose);



            if (workContext != null)

                exceptionAggregator.Execute(workContext.Dispose);



            exceptionAggregator.ThrowIfNeeded();



            log.Debug("Finished shutdown the following database: {0}", Name ?? Constants.SystemDatabase);

        }



        public void StopBackgroundWorkers()

        {

            workContext.StopWork();

            if (indexingBackgroundTask != null)

                indexingBackgroundTask.Wait();

            if (reducingBackgroundTask != null)

                reducingBackgroundTask.Wait();



            backgroundWorkersSpun = false;

        }



        public void StopIndexingWorkers()

        {

            workContext.StopIndexing();

            try

            {

                indexingBackgroundTask.Wait();

            }

            catch (Exception e)

            {

                log.WarnException("Error while trying to stop background indexing", e);

            }

            try

            {

                reducingBackgroundTask.Wait();

            }

            catch (Exception e)

            {

                log.WarnException("Error while trying to stop background reducing", e);

            }



            backgroundWorkersSpun = false;

        }



        public WorkContext WorkContext

        {

            get { return workContext; }

        }



        private volatile bool backgroundWorkersSpun;



        public void SpinBackgroundWorkers()

        {

            if (backgroundWorkersSpun)

                throw new InvalidOperationException("The background workers has already been spun and cannot be spun again");



            backgroundWorkersSpun = true;



            workContext.StartWork();

            indexingBackgroundTask = Task.Factory.StartNew(

                indexingExecuter.Execute,

                CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);

            reducingBackgroundTask = Task.Factory.StartNew(

                new ReducingExecuter(workContext).Execute,

                CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);

        }



        public void SpinIndexingWorkers()

        {

            if (backgroundWorkersSpun)

                throw new InvalidOperationException("The background workers has already been spun and cannot be spun again");



            backgroundWorkersSpun = true;



            workContext.StartIndexing();

            indexingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(

                indexingExecuter.Execute,

                CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);

            reducingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(

                new ReducingExecuter(workContext).Execute,

                CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);

        }



        public void RaiseNotifications(DocumentChangeNotification obj, RavenJObject metadata)

        {

            TransportState.Send(obj);

            var onDocumentChange = OnDocumentChange;

            if (onDocumentChange != null)

                onDocumentChange(this, obj, metadata);

        }



        public void RaiseNotifications(IndexChangeNotification obj)

        {

            TransportState.Send(obj);

        }



        public void RaiseNotifications(ReplicationConflictNotification obj)

        {

            TransportState.Send(obj);

        }



		public void RaiseNotifications(BulkInsertChangeNotification obj)

		{

			TransportState.Send(obj);

		}



        public event Action<DocumentDatabase, DocumentChangeNotification, RavenJObject> OnDocumentChange;



        public void RunIdleOperations()

        {

            var tryEnter = Monitor.TryEnter(idleLocker);

            try

            {

                if (tryEnter == false)

                    return;

                TransportState.OnIdle();

                IndexStorage.RunIdleOperations();

                ClearCompletedPendingTasks();

            }

            finally

            {

                if (tryEnter)

                    Monitor.Exit(idleLocker);

            }

        }



        private void ClearCompletedPendingTasks()

        {

            foreach (var taskAndState in pendingTasks)

            {

                var task = taskAndState.Value.Task;

                if (task.IsCompleted || task.IsCanceled || task.IsFaulted)

                {

                    PendingTaskAndState value;

                    pendingTasks.TryRemove(taskAndState.Key, out value);

                }

                if (task.Exception != null)

                {

                    log.InfoException("Failed to execute background task " + taskAndState.Key, task.Exception);

                }

            }

        }



        public JsonDocument Get(string key, TransactionInformation transactionInformation)

        {

            if (key == null)

                throw new ArgumentNullException("key");

            key = key.Trim();



            JsonDocument document = null;

            if (transactionInformation == null ||

                inFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)

            {

                // first we check the dtc state, then the storage, to avoid race conditions

                var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(transactionInformation, key);



                TransactionalStorage.Batch(actions => { document = actions.Documents.DocumentByKey(key, transactionInformation); });



                if (nonAuthoritativeInformationBehavior != null)

                    document = nonAuthoritativeInformationBehavior(document);

            }



            DocumentRetriever.EnsureIdInMetadata(document);



            return new DocumentRetriever(null, ReadTriggers, inFlightTransactionalState)

                .ExecuteReadTriggers(document, transactionInformation, ReadOperation.Load);

        }



        public JsonDocumentMetadata GetDocumentMetadata(string key, TransactionInformation transactionInformation)

        {

            if (key == null)

                throw new ArgumentNullException("key");

            key = key.Trim();

            JsonDocumentMetadata document = null;

            if (transactionInformation == null ||

                inFlightTransactionalState.TryGet(key, transactionInformation, out document) == false)

            {

                var nonAuthoritativeInformationBehavior = inFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocumentMetadata>(transactionInformation, key);

                TransactionalStorage.Batch(actions =>

                {

                    document = actions.Documents.DocumentMetadataByKey(key, transactionInformation);

                });

                if (nonAuthoritativeInformationBehavior != null)

                    document = nonAuthoritativeInformationBehavior(document);

            }



            DocumentRetriever.EnsureIdInMetadata(document);

            return new DocumentRetriever(null, ReadTriggers, inFlightTransactionalState)

                .ProcessReadVetoes(document, transactionInformation, ReadOperation.Load);

        }



        public PutResult Put(string key, Etag etag, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)

        {

            workContext.PerformanceCounters.DocsPerSecond.Increment();

            key = string.IsNullOrWhiteSpace(key) ? Guid.NewGuid().ToString() : key.Trim();

            RemoveReservedProperties(document);

            RemoveMetadataReservedProperties(metadata);

            Etag newEtag = Etag.Empty;



            using (DocumentLock.Lock())

            {

                TransactionalStorage.Batch(actions =>

                {

                    if (key.EndsWith("/"))

                    {

                        key += GetNextIdentityValueWithoutOverwritingOnExistingDocuments(key, actions,

                                                                                         transactionInformation);

                    }

                    AssertPutOperationNotVetoed(key, metadata, document, transactionInformation);

                    if (transactionInformation == null)

                    {

                        if (inFlightTransactionalState.IsModified(key))

                            throw new ConcurrencyException("PUT attempted on : " + key +

                                                           " while it is being locked by another transaction");



                        PutTriggers.Apply(trigger => trigger.OnPut(key, document, metadata, null));



                        var addDocumentResult = actions.Documents.AddDocument(key, etag, document, metadata);

                        newEtag = addDocumentResult.Etag;



                        CheckReferenceBecauseOfDocumentUpdate(key, actions);

                        metadata[Constants.LastModified] = addDocumentResult.SavedAt;

                        metadata.EnsureSnapshot(

                            "Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");

                        document.EnsureSnapshot(

                            "Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");



                        actions.AfterStorageCommitBeforeWorkNotifications(new JsonDocument

                        {

                            Metadata = metadata,

                            Key = key,

                            DataAsJson = document,

                            Etag = newEtag,

                            LastModified = addDocumentResult.SavedAt,

                            SkipDeleteFromIndex = addDocumentResult.Updated == false

                        }, documents => prefetcher.AfterStorageCommitBeforeWorkNotifications(PrefetchingUser.Indexer, documents));



						if (addDocumentResult.Updated)

							prefetcher.AfterUpdate(key, addDocumentResult.PrevEtag);



                        PutTriggers.Apply(trigger => trigger.AfterPut(key, document, metadata, newEtag, null));



                        TransactionalStorage

                            .ExecuteImmediatelyOrRegisterForSynchronization(() =>

                            {

                                PutTriggers.Apply(trigger => trigger.AfterCommit(key, document, metadata, newEtag));

                                RaiseNotifications(new DocumentChangeNotification

                                {

                                    Id = key,

                                    Type = DocumentChangeTypes.Put,

                                    Etag = newEtag,

                                }, metadata);

                            });



                        workContext.ShouldNotifyAboutWork(() => "PUT " + key);

                    }

                    else

                    {

                        var doc = actions.Documents.DocumentMetadataByKey(key, null);

                        newEtag = inFlightTransactionalState.AddDocumentInTransaction(key, etag, document, metadata,

                                                                                      transactionInformation,

                                                                                      doc == null

                                                                                          ? Etag.Empty

                                                                                          : doc.Etag,

                                                                                      sequentialUuidGenerator);

                    }

                });



                log.Debug("Put document {0} with etag {1}", key, newEtag);



                return new PutResult

                {

                    Key = key,

                    ETag = newEtag

                };

            }

        }



        internal void CheckReferenceBecauseOfDocumentUpdate(string key, IStorageActionsAccessor actions)

        {

            TouchedDocumentInfo touch;

            recentTouches.TryRemove(key, out touch);



            foreach (var referencing in actions.Indexing.GetDocumentsReferencing(key))

            {

                Etag preTouchEtag;

                Etag afterTouchEtag;

                try

                {

                    actions.Documents.TouchDocument(referencing, out preTouchEtag, out afterTouchEtag);

                }

                catch (ConcurrencyException)

                {

                    continue;

                }



                if (preTouchEtag == null || afterTouchEtag == null)

                    continue;



                actions.General.MaybePulseTransaction();



                recentTouches.Set(referencing, new TouchedDocumentInfo

                {

                    PreTouchEtag = preTouchEtag,

                    TouchedEtag = afterTouchEtag

                });

            }

        }



        public long GetNextIdentityValueWithoutOverwritingOnExistingDocuments(string key,

            IStorageActionsAccessor actions,

            TransactionInformation transactionInformation)

        {

            int tries;

            return GetNextIdentityValueWithoutOverwritingOnExistingDocuments(key, actions, transactionInformation, out tries);

        }



        public long GetNextIdentityValueWithoutOverwritingOnExistingDocuments(string key,

            IStorageActionsAccessor actions,

            TransactionInformation transactionInformation,

            out int tries)

        {

            long nextIdentityValue = actions.General.GetNextIdentityValue(key);



            if (actions.Documents.DocumentMetadataByKey(key + nextIdentityValue, transactionInformation) == null)

            {

                tries = 1;

                return nextIdentityValue;

            }

            tries = 1;

            // there is already a document with this id, this means that we probably need to search

            // for an opening in potentially large data set. 

            var lastKnownBusy = nextIdentityValue;

            var maybeFree = nextIdentityValue * 2;

            var lastKnownFree = long.MaxValue;

            while (true)

            {

                tries++;

                if (actions.Documents.DocumentMetadataByKey(key + maybeFree, transactionInformation) == null)

                {

                    if (lastKnownBusy + 1 == maybeFree)

                    {

                        actions.General.SetIdentityValue(key, maybeFree);

                        return maybeFree;

                    }

                    lastKnownFree = maybeFree;

                    maybeFree = Math.Max(maybeFree - (maybeFree - lastKnownBusy) / 2, lastKnownBusy + 1);



                }

                else

                {

                    lastKnownBusy = maybeFree;

                    maybeFree = Math.Min(lastKnownFree, maybeFree * 2);

                }

            }

        }



        private void AssertPutOperationNotVetoed(string key, RavenJObject metadata, RavenJObject document, TransactionInformation transactionInformation)

        {

            var vetoResult = PutTriggers

                .Select(trigger => new { Trigger = trigger, VetoResult = trigger.AllowPut(key, document, metadata, transactionInformation) })

                .FirstOrDefault(x => x.VetoResult.IsAllowed == false);

            if (vetoResult != null)

            {

                throw new OperationVetoedException("PUT vetoed on document " + key + " by " + vetoResult.Trigger + " because: " + vetoResult.VetoResult.Reason);

            }

        }



        private void AssertAttachmentPutOperationNotVetoed(string key, RavenJObject metadata, Stream data)

        {

            var vetoResult = AttachmentPutTriggers

                .Select(trigger => new { Trigger = trigger, VetoResult = trigger.AllowPut(key, data, metadata) })

                .FirstOrDefault(x => x.VetoResult.IsAllowed == false);

            if (vetoResult != null)

            {

	            throw new OperationVetoedException("PUT vetoed on attachment " + key + " by " + vetoResult.Trigger +

	                                               " because: " + vetoResult.VetoResult.Reason);

            }

        }



        private void AssertAttachmentDeleteOperationNotVetoed(string key)

        {

            var vetoResult = AttachmentDeleteTriggers

                .Select(trigger => new { Trigger = trigger, VetoResult = trigger.AllowDelete(key) })

                .FirstOrDefault(x => x.VetoResult.IsAllowed == false);

            if (vetoResult != null)

            {

	            throw new OperationVetoedException("DELETE vetoed on attachment " + key + " by " + vetoResult.Trigger +

	                                               " because: " + vetoResult.VetoResult.Reason);

            }

        }



        private void AssertDeleteOperationNotVetoed(string key, TransactionInformation transactionInformation)

        {

            var vetoResult = DeleteTriggers

                .Select(trigger => new { Trigger = trigger, VetoResult = trigger.AllowDelete(key, transactionInformation) })

                .FirstOrDefault(x => x.VetoResult.IsAllowed == false);

            if (vetoResult != null)

            {

	            throw new OperationVetoedException("DELETE vetoed on document " + key + " by " + vetoResult.Trigger +

	                                               " because: " + vetoResult.VetoResult.Reason);

            }

        }



        private static void RemoveMetadataReservedProperties(RavenJObject metadata)

        {

            RemoveReservedProperties(metadata);

            metadata.Remove("Raven-Last-Modified");

            metadata.Remove("Last-Modified");

        }



        private static void RemoveReservedProperties(RavenJObject document)

        {

            document.Remove(string.Empty);

            var toRemove = document.Keys.Where(propertyName => propertyName.StartsWith("@") || headersToIgnoreServer.Contains(propertyName)).ToList();

            foreach (var propertyName in toRemove)

            {

                document.Remove(propertyName);

            }

        }



        private static readonly HashSet<string> headersToIgnoreServer = new HashSet<string>(StringComparer.OrdinalIgnoreCase)

		{

			Constants.RavenLastModified,

		};



        public bool Delete(string key, Etag etag, TransactionInformation transactionInformation)

        {

            RavenJObject metadata;

            return Delete(key, etag, transactionInformation, out metadata);

        }



        public bool Delete(string key, Etag etag, TransactionInformation transactionInformation, out RavenJObject metadata)

        {

            if (key == null)

                throw new ArgumentNullException("key");

            key = key.Trim();



            var deleted = false;

            log.Debug("Delete a document with key: {0} and etag {1}", key, etag);

            RavenJObject metadataVar = null;



            using (DocumentLock.Lock())

            {

                TransactionalStorage.Batch(actions =>

                {

                    AssertDeleteOperationNotVetoed(key, transactionInformation);

                    if (transactionInformation == null)

                    {

                        DeleteTriggers.Apply(trigger => trigger.OnDelete(key, null));



                        Etag deletedETag;

                        if (actions.Documents.DeleteDocument(key, etag, out metadataVar, out deletedETag))

                        {

                            deleted = true;

                            actions.Indexing.RemoveAllDocumentReferencesFrom(key);

                            WorkContext.MarkDeleted(key);



                            CheckReferenceBecauseOfDocumentUpdate(key, actions);



                            foreach (var indexName in IndexDefinitionStorage.IndexNames)

                            {

                                AbstractViewGenerator abstractViewGenerator =

                                    IndexDefinitionStorage.GetViewGenerator(indexName);

                                if (abstractViewGenerator == null)

                                    continue;



                                var token = metadataVar.Value<string>(Constants.RavenEntityName);



                                if (token != null && // the document has a entity name

                                    abstractViewGenerator.ForEntityNames.Count > 0)

                                    // the index operations on specific entities

                                {

                                    if (abstractViewGenerator.ForEntityNames.Contains(token) == false)

                                        continue;

                                }



                                string indexNameCopy = indexName;

                                var task = actions.GetTask(x => x.Index == indexNameCopy, new RemoveFromIndexTask

                                {

                                    Index = indexNameCopy

                                });

                                task.Keys.Add(key);

                            }

                            if (deletedETag != null)

                                prefetcher.AfterDelete(key, deletedETag);

                            DeleteTriggers.Apply(trigger => trigger.AfterDelete(key, null));

                        }



                        TransactionalStorage

                            .ExecuteImmediatelyOrRegisterForSynchronization(() =>

                            {

                                DeleteTriggers.Apply(trigger => trigger.AfterCommit(key));

                                RaiseNotifications(new DocumentChangeNotification

                                {

                                    Id = key,

                                    Type = DocumentChangeTypes.Delete,

                                }, metadataVar);

                            });



                    }

                    else

                    {

                        var doc = actions.Documents.DocumentMetadataByKey(key, null);



                        inFlightTransactionalState.DeleteDocumentInTransaction(transactionInformation, key,

                                                                               etag,

                                                                               doc == null ? Etag.Empty : doc.Etag,

                                                                               sequentialUuidGenerator);

                        deleted = doc != null;

                    }



                    workContext.ShouldNotifyAboutWork(() => "DEL " + key);

                });



                metadata = metadataVar;

                return deleted;

            }

        }



        public bool HasTransaction(string txId)

        {

            return inFlightTransactionalState.HasTransaction(txId);

        }



		public void PrepareTransaction(string txId)

		{

			try

			{

				inFlightTransactionalState.Prepare(txId);

				log.Debug("Prepare of tx {0} completed", txId);

			}

			catch (Exception e)

			{

				if (TransactionalStorage.HandleException(e))

					return;

				throw;

			}

		}



        public void Commit(string txId)

        {

            try

            {

                using (DocumentLock.Lock())

                {

                    try

                    {

                        inFlightTransactionalState.Commit(txId);

                        log.Debug("Commit of tx {0} completed", txId);

                        workContext.ShouldNotifyAboutWork(() => "DTC transaction commited");

                    }

                    finally

                    {

                        inFlightTransactionalState.Rollback(txId); // this is where we actually remove the tx

                    }

                }

            }

            catch (Exception e)

            {

	            if (TransactionalStorage.HandleException(e))

		            return;

	            throw;

            }

            finally

            {

	            workContext.HandleWorkNotifications();

            }

        }





        public void Rollback(string txId)

        {

            inFlightTransactionalState.Rollback(txId);

        }



        [MethodImpl(MethodImplOptions.Synchronized)]

        public string PutTransform(string name, TransformerDefinition definition)

        {

            if (name == null) throw new ArgumentNullException("name");

            if (definition == null) throw new ArgumentNullException("definition");



            name = name.Trim();



            var existingDefinition = IndexDefinitionStorage.GetTransformerDefinition(name);

            if (existingDefinition != null && existingDefinition.Equals(definition))

                return name; // no op for the same transformer



            IndexDefinitionStorage.CreateAndPersistTransform(definition);

            IndexDefinitionStorage.AddTransform(name, definition);



            return name;

        }



        // only one index can be created at any given time

        // the method already handle attempts to create the same index, so we don't have to 

        // worry about this.

        [MethodImpl(MethodImplOptions.Synchro…

Large files files are truncated, but you can click here to view the full file