PageRenderTime 37ms CodeModel.GetById 3ms app.highlight 26ms RepoModel.GetById 1ms app.codeStats 0ms

/ToMigrate/Raven.Database/Actions/IndexActions.cs

http://github.com/ayende/ravendb
C# | 869 lines | 704 code | 122 blank | 43 comment | 107 complexity | 5bff0fcbc564d25be91929303d8c4280 MD5 | raw file
  1// -----------------------------------------------------------------------
  2//  <copyright file="IndexActions.cs" company="Hibernating Rhinos LTD">
  3//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
  4//  </copyright>
  5// -----------------------------------------------------------------------
  6using System;
  7using System.Collections.Generic;
  8using System.Diagnostics;
  9using System.Globalization;
 10using System.IO;
 11using System.Linq;
 12using System.Runtime.CompilerServices;
 13using System.Text;
 14using System.Threading;
 15using System.Threading.Tasks;
 16
 17using Raven.Abstractions;
 18using Raven.Abstractions.Data;
 19using Raven.Abstractions.Exceptions;
 20using Raven.Abstractions.Indexing;
 21using Raven.Abstractions.Logging;
 22using Raven.Abstractions.Util.Encryptors;
 23using Raven.Database.Data;
 24using Raven.Database.Extensions;
 25using Raven.Database.Impl;
 26using Raven.Database.Indexing;
 27using Raven.Database.Linq;
 28using Raven.Database.Queries;
 29using Raven.Database.Storage;
 30using Raven.Database.Util;
 31using Raven.Json.Linq;
 32using Sparrow;
 33
 34namespace Raven.Database.Actions
 35{
 36    public class IndexActions : ActionsBase
 37    {
 38        private volatile bool isPrecomputedBatchForNewIndexIsRunning;
 39        private readonly object precomputedLock = new object();
 40
 41        public IndexActions(DocumentDatabase database, SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo> recentTouches, IUuidGenerator uuidGenerator, ILog log)
 42            : base(database, recentTouches, uuidGenerator, log)
 43        {
 44        }
 45
 46        internal IndexDefinition[] Definitions
 47        {
 48            get { return Database.IndexDefinitionStorage.IndexDefinitions.Select(inx => inx.Value).ToArray(); }
 49        }
 50
 51        public string[] GetIndexFields(string index)
 52        {
 53            var abstractViewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
 54            return abstractViewGenerator == null ? new string[0] : abstractViewGenerator.Fields;
 55        }
 56
 57        public Etag GetIndexEtag(string indexName, Etag previousEtag, string resultTransformer = null)
 58        {
 59            Etag lastDocEtag = Etag.Empty;
 60            Etag lastIndexedEtag = null;
 61            Etag lastReducedEtag = null;
 62            bool isStale = false;
 63            int touchCount = 0;
 64            TransactionalStorage.Batch(accessor =>
 65            {
 66                var indexInstance = Database.IndexStorage.GetIndexInstance(indexName);
 67                if (indexInstance == null)
 68                    return;
 69                isStale = (indexInstance.IsMapIndexingInProgress) ||
 70                          accessor.Staleness.IsIndexStale(indexInstance.indexId, null, null);
 71                lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
 72                var indexStats = accessor.Indexing.GetIndexStats(indexInstance.indexId);
 73                if (indexStats != null)
 74                {
 75                    lastReducedEtag = indexStats.LastReducedEtag;
 76                    lastIndexedEtag = indexStats.LastIndexedEtag;
 77                }
 78                touchCount = accessor.Staleness.GetIndexTouchCount(indexInstance.indexId);
 79            });
 80
 81
 82            var indexDefinition = GetIndexDefinition(indexName);
 83            if (indexDefinition == null)
 84                return Etag.Empty; // this ensures that we will get the normal reaction of IndexNotFound later on.
 85
 86            var list = new List<byte>();
 87            list.AddRange(indexDefinition.GetIndexHash());
 88            list.AddRange(Encoding.Unicode.GetBytes(indexName));
 89            if (string.IsNullOrWhiteSpace(resultTransformer) == false)
 90            {
 91                var abstractTransformer = IndexDefinitionStorage.GetTransformer(resultTransformer);
 92                if (abstractTransformer == null)
 93                    throw new InvalidOperationException("The result transformer: " + resultTransformer + " was not found");
 94                list.AddRange(abstractTransformer.GetHashCodeBytes());
 95            }
 96            list.AddRange(lastDocEtag.ToByteArray());
 97            list.AddRange(BitConverter.GetBytes(touchCount));
 98            list.AddRange(BitConverter.GetBytes(isStale));
 99            if (lastReducedEtag != null)
100            {
101                list.AddRange(lastReducedEtag.ToByteArray());
102            }
103            if (lastIndexedEtag != null)
104            {
105                list.AddRange(lastIndexedEtag.ToByteArray());
106            }
107            list.AddRange(BitConverter.GetBytes(UuidGenerator.LastDocumentTransactionEtag));
108
109            var indexEtag = Etag.FromHash(Hashing.Metro128.Calculate(list.ToArray()));
110
111            if (previousEtag != null && previousEtag != indexEtag)
112            {
113                // the index changed between the time when we got it and the time 
114                // we actually call this, we need to return something random so that
115                // the next time we won't get 304
116
117                return Etag.InvalidEtag;
118            }
119
120            return indexEtag;
121        }
122
123        internal void CheckReferenceBecauseOfDocumentUpdate(string key, IStorageActionsAccessor actions, string[] participatingIds = null)
124        {
125            TouchedDocumentInfo touch;
126            RecentTouches.TryRemove(key, out touch);
127            Stopwatch sp = null;
128            int count = 0;
129
130            using (Database.TransactionalStorage.DisableBatchNesting())
131            {
132                // in external transaction number of references will be >= from current transaction references
133                Database.TransactionalStorage.Batch(externalActions =>
134                {
135                    var referencingKeys = externalActions.Indexing.GetDocumentsReferencing(key);
136                    if (participatingIds != null)
137                        referencingKeys = referencingKeys.Except(participatingIds);
138
139                    foreach (var referencing in referencingKeys)
140                    {
141                        Etag preTouchEtag = null;
142                        Etag afterTouchEtag = null;
143                        try
144                        {
145                            count++;
146                            actions.Documents.TouchDocument(referencing, out preTouchEtag, out afterTouchEtag);
147
148                            if (afterTouchEtag != null)
149                            {
150                                var docMetadata = actions.Documents.DocumentMetadataByKey(referencing);
151
152                                if (docMetadata != null)
153                                {
154                                    var entityName = docMetadata.Metadata.Value<string>(Constants.RavenEntityName);
155
156                                    if (string.IsNullOrEmpty(entityName) == false) 
157                                        Database.LastCollectionEtags.Update(entityName, afterTouchEtag);
158                                }
159                            }
160                        }
161                        catch (ConcurrencyException)
162                        {
163                        }
164
165                        if (preTouchEtag == null || afterTouchEtag == null)
166                            continue;
167
168                        if (actions.General.MaybePulseTransaction())
169                        {
170                            if (sp == null)
171                                sp = Stopwatch.StartNew();
172                            if (sp.Elapsed >= TimeSpan.FromSeconds(30))
173                            {
174                                throw new TimeoutException("Early failure when checking references for document '" + key + "', we waited over 30 seconds to touch all of the documents referenced by this document.\r\n" +
175                                                           "The operation (and transaction) has been aborted, since to try longer (we already touched " + count + " documents) risk a thread abort.\r\n" +
176                                                           "Consider restructuring your indexes to avoid LoadDocument on such a popular document.");
177                            }
178                        }
179
180                        RecentTouches.Set(referencing, new TouchedDocumentInfo
181                        {
182                            PreTouchEtag = preTouchEtag,
183                            TouchedEtag = afterTouchEtag
184                        });
185                    }
186                });
187            }
188        }
189
190        private static void IsIndexNameValid(string name)
191        {
192            var error = string.Format("Index name {0} not permitted. ", name).Replace("//", "__");
193
194            if (name.StartsWith("dynamic/", StringComparison.OrdinalIgnoreCase))
195            {
196                throw new ArgumentException(error + "Index names starting with dynamic_ or dynamic/ are reserved!", "name");
197            }
198
199            if (name.Equals("dynamic", StringComparison.OrdinalIgnoreCase))
200            {
201                throw new ArgumentException(error + "Index name dynamic is reserved!", "name");
202            }
203
204            if (name.Contains("//"))
205            {
206                throw new ArgumentException(error + "Index name cannot contain // (double slashes)", "name");
207            }
208        }
209
210        public bool IndexHasChanged(string name, IndexDefinition definition)
211        {
212            if (name == null)
213                throw new ArgumentNullException("name");
214
215            name = name.Trim();
216            IsIndexNameValid(name);
217
218            var existingIndex = IndexDefinitionStorage.GetIndexDefinition(name);
219            if (existingIndex == null)
220                return true;
221
222            var creationOption = FindIndexCreationOptions(definition, ref name);
223            return creationOption != IndexCreationOptions.Noop;
224        }
225
226        // only one index can be created at any given time
227        // the method already handle attempts to create the same index, so we don't have to 
228        // worry about this.
229        [MethodImpl(MethodImplOptions.Synchronized)]
230        public string PutIndex(string name, IndexDefinition definition)
231        {
232            return PutIndexInternal(name, definition);
233        }
234
235        private string PutIndexInternal(string name, IndexDefinition definition, bool disableIndexBeforePut = false, bool isUpdateBySideSide = false, IndexCreationOptions? creationOptions = null)
236        {
237            if (name == null)
238                throw new ArgumentNullException("name");
239
240            name = name.Trim();
241            IsIndexNameValid(name);
242
243            var existingIndex = IndexDefinitionStorage.GetIndexDefinition(name);
244            if (existingIndex != null)
245            {
246                switch (existingIndex.LockMode)
247                {
248                    case IndexLockMode.SideBySide:
249                        if (isUpdateBySideSide == false)
250                        {
251                            Log.Info("Index {0} not saved because it might be only updated by side-by-side index");
252                            throw new InvalidOperationException("Can not overwrite locked index: " + name + ". This index can be only updated by side-by-side index.");
253                        }
254                        break;
255                    case IndexLockMode.LockedIgnore:
256                        Log.Info("Index {0} not saved because it was lock (with ignore)", name);
257                        return null;
258
259                    case IndexLockMode.LockedError:
260                        throw new InvalidOperationException("Can not overwrite locked index: " + name);
261                }
262            }
263
264            AssertAnalyzersValid(definition);
265
266            switch (creationOptions ?? FindIndexCreationOptions(definition, ref name))
267            {
268                case IndexCreationOptions.Noop:
269                    return null;
270                case IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex:
271                    // ensure that the code can compile
272                    new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance();
273                    IndexDefinitionStorage.UpdateIndexDefinitionWithoutUpdatingCompiledIndex(definition);
274                    return null;
275                case IndexCreationOptions.Update:
276                    // ensure that the code can compile
277                    new DynamicViewCompiler(definition.Name, definition, Database.Extensions, IndexDefinitionStorage.IndexDefinitionsPath, Database.Configuration).GenerateInstance();
278                    DeleteIndex(name);
279                    break;
280            }
281
282            PutNewIndexIntoStorage(name, definition, disableIndexBeforePut);
283
284            WorkContext.ClearErrorsFor(name);
285
286            TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new IndexChangeNotification
287            {
288                Name = name,
289                Type = IndexChangeTypes.IndexAdded,
290                Version = definition.IndexVersion
291            }));
292
293            return name;
294        }
295
296        [MethodImpl(MethodImplOptions.Synchronized)]
297        public string[] PutIndexes(IndexToAdd[] indexesToAdd)
298        {
299            var createdIndexes = new List<string>();
300            var prioritiesList = new List<IndexingPriority>();
301            try
302            {
303                foreach (var indexToAdd in indexesToAdd)
304                {
305                    var nameToAdd = PutIndexInternal(indexToAdd.Name, indexToAdd.Definition, disableIndexBeforePut: true);
306                    if (nameToAdd == null)
307                        continue;
308
309                    createdIndexes.Add(nameToAdd);
310                    prioritiesList.Add(indexToAdd.Priority);
311                }
312
313                var indexesIds = createdIndexes.Select(x => Database.IndexStorage.GetIndexInstance(x).indexId).ToArray();
314                Database.TransactionalStorage.Batch(accessor => accessor.Indexing.SetIndexesPriority(indexesIds, prioritiesList.ToArray()));
315            
316                for (var i = 0; i < createdIndexes.Count; i++)
317                {
318                    var index = createdIndexes[i];
319                    var priority = prioritiesList[i];
320
321                    var instance = Database.IndexStorage.GetIndexInstance(index);
322                    instance.Priority = priority;
323                }
324
325                return createdIndexes.ToArray();
326            }
327            catch (Exception e)
328            {
329                Log.WarnException("Could not create index batch", e);
330                foreach (var index in createdIndexes)
331                {
332                    DeleteIndex(index);
333                }
334                throw;
335            }
336        }
337
338        [MethodImpl(MethodImplOptions.Synchronized)]
339        public SideBySideIndexInfo[] PutSideBySideIndexes(IndexToAdd[] indexesToAdd)
340        {
341            var createdIndexes = new List<SideBySideIndexInfo>();
342            var prioritiesList = new List<IndexingPriority>();
343            try
344            {
345                foreach (var indexToAdd in indexesToAdd)
346                {
347                    var originalIndexName = indexToAdd.Name.Trim();
348                    var indexName = Constants.SideBySideIndexNamePrefix + originalIndexName;
349                    var isSideBySide = true;
350
351                    IndexCreationOptions? creationOptions = null;
352                    //if there is no existing side by side index, we might need to update the old index
353                    if (IndexDefinitionStorage.GetIndexDefinition(indexName) == null)
354                    {
355                        var originalIndexCreationOptions = FindIndexCreationOptions(indexToAdd.Definition, ref originalIndexName);
356                        switch (originalIndexCreationOptions)
357                        {
358                            case IndexCreationOptions.Noop:
359                                continue;
360                            case IndexCreationOptions.Create:
361                            case IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex:
362                                //cases in which we don't need to create a side by side index:
363                                //1) index doesn't exist => need to create a new regular index
364                                //2) there is an existing index and we need to update its definition without reindexing
365                                indexName = originalIndexName;
366                                isSideBySide = false;
367                                creationOptions = originalIndexCreationOptions;
368                                break;
369                        }
370                    }
371
372                    var nameToAdd = PutIndexInternal(indexName, indexToAdd.Definition, disableIndexBeforePut: true, isUpdateBySideSide: true, creationOptions: creationOptions);
373                    if (nameToAdd == null)
374                        continue;
375
376                    createdIndexes.Add(new SideBySideIndexInfo
377                    {
378                        OriginalName = originalIndexName,
379                        Name = nameToAdd,
380                        IsSideBySide = isSideBySide
381                    });
382                    prioritiesList.Add(indexToAdd.Priority);
383                }
384
385                var indexesIds = createdIndexes.Select(x => Database.IndexStorage.GetIndexInstance(x.Name).indexId).ToArray();
386                Database.TransactionalStorage.Batch(accessor => accessor.Indexing.SetIndexesPriority(indexesIds, prioritiesList.ToArray()));
387
388                for (var i = 0; i < createdIndexes.Count; i++)
389                {
390                    var index = createdIndexes[i].Name;
391                    var priority = prioritiesList[i];
392
393                    var instance = Database.IndexStorage.GetIndexInstance(index);
394                    instance.Priority = priority;
395                }
396
397                return createdIndexes.ToArray();
398            }
399            catch (Exception e)
400            {
401                Log.WarnException("Could not create index batch", e);
402                foreach (var index in createdIndexes)
403                {
404                    DeleteIndex(index.Name);
405                }
406                throw;
407            }
408        }
409
410        public class SideBySideIndexInfo
411        {
412            public string OriginalName { get; set; }
413
414            public string Name { get; set; }
415
416            public bool IsSideBySide { get; set; }
417        }
418
419        private static void AssertAnalyzersValid(IndexDefinition indexDefinition)
420        {
421            foreach (var analyzer in indexDefinition.Analyzers)
422            {
423                //this throws if the type cannot be found
424                IndexingExtensions.GetAnalyzerType(analyzer.Key, analyzer.Value);
425            }
426        }
427
428        internal void PutNewIndexIntoStorage(string name, IndexDefinition definition, bool disableIndex = false)
429        {
430            Debug.Assert(Database.IndexStorage != null);
431            Debug.Assert(TransactionalStorage != null);
432            Debug.Assert(WorkContext != null);
433
434            Index index = null;
435            TransactionalStorage.Batch(actions =>
436            {
437                var maxId = 0;
438                if (Database.IndexStorage.Indexes.Length > 0)
439                {
440                    maxId = Database.IndexStorage.Indexes.Max();
441                }
442                definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions);
443                if (definition.IndexId <= maxId)
444                {
445                    actions.General.SetIdentityValue("IndexId", maxId + 1);
446                    definition.IndexId = (int)Database.Documents.GetNextIdentityValueWithoutOverwritingOnExistingDocuments("IndexId", actions);
447                }
448
449                IndexDefinitionStorage.RegisterNewIndexInThisSession(name, definition);
450
451                // this has to happen in this fashion so we will expose the in memory status after the commit, but 
452                // before the rest of the world is notified about this.
453
454                IndexDefinitionStorage.CreateAndPersistIndex(definition);
455                Database.IndexStorage.CreateIndexImplementation(definition);
456                index = Database.IndexStorage.GetIndexInstance(definition.IndexId);
457
458                // If we execute multiple indexes at once and want to activate them all at once we will disable the index from the endpoint
459                if (disableIndex)
460                    index.Priority = IndexingPriority.Disabled;
461
462                //ensure that we don't start indexing it right away, let the precomputation run first, if applicable
463                index.IsMapIndexingInProgress = true;
464                if (definition.IsTestIndex)
465                    index.MarkQueried(); // test indexes should be mark queried, so the cleanup task would not delete them immediately
466
467                InvokeSuggestionIndexing(name, definition, index);
468
469                actions.Indexing.AddIndex(definition.IndexId, definition.IsMapReduce);
470            });
471
472            Debug.Assert(index != null);
473
474            Action precomputeTask = null;
475            if (WorkContext.RunIndexing &&
476                name.Equals(Constants.DocumentsByEntityNameIndex, StringComparison.InvariantCultureIgnoreCase) == false &&
477                Database.IndexStorage.HasIndex(Constants.DocumentsByEntityNameIndex) && isPrecomputedBatchForNewIndexIsRunning == false)
478            {
479                // optimization of handling new index creation when the number of document in a database is significantly greater than
480                // number of documents that this index applies to - let us use built-in RavenDocumentsByEntityName to get just appropriate documents
481
482                precomputeTask = TryCreateTaskForApplyingPrecomputedBatchForNewIndex(index, definition);
483            }
484            else
485            {
486                index.IsMapIndexingInProgress = false;// we can't apply optimization, so we'll make it eligible for running normally
487            }
488
489            // The act of adding it here make it visible to other threads
490            // we have to do it in this way so first we prepare all the elements of the 
491            // index, then we add it to the storage in a way that make it public
492            IndexDefinitionStorage.AddIndex(definition.IndexId, definition);
493
494            // we start the precomuteTask _after_ we finished adding the index
495            if (precomputeTask != null)
496            {
497                precomputeTask();
498            }
499
500            WorkContext.ShouldNotifyAboutWork(() => "PUT INDEX " + name);
501            WorkContext.NotifyAboutWork();
502        }
503
504        private Action TryCreateTaskForApplyingPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
505        {
506            var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);
507            if (generator.ForEntityNames.Count == 0 && index.IsTestIndex == false)
508            {
509                // we don't optimize if we don't have what to optimize _on_, we know this is going to return all docs.
510                // no need to try to optimize that, then
511                index.IsMapIndexingInProgress = false;
512                return null;
513            }
514
515            lock (precomputedLock)
516            {
517                if (isPrecomputedBatchForNewIndexIsRunning)
518                {
519                    index.IsMapIndexingInProgress = false;
520                    return null;
521                }
522
523                isPrecomputedBatchForNewIndexIsRunning = true;
524            }
525
526            try
527            {
528                var cts = new CancellationTokenSource();
529                var task = new Task(() =>
530                {
531                    try
532                    {
533                        ApplyPrecomputedBatchForNewIndex(index, generator, index.IsTestIndex == false ? Database.Configuration.Core.MaxNumberOfItemsToProcessInSingleBatch : Database.Configuration.Indexing.MaxNumberOfItemsToProcessInTestIndexes, cts);
534                    }
535                    catch (Exception e)
536                    {
537                        Log.Warn("Could not apply precomputed batch for index " + index, e);
538                    }
539                    finally
540                    {
541                        isPrecomputedBatchForNewIndexIsRunning = false;
542                        index.IsMapIndexingInProgress = false;
543                        WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
544                        WorkContext.NotifyAboutWork();
545                    }
546                }, TaskCreationOptions.LongRunning);
547
548                return () =>
549                {
550                    try
551                    {
552                        task.Start();
553
554                        long id;
555                        Database
556                            .Tasks
557                            .AddTask(
558                                task,
559                                new TaskBasedOperationState(task),
560                                new TaskActions.PendingTaskDescription
561                                {
562                                    StartTime = DateTime.UtcNow,
563                                    Payload = index.PublicName,
564                                    TaskType = TaskActions.PendingTaskType.NewIndexPrecomputedBatch
565                                },
566                                out id,
567                                cts);
568                    }
569                    catch (Exception)
570                    {
571                        index.IsMapIndexingInProgress = false;
572                        isPrecomputedBatchForNewIndexIsRunning = false;
573                        throw;
574                    }
575                };
576            }
577            catch (Exception)
578            {
579                index.IsMapIndexingInProgress = false;
580                isPrecomputedBatchForNewIndexIsRunning = false;
581                throw;
582            }
583        }
584
585        private void ApplyPrecomputedBatchForNewIndex(Index index, AbstractViewGenerator generator, int pageSize, CancellationTokenSource cts)
586        {
587            PrecomputedIndexingBatch result = null;
588
589            var docsToIndex = new List<JsonDocument>();
590            TransactionalStorage.Batch(actions =>
591            {
592                var query = GetQueryForAllMatchingDocumentsForIndex(generator);
593
594                using (var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, WorkContext.CancellationToken))
595                using (var op = new QueryActions.DatabaseQueryOperation(Database, Constants.DocumentsByEntityNameIndex, new IndexQuery
596                {
597                    Query = query,
598                    PageSize = pageSize
599                }, actions, linked)
600                {
601                    ShouldSkipDuplicateChecking = true
602                })
603                {
604                    op.Init();
605                    if (op.Header.TotalResults > Database.Configuration.Core.MaxNumberOfItemsToProcessInSingleBatch)
606                    {
607                        // we don't apply this optimization if the total number of results 
608                        // to index is more than the max numbers to index in a single batch. 
609                        // The idea here is that we need to keep the amount
610                        // of memory we use to a manageable level even when introducing a new index to a BIG 
611                        // database
612                        try
613                        {
614                            cts.Cancel();
615                            // we have to run just a little bit of the query to properly setup the disposal
616                            op.Execute(o => { });
617                        }
618                        catch (OperationCanceledException)
619                        {
620                        }
621                        return;
622                    }
623
624                    if (Log.IsDebugEnabled)
625                    Log.Debug("For new index {0}, using precomputed indexing batch optimization for {1} docs", index,
626                              op.Header.TotalResults);
627                    op.Execute(document =>
628                    {
629                        var metadata = document.Value<RavenJObject>(Constants.Metadata);
630                        var key = metadata.Value<string>("@id");
631                        var etag = Etag.Parse(metadata.Value<string>("@etag"));
632                        var lastModified = DateTime.Parse(metadata.Value<string>(Constants.LastModified));
633                        document.Remove(Constants.Metadata);
634
635                        var doc = new JsonDocument
636                        {
637                            DataAsJson = document,
638                            Etag = etag,
639                            Key = key,
640                            LastModified = lastModified,
641                            SkipDeleteFromIndex = true,
642                            Metadata = metadata
643                        };
644
645                        docsToIndex.Add(doc);
646                    });
647                    result = new PrecomputedIndexingBatch
648                    {
649                        LastIndexed = op.Header.IndexEtag,
650                        LastModified = op.Header.IndexTimestamp,
651                        Documents = docsToIndex,
652                        Index = index
653                    };
654                }
655            });
656
657            if (result != null && result.Documents != null && result.Documents.Count >= 0)
658            {
659                using (var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, WorkContext.CancellationToken))
660                {
661                    Database.IndexingExecuter.IndexPrecomputedBatch(result, linked.Token);
662
663                    if (index.IsTestIndex) 
664                        TransactionalStorage.Batch(accessor => accessor.Indexing.TouchIndexEtag(index.IndexId));
665                }
666            }
667
668        }
669
670        private string GetQueryForAllMatchingDocumentsForIndex(AbstractViewGenerator generator)
671        {
672            var terms = new TermsQueryRunner(Database)
673                .GetTerms(Constants.DocumentsByEntityNameIndex, "Tag", null, int.MaxValue);
674
675            var sb = new StringBuilder();
676
677            foreach (var entityName in generator.ForEntityNames)
678            {
679                bool added = false;
680                foreach (var term in terms)
681                {
682                    if (string.Equals(entityName, term, StringComparison.OrdinalIgnoreCase))
683                    {
684                        AppendTermToQuery(term, sb);
685                        added = true;
686                    }
687                }
688                if (added == false)
689                    AppendTermToQuery(entityName, sb);
690            }
691
692            return sb.ToString();
693        }
694
695        private static void AppendTermToQuery(string term, StringBuilder sb)
696        {
697                        if (sb.Length != 0)
698                            sb.Append(" OR ");
699
700                        sb.Append("Tag:[[").Append(term).Append("]]");
701                    }
702
703        private void InvokeSuggestionIndexing(string name, IndexDefinition definition, Index index)
704        {
705            foreach (var suggestion in definition.SuggestionsOptions)
706            {
707                var field = suggestion;
708
709                var indexExtensionKey = MonoHttpUtility.UrlEncode(field);
710
711                var suggestionQueryIndexExtension = new SuggestionQueryIndexExtension(
712                    index,
713                     WorkContext,
714                     Path.Combine(Database.Configuration.Core.IndexStoragePath, "Raven-Suggestions", name, indexExtensionKey),
715                     Database.Configuration.Core.RunInMemory,
716                     field);
717
718                Database.IndexStorage.SetIndexExtension(name, indexExtensionKey, suggestionQueryIndexExtension);
719            }
720        }
721
722        private IndexCreationOptions FindIndexCreationOptions(IndexDefinition definition, ref string name)
723        {
724            definition.Name = name;
725            definition.RemoveDefaultValues();
726            IndexDefinitionStorage.ResolveAnalyzers(definition);
727            var findIndexCreationOptions = IndexDefinitionStorage.FindIndexCreationOptions(definition);
728            return findIndexCreationOptions;
729        }
730
731        internal Task StartDeletingIndexDataAsync(int id, string indexName)
732        {
733            var sp = Stopwatch.StartNew();
734            //remove the header information in a sync process
735            TransactionalStorage.Batch(actions => actions.Indexing.PrepareIndexForDeletion(id));
736            var deleteIndexTask = Task.Run(() =>
737            {
738                Debug.Assert(Database.IndexStorage != null);
739                Log.Info("Starting async deletion of index {0}", indexName);
740                Database.IndexStorage.DeleteIndexData(id); // Data can take a while
741
742                TransactionalStorage.Batch(actions =>
743                {
744                    // And Esent data can take a while too
745                    actions.Indexing.DeleteIndex(id, WorkContext.CancellationToken);
746                    if (WorkContext.CancellationToken.IsCancellationRequested)
747                        return;
748
749                    actions.Lists.Remove("Raven/Indexes/PendingDeletion", id.ToString(CultureInfo.InvariantCulture));
750                });
751            });
752
753            long taskId;
754            Database.Tasks.AddTask(deleteIndexTask, null, new TaskActions.PendingTaskDescription
755                                                          {
756                                                              StartTime = SystemTime.UtcNow,
757                                                              TaskType = TaskActions.PendingTaskType.IndexDeleteOperation,
758                                                              Payload = indexName
759                                                          }, out taskId);
760
761            deleteIndexTask.ContinueWith(t =>
762            {
763                if (t.IsFaulted || t.IsCanceled)
764                {
765                    Log.WarnException("Failure when deleting index " + indexName, t.Exception);
766                }
767                else
768                {
769                    Log.Info("The async deletion of index {0} was completed in {1}", indexName, sp.Elapsed);
770                }
771            });
772
773            return deleteIndexTask;
774        }
775
776        public RavenJArray GetIndexNames(int start, int pageSize)
777        {
778            return new RavenJArray(
779                IndexDefinitionStorage.IndexNames.Skip(start).Take(pageSize)
780                    .Select(s => new RavenJValue(s))
781                );
782        }
783
784        public RavenJArray GetIndexes(int start, int pageSize)
785        {
786            return new RavenJArray(
787                from indexName in IndexDefinitionStorage.IndexNames.Skip(start).Take(pageSize)
788                let indexDefinition = IndexDefinitionStorage.GetIndexDefinition(indexName)
789                select new RavenJObject
790                {
791                    {"name", new RavenJValue(indexName)},
792                    {"definition", indexDefinition != null ? RavenJObject.FromObject(indexDefinition) : null},
793                });
794        }
795
796        public IndexDefinition GetIndexDefinition(string index)
797        {
798            return IndexDefinitionStorage.GetIndexDefinition(index);
799        }
800
801        [MethodImpl(MethodImplOptions.Synchronized)]
802        public void ResetIndex(string index)
803        {
804            var indexDefinition = IndexDefinitionStorage.GetIndexDefinition(index);
805            if (indexDefinition == null)
806                throw new InvalidOperationException("There is no index named: " + index);
807            DeleteIndex(index);
808            PutIndex(index, indexDefinition);
809        }
810
811        [MethodImpl(MethodImplOptions.Synchronized)]
812        public bool DeleteIndex(string name)
813        {
814            var instance = IndexDefinitionStorage.GetIndexDefinition(name);
815            if (instance == null) 
816                return false;
817
818            DeleteIndex(instance);
819            return true;
820        }
821
822        internal void DeleteIndex(IndexDefinition instance, bool removeByNameMapping = true, bool clearErrors = true, bool removeIndexReplaceDocument = true, bool isSideBySideReplacement = false)
823        {
824            using (IndexDefinitionStorage.TryRemoveIndexContext())
825            {
826                if (instance == null) 
827                    return;
828
829                // Set up a flag to signal that this is something we're doing
830                TransactionalStorage.Batch(actions => actions.Lists.Set("Raven/Indexes/PendingDeletion", instance.IndexId.ToString(CultureInfo.InvariantCulture), (RavenJObject.FromObject(new
831                {
832                    TimeOfOriginalDeletion = SystemTime.UtcNow,
833                    instance.IndexId,
834                    IndexName = instance.Name,
835                    instance.IndexVersion
836                })), UuidType.Tasks));
837
838                // Delete the main record synchronously
839                IndexDefinitionStorage.RemoveIndex(instance.IndexId, removeByNameMapping);
840                Database.IndexStorage.DeleteIndex(instance.IndexId);
841
842                if (clearErrors)
843                    WorkContext.ClearErrorsFor(instance.Name);
844
845                if (removeIndexReplaceDocument && instance.IsSideBySideIndex)
846                {
847                    Database.Documents.Delete(Constants.IndexReplacePrefix + instance.Name, null, null);
848                }
849
850                // And delete the data in the background
851                StartDeletingIndexDataAsync(instance.IndexId, instance.Name);
852
853
854                var indexChangeType = isSideBySideReplacement ? IndexChangeTypes.SideBySideReplace : IndexChangeTypes.IndexRemoved;
855
856                // We raise the notification now because as far as we're concerned it is done *now*
857                TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => 
858                    Database.Notifications.RaiseNotifications(new IndexChangeNotification
859                    {
860                        Name = instance.Name,
861                        Type = indexChangeType,
862                        Version = instance.IndexVersion
863                    })
864                );
865            }
866        }
867
868    }
869}