PageRenderTime 59ms CodeModel.GetById 25ms RepoModel.GetById 0ms app.codeStats 0ms

/ToMigrate/Raven.Database/Indexing/IndexStorage.cs

http://github.com/ayende/ravendb
C# | 1691 lines | 1359 code | 285 blank | 47 comment | 242 complexity | 5b0b2dda17805dbd210707531b5ba65d MD5 | raw file
Possible License(s): GPL-3.0, MPL-2.0-no-copyleft-exception, LGPL-2.1, Apache-2.0, BSD-3-Clause, CC-BY-SA-3.0
  1. //-----------------------------------------------------------------------
  2. // <copyright file="IndexStorage.cs" company="Hibernating Rhinos LTD">
  3. // Copyright (c) Hibernating Rhinos LTD. All rights reserved.
  4. // </copyright>
  5. //-----------------------------------------------------------------------
  6. using System;
  7. using System.Collections.Concurrent;
  8. using System.Collections.Generic;
  9. using System.ComponentModel.Composition;
  10. using System.Diagnostics;
  11. using System.Globalization;
  12. using System.IO;
  13. using System.Linq;
  14. using System.Runtime.ConstrainedExecution;
  15. using System.Text;
  16. using System.Threading;
  17. using System.Threading.Tasks;
  18. using Lucene.Net.Analysis;
  19. using Lucene.Net.Index;
  20. using Lucene.Net.Search;
  21. using Lucene.Net.Store;
  22. using Lucene.Net.Util;
  23. using Raven.Abstractions;
  24. using Raven.Abstractions.Data;
  25. using Raven.Abstractions.Extensions;
  26. using Raven.Abstractions.Indexing;
  27. using Raven.Abstractions.Logging;
  28. using Raven.Abstractions.MEF;
  29. using Raven.Abstractions.Util;
  30. using Raven.Database.Actions;
  31. using Raven.Database.Config;
  32. using Raven.Database.Data;
  33. using Raven.Database.Extensions;
  34. using Raven.Database.Impl;
  35. using Raven.Database.Linq;
  36. using Raven.Database.Plugins;
  37. using Raven.Database.Queries;
  38. using Raven.Database.Storage;
  39. using Raven.Database.Util;
  40. using Raven.Imports.Newtonsoft.Json;
  41. using Raven.Imports.Newtonsoft.Json.Linq;
  42. using Raven.Json.Linq;
  43. using Constants = Raven.Abstractions.Data.Constants;
  44. using Directory = Lucene.Net.Store.Directory;
  45. namespace Raven.Database.Indexing
  46. {
  47. /// <summary>
  48. /// Thread safe, single instance for the entire application
  49. /// </summary>
  50. public class IndexStorage : CriticalFinalizerObject, IDisposable
  51. {
  52. private readonly DocumentDatabase documentDatabase;
  53. private const string IndexVersion = "2.0.0.1";
  54. private const string MapReduceIndexVersion = "2.5.0.1";
  55. private readonly IndexDefinitionStorage indexDefinitionStorage;
  56. private readonly RavenConfiguration configuration;
  57. private readonly string path;
  58. private static readonly ILog log = LogManager.GetCurrentClassLogger();
  59. private static readonly ILog startupLog = LogManager.GetLogger(typeof(IndexStorage).FullName + ".Startup");
  60. private readonly Analyzer dummyAnalyzer = new SimpleAnalyzer();
  61. private DateTime latestPersistedQueryTime;
  62. private readonly FileStream crashMarker;
  63. private ConcurrentDictionary<int, Index> indexes =
  64. new ConcurrentDictionary<int, Index>();
  65. public class RegisterLowMemoryHandler : ILowMemoryHandler
  66. {
  67. static RegisterLowMemoryHandler _instance;
  68. public static void Setup()
  69. {
  70. if (_instance != null)
  71. return;
  72. lock (typeof(RegisterLowMemoryHandler))
  73. {
  74. if (_instance != null)
  75. return;
  76. _instance = new RegisterLowMemoryHandler();
  77. MemoryStatistics.RegisterLowMemoryHandler(_instance);
  78. }
  79. }
  80. public void HandleLowMemory()
  81. {
  82. FieldCache_Fields.DEFAULT.PurgeAllCaches();
  83. }
  84. public void SoftMemoryRelease()
  85. {
  86. }
  87. public LowMemoryHandlerStatistics GetStats()
  88. {
  89. var cacheEntries = FieldCache_Fields.DEFAULT.GetCacheEntries();
  90. var memorySum = cacheEntries.Sum(x =>
  91. {
  92. var curEstimator = new RamUsageEstimator(false);
  93. return curEstimator.EstimateRamUsage(x);
  94. });
  95. return new LowMemoryHandlerStatistics
  96. {
  97. Name = "LuceneLowMemoryHandler",
  98. EstimatedUsedMemory = memorySum,
  99. Metadata = new
  100. {
  101. CachedEntriesAmount = cacheEntries.Length
  102. }
  103. };
  104. }
  105. }
  106. public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, RavenConfiguration configuration, DocumentDatabase documentDatabase)
  107. {
  108. try
  109. {
  110. RegisterLowMemoryHandler.Setup();
  111. this.indexDefinitionStorage = indexDefinitionStorage;
  112. this.configuration = configuration;
  113. this.documentDatabase = documentDatabase;
  114. path = configuration.Core.IndexStoragePath;
  115. if (System.IO.Directory.Exists(path) == false && configuration.Core.RunInMemory == false)
  116. System.IO.Directory.CreateDirectory(path);
  117. if (configuration.Core.RunInMemory == false)
  118. {
  119. var crashMarkerPath = Path.Combine(path, "indexing.crash-marker");
  120. if (File.Exists(crashMarkerPath))
  121. {
  122. // the only way this can happen is if we crashed because of a power outage
  123. // in this case, we consider all open indexes to be corrupt and force them
  124. // to be reset. This is because to get better perf, we don't flush the files to disk,
  125. // so in the case of a power outage, we can't be sure that there wasn't still stuff in
  126. // the OS buffer that wasn't written yet.
  127. configuration.Indexing.ResetIndexOnUncleanShutdown = true;
  128. }
  129. // The delete on close ensures that the only way this file will exists is if there was
  130. // a power outage while the server was running.
  131. crashMarker = File.Create(crashMarkerPath, 16, FileOptions.DeleteOnClose);
  132. }
  133. if (log.IsDebugEnabled)
  134. log.Debug("Start opening indexes. There are {0} indexes that need to be loaded", indexDefinitionStorage.IndexNames.Length);
  135. BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(documentDatabase.WorkContext, indexDefinitionStorage.IndexNames,
  136. name =>
  137. {
  138. var index = OpenIndex(name, onStartup: true, forceFullIndexCheck: false);
  139. if (index != null)
  140. indexes.TryAdd(index.IndexId, index);
  141. if (startupLog.IsDebugEnabled)
  142. startupLog.Debug("{0}/{1} indexes loaded", indexes.Count, indexDefinitionStorage.IndexNames.Length);
  143. });
  144. if (log.IsDebugEnabled)
  145. log.Debug("Index storage initialized. All indexes have been opened.");
  146. }
  147. catch (Exception e)
  148. {
  149. log.WarnException("Could not create index storage", e);
  150. try
  151. {
  152. Dispose();
  153. }
  154. catch (Exception ex)
  155. {
  156. log.FatalException("Failed to dispose when already getting an error during ctor", ex);
  157. }
  158. throw;
  159. }
  160. }
  161. private Index OpenIndex(string indexName, bool onStartup, bool forceFullIndexCheck)
  162. {
  163. if (indexName == null)
  164. throw new ArgumentNullException("indexName");
  165. if (startupLog.IsDebugEnabled)
  166. startupLog.Debug("Loading saved index {0}", indexName);
  167. var indexDefinition = indexDefinitionStorage.GetIndexDefinition(indexName);
  168. if (indexDefinition == null)
  169. return null;
  170. Index indexImplementation = null;
  171. bool resetTried = false;
  172. bool recoveryTried = false;
  173. string[] keysToDeleteAfterRecovery = null;
  174. while (true)
  175. {
  176. Directory luceneDirectory = null;
  177. try
  178. {
  179. luceneDirectory = OpenOrCreateLuceneDirectory(indexDefinition, createIfMissing: resetTried, forceFullExistingIndexCheck: forceFullIndexCheck);
  180. indexImplementation = CreateIndexImplementation(indexDefinition, luceneDirectory);
  181. CheckIndexState(luceneDirectory, indexDefinition, indexImplementation, resetTried);
  182. if (forceFullIndexCheck)
  183. {
  184. // the above index check might pass however an index writer creation can still throw an exception
  185. // so we need to check it here to avoid crashing in runtime
  186. new IndexWriter(luceneDirectory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
  187. }
  188. var simpleIndex = indexImplementation as SimpleIndex; // no need to do this on m/r indexes, since we rebuild them from saved data anyway
  189. if (simpleIndex != null && keysToDeleteAfterRecovery != null)
  190. {
  191. // remove keys from index that were deleted after creating commit point
  192. simpleIndex.RemoveDirectlyFromIndex(keysToDeleteAfterRecovery, GetLastEtagForIndex(simpleIndex));
  193. }
  194. LoadExistingSuggestionsExtentions(indexDefinition.Name, indexImplementation);
  195. documentDatabase.TransactionalStorage.Batch(accessor =>
  196. {
  197. IndexStats indexStats = accessor.Indexing.GetIndexStats(indexDefinition.IndexId);
  198. if (indexStats != null)
  199. indexImplementation.Priority = indexStats.Priority;
  200. var read = accessor.Lists.Read("Raven/Indexes/QueryTime", indexName);
  201. if (read == null)
  202. {
  203. if (IsIdleAutoIndex(indexImplementation))
  204. indexImplementation.MarkQueried(); // prevent index abandoning right after startup
  205. return;
  206. }
  207. var dateTime = read.Data.Value<DateTime>("LastQueryTime");
  208. if (IsIdleAutoIndex(indexImplementation) && SystemTime.UtcNow - dateTime > configuration.Indexing.TimeToWaitBeforeRunningAbandonedIndexes.AsTimeSpan)
  209. indexImplementation.MarkQueried(); // prevent index abandoning right after startup
  210. else
  211. indexImplementation.MarkQueried(dateTime);
  212. if (dateTime > latestPersistedQueryTime)
  213. latestPersistedQueryTime = dateTime;
  214. });
  215. break;
  216. }
  217. catch (Exception e)
  218. {
  219. if (resetTried)
  220. throw new InvalidOperationException("Could not open / create index" + indexName + ", reset already tried", e);
  221. if (indexImplementation != null)
  222. indexImplementation.Dispose();
  223. if (recoveryTried == false && luceneDirectory != null)
  224. {
  225. recoveryTried = true;
  226. startupLog.WarnException("Could not open index " + indexName + ". Trying to recover index", e);
  227. keysToDeleteAfterRecovery = TryRecoveringIndex(indexDefinition, luceneDirectory);
  228. }
  229. else
  230. {
  231. resetTried = true;
  232. startupLog.WarnException("Could not open index " + indexName + ". Recovery operation failed, forcibly resetting index", e);
  233. TryResettingIndex(indexName, indexDefinition, onStartup);
  234. }
  235. }
  236. }
  237. return indexImplementation;
  238. }
  239. private void CheckIndexState(Directory directory, IndexDefinition indexDefinition, Index index, bool resetTried)
  240. {
  241. //if (configuration.ResetIndexOnUncleanShutdown == false)
  242. // return;
  243. // 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty
  244. // 2. If no 'LastEtag' in commitData then we consider it an invalid index
  245. // 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag)
  246. var commitData = IndexReader.GetCommitUserData(directory);
  247. if (index.IsMapReduce)
  248. CheckMapReduceIndexState(commitData, resetTried);
  249. else
  250. CheckMapIndexState(commitData, indexDefinition, index);
  251. }
  252. private void CheckMapIndexState(IDictionary<string, string> commitData, IndexDefinition indexDefinition, Index index)
  253. {
  254. string value;
  255. Etag lastEtag = null;
  256. if (commitData != null && commitData.TryGetValue("LastEtag", out value))
  257. Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail
  258. var lastStoredEtag = GetLastEtagForIndex(index) ?? Etag.Empty;
  259. lastEtag = lastEtag ?? Etag.Empty;
  260. if (EtagUtil.IsGreaterThanOrEqual(lastEtag, lastStoredEtag))
  261. return;
  262. log.Info(string.Format("Resetting index '{0} ({1})'. Last stored etag: {2}. Last commit etag: {3}.", indexDefinition.Name, index.indexId, lastStoredEtag, lastEtag));
  263. var now = SystemTime.UtcNow;
  264. ResetLastIndexedEtag(indexDefinition, lastEtag, now);
  265. }
  266. private static void CheckMapReduceIndexState(IDictionary<string, string> commitData, bool resetTried)
  267. {
  268. if (resetTried)
  269. return;
  270. string marker;
  271. long commitMarker;
  272. var valid = commitData != null
  273. && commitData.TryGetValue("Marker", out marker)
  274. && long.TryParse(marker, out commitMarker)
  275. && commitMarker == RavenIndexWriter.CommitMarker;
  276. if (valid == false)
  277. throw new InvalidOperationException("Map-Reduce index corruption detected.");
  278. }
  279. private static bool IsIdleAutoIndex(Index index)
  280. {
  281. return index.PublicName.StartsWith("Auto/") && index.Priority == IndexingPriority.Idle;
  282. }
  283. private void TryResettingIndex(string indexName, IndexDefinition indexDefinition, bool onStartup)
  284. {
  285. try
  286. {
  287. Action reset = () =>
  288. {
  289. try
  290. {
  291. documentDatabase.Indexes.DeleteIndex(indexDefinition, removeIndexReplaceDocument: false);
  292. documentDatabase.Indexes.PutNewIndexIntoStorage(indexName, indexDefinition);
  293. var indexReplaceDocumentKey = Constants.IndexReplacePrefix + indexName;
  294. var indexReplaceDocument = documentDatabase.Documents.Get(indexReplaceDocumentKey);
  295. if (indexReplaceDocument == null)
  296. return;
  297. documentDatabase.Documents.Put(indexReplaceDocumentKey, null, indexReplaceDocument.DataAsJson, indexReplaceDocument.Metadata, null);
  298. }
  299. catch (Exception e)
  300. {
  301. throw new InvalidOperationException("Could not finalize reseting of index: " + indexName, e);
  302. }
  303. };
  304. if (onStartup)
  305. {
  306. // we have to defer the work here until the database is actually ready for work
  307. documentDatabase.OnIndexingWiringComplete += reset;
  308. }
  309. else
  310. {
  311. reset();
  312. }
  313. var indexFullPath = Path.Combine(path, indexDefinition.IndexId.ToString(CultureInfo.InvariantCulture));
  314. IOExtensions.DeleteDirectory(indexFullPath);
  315. var suggestionsForIndex = Path.Combine(configuration.Core.IndexStoragePath, "Raven-Suggestions", indexName);
  316. IOExtensions.DeleteDirectory(suggestionsForIndex);
  317. }
  318. catch (Exception exception)
  319. {
  320. throw new InvalidOperationException("Could not reset index " + indexName, exception);
  321. }
  322. }
  323. private string[] TryRecoveringIndex(IndexDefinition indexDefinition,
  324. Directory luceneDirectory)
  325. {
  326. string[] keysToDeleteAfterRecovery = null;
  327. if (indexDefinition.IsMapReduce == false)
  328. {
  329. IndexCommitPoint commitUsedToRestore;
  330. if (TryReusePreviousCommitPointsToRecoverIndex(luceneDirectory,
  331. indexDefinition, path,
  332. out commitUsedToRestore,
  333. out keysToDeleteAfterRecovery))
  334. {
  335. ResetLastIndexedEtag(indexDefinition, commitUsedToRestore.HighestCommitedETag, commitUsedToRestore.TimeStamp);
  336. }
  337. }
  338. else
  339. {
  340. RegenerateMapReduceIndex(luceneDirectory, indexDefinition);
  341. }
  342. return keysToDeleteAfterRecovery;
  343. }
  344. private void LoadExistingSuggestionsExtentions(string indexName, Index indexImplementation)
  345. {
  346. var suggestionsForIndex = Path.Combine(configuration.Core.IndexStoragePath, "Raven-Suggestions", indexName);
  347. if (!System.IO.Directory.Exists(suggestionsForIndex))
  348. return;
  349. try
  350. {
  351. var directories = System.IO.Directory.GetDirectories(suggestionsForIndex);
  352. if (directories.Any(dir => dir.Contains("-")))
  353. {
  354. // Legacy handling:
  355. // Previously we had separate folder with suggestions for each triple: (field, distanceType, accuracy)
  356. // Now we have field only.
  357. // Legacy naming convention was: field-{distanceType}-{accuracy}
  358. // since when we have - (dash) in SOME folder name it seems to be legacy
  359. HandleLegacySuggestions(directories);
  360. // Refresh directories list as handling legacy might rename or delete some of them.
  361. directories = System.IO.Directory.GetDirectories(suggestionsForIndex);
  362. }
  363. foreach (var directory in directories)
  364. {
  365. IndexSearcher searcher;
  366. using (indexImplementation.GetSearcher(out searcher))
  367. {
  368. var key = Path.GetFileName(directory);
  369. var field = MonoHttpUtility.UrlDecode(key);
  370. var extension = new SuggestionQueryIndexExtension(
  371. indexImplementation,
  372. documentDatabase.WorkContext,
  373. Path.Combine(configuration.Core.IndexStoragePath, "Raven-Suggestions", indexName, key),
  374. searcher.IndexReader.Directory() is RAMDirectory,
  375. field);
  376. indexImplementation.SetExtension(key, extension);
  377. }
  378. }
  379. }
  380. catch (Exception e)
  381. {
  382. log.WarnException("Could not open suggestions for index " + indexName + ", resetting the index", e);
  383. try
  384. {
  385. IOExtensions.DeleteDirectory(suggestionsForIndex);
  386. }
  387. catch (Exception)
  388. {
  389. // ignore the failure
  390. }
  391. throw;
  392. }
  393. }
  394. internal static void HandleLegacySuggestions(string[] directories)
  395. {
  396. var alreadySeenFields = new HashSet<string>();
  397. foreach (var directory in directories)
  398. {
  399. var key = Path.GetFileName(directory);
  400. var parentDir = System.IO.Directory.GetParent(directory).FullName;
  401. if (key.Contains("-"))
  402. {
  403. var tokens = key.Split('-');
  404. var field = tokens[0];
  405. if (alreadySeenFields.Contains(field))
  406. {
  407. log.Info("Removing legacy suggestions: {0}", directory);
  408. IOExtensions.DeleteDirectory(directory);
  409. }
  410. else
  411. {
  412. alreadySeenFields.Add(field);
  413. var newLocation = Path.Combine(parentDir, field);
  414. log.Info("Moving suggestions from: {0} to {1}", directory, newLocation);
  415. System.IO.Directory.Move(directory, newLocation);
  416. }
  417. }
  418. else
  419. {
  420. alreadySeenFields.Add(key);
  421. }
  422. }
  423. }
  424. protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, bool createIfMissing = true, bool forceFullExistingIndexCheck = false)
  425. {
  426. Lucene.Net.Store.Directory directory;
  427. if (configuration.Core.RunInMemory ||
  428. (indexDefinition.IsMapReduce == false && // there is no point in creating map/reduce indexes in memory, we write the intermediate results to disk anyway
  429. indexDefinitionStorage.IsNewThisSession(indexDefinition) &&
  430. indexDefinition.DisableInMemoryIndexing == false &&
  431. configuration.Indexing.DisableInMemoryIndexing == false &&
  432. forceFullExistingIndexCheck == false))
  433. {
  434. directory = new RAMDirectory();
  435. new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose(); // creating index structure
  436. }
  437. else
  438. {
  439. var indexDirectory = indexDefinition.IndexId.ToString();
  440. var indexFullPath = Path.Combine(path, indexDirectory);
  441. directory = new LuceneCodecDirectory(indexFullPath, documentDatabase.IndexCodecs.OfType<AbstractIndexCodec>());
  442. if (!IndexReader.IndexExists(directory))
  443. {
  444. if (createIfMissing == false)
  445. throw new InvalidOperationException(string.Format("Index directory '{0}' does not exists for '{1}' index.", indexFullPath, indexDefinition.Name));
  446. WriteIndexVersion(directory, indexDefinition);
  447. //creating index structure if we need to
  448. new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
  449. }
  450. else
  451. {
  452. EnsureIndexVersionMatches(directory, indexDefinition);
  453. if (forceFullExistingIndexCheck == false)
  454. {
  455. if (directory.FileExists("write.lock")) // force lock release, because it was still open when we shut down
  456. {
  457. IndexWriter.Unlock(directory);
  458. // for some reason, just calling unlock doesn't remove this file
  459. directory.DeleteFile("write.lock");
  460. }
  461. if (directory.FileExists("writing-to-index.lock")) // we had an unclean shutdown
  462. {
  463. if (configuration.Indexing.ResetIndexOnUncleanShutdown)
  464. throw new InvalidOperationException(string.Format("Rude shutdown detected on '{0}' index in '{1}' directory.", indexDefinition.Name, indexFullPath));
  465. CheckIndexAndTryToFix(directory, indexDefinition);
  466. directory.DeleteFile("writing-to-index.lock");
  467. }
  468. }
  469. else
  470. {
  471. IndexWriter.Unlock(directory);
  472. if (directory.FileExists("write.lock"))
  473. directory.DeleteFile("write.lock");
  474. CheckIndexAndTryToFix(directory, indexDefinition);
  475. if (directory.FileExists("writing-to-index.lock"))
  476. directory.DeleteFile("writing-to-index.lock");
  477. }
  478. }
  479. }
  480. return directory;
  481. }
  482. private void RegenerateMapReduceIndex(Directory directory, IndexDefinition indexDefinition)
  483. {
  484. // remove old index data
  485. var dirOnDisk = Path.Combine(path, indexDefinition.IndexId.ToString());
  486. IOExtensions.DeleteDirectory(dirOnDisk);
  487. // initialize by new index
  488. System.IO.Directory.CreateDirectory(dirOnDisk);
  489. WriteIndexVersion(directory, indexDefinition);
  490. new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
  491. var start = 0;
  492. const int take = 100;
  493. documentDatabase.TransactionalStorage.Batch(actions =>
  494. {
  495. IList<ReduceTypePerKey> reduceKeysAndTypes;
  496. do
  497. {
  498. reduceKeysAndTypes = actions.MapReduce.GetReduceKeysAndTypes(indexDefinition.IndexId, start, take).ToList();
  499. start += take;
  500. var keysToScheduleOnLevel2 =
  501. reduceKeysAndTypes.Where(x => x.OperationTypeToPerform == ReduceType.MultiStep).ToList();
  502. var keysToScheduleOnLevel0 =
  503. reduceKeysAndTypes.Where(x => x.OperationTypeToPerform == ReduceType.SingleStep).ToList();
  504. var itemsToScheduleOnLevel2 = keysToScheduleOnLevel2.Select(x => new ReduceKeyAndBucket(0, x.ReduceKey)).ToList();
  505. var itemsToScheduleOnLevel0 = new List<ReduceKeyAndBucket>();
  506. foreach (var reduceKey in keysToScheduleOnLevel0.Select(x => x.ReduceKey))
  507. {
  508. var mappedBuckets = actions.MapReduce.GetMappedBuckets(indexDefinition.IndexId, reduceKey, CancellationToken.None).Distinct();
  509. itemsToScheduleOnLevel0.AddRange(mappedBuckets.Select(x => new ReduceKeyAndBucket(x, reduceKey)));
  510. }
  511. foreach (var itemToReduce in itemsToScheduleOnLevel2)
  512. {
  513. actions.MapReduce.ScheduleReductions(indexDefinition.IndexId, 2, itemToReduce);
  514. actions.General.MaybePulseTransaction();
  515. }
  516. foreach (var itemToReduce in itemsToScheduleOnLevel0)
  517. {
  518. actions.MapReduce.ScheduleReductions(indexDefinition.IndexId, 0, itemToReduce);
  519. actions.General.MaybePulseTransaction();
  520. }
  521. } while (reduceKeysAndTypes.Count > 0);
  522. });
  523. }
  524. private void ResetLastIndexedEtag(IndexDefinition indexDefinition, Etag lastIndexedEtag, DateTime timestamp)
  525. {
  526. documentDatabase.TransactionalStorage.Batch(
  527. accessor =>
  528. accessor.Indexing.UpdateLastIndexed(indexDefinition.IndexId, lastIndexedEtag, timestamp));
  529. }
  530. internal Etag GetLastEtagForIndex(Index index)
  531. {
  532. if (index.IsMapReduce)
  533. return null;
  534. IndexStats stats = null;
  535. documentDatabase.TransactionalStorage.Batch(accessor => stats = accessor.Indexing.GetIndexStats(index.IndexId));
  536. return stats != null ? stats.LastIndexedEtag : Etag.Empty;
  537. }
  538. public static string IndexVersionFileName(IndexDefinition indexDefinition)
  539. {
  540. if (indexDefinition.IsMapReduce)
  541. return "mapReduce.version";
  542. return "index.version";
  543. }
  544. public static void WriteIndexVersion(Directory directory, IndexDefinition indexDefinition)
  545. {
  546. var version = IndexVersion;
  547. if (indexDefinition.IsMapReduce)
  548. {
  549. version = MapReduceIndexVersion;
  550. }
  551. using (var indexOutput = directory.CreateOutput(IndexVersionFileName(indexDefinition)))
  552. {
  553. indexOutput.WriteString(version);
  554. indexOutput.Flush();
  555. }
  556. }
  557. private static void EnsureIndexVersionMatches(Directory directory, IndexDefinition indexDefinition)
  558. {
  559. var versionToCheck = IndexVersion;
  560. if (indexDefinition.IsMapReduce)
  561. {
  562. versionToCheck = MapReduceIndexVersion;
  563. }
  564. var indexVersion = IndexVersionFileName(indexDefinition);
  565. if (directory.FileExists(indexVersion) == false)
  566. {
  567. throw new InvalidOperationException("Could not find " + indexVersion + " " + indexDefinition.IndexId + ", resetting index");
  568. }
  569. using (var indexInput = directory.OpenInput(indexVersion))
  570. {
  571. var versionFromDisk = indexInput.ReadString();
  572. if (versionFromDisk != versionToCheck)
  573. throw new InvalidOperationException("Index " + indexDefinition.IndexId + " is of version " + versionFromDisk +
  574. " which is not compatible with " + versionToCheck + ", resetting index");
  575. }
  576. }
  577. private static void CheckIndexAndTryToFix(Directory directory, IndexDefinition indexDefinition)
  578. {
  579. startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDefinition.Name);
  580. var memoryStream = new MemoryStream();
  581. var stringWriter = new StreamWriter(memoryStream);
  582. var checkIndex = new CheckIndex(directory);
  583. if (startupLog.IsWarnEnabled)
  584. checkIndex.SetInfoStream(stringWriter);
  585. var sp = Stopwatch.StartNew();
  586. var status = checkIndex.CheckIndex_Renamed_Method();
  587. sp.Stop();
  588. if (startupLog.IsWarnEnabled)
  589. {
  590. startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDefinition.Name, sp.Elapsed, status.clean);
  591. memoryStream.Position = 0;
  592. log.Warn(new StreamReader(memoryStream).ReadToEnd());
  593. }
  594. if (status.clean)
  595. return;
  596. startupLog.Warn("Attempting to fix index: {0}", indexDefinition.Name);
  597. sp.Restart();
  598. checkIndex.FixIndex(status);
  599. startupLog.Warn("Fixed index {0} in {1}", indexDefinition.Name, sp.Elapsed);
  600. }
  601. public void StoreCommitPoint(string indexName, IndexCommitPoint indexCommit)
  602. {
  603. if (indexCommit.SegmentsInfo == null || indexCommit.SegmentsInfo.IsIndexCorrupted)
  604. return;
  605. var directoryName = indexCommit.SegmentsInfo.Generation.ToString("0000000000000000000", CultureInfo.InvariantCulture);
  606. var commitPointDirectory = new IndexCommitPointDirectory(path, indexName, directoryName);
  607. if (System.IO.Directory.Exists(commitPointDirectory.AllCommitPointsFullPath) == false)
  608. {
  609. System.IO.Directory.CreateDirectory(commitPointDirectory.AllCommitPointsFullPath);
  610. }
  611. System.IO.Directory.CreateDirectory(commitPointDirectory.FullPath);
  612. using (var commitPointFile = File.Create(commitPointDirectory.FileFullPath))
  613. using (var sw = new StreamWriter(commitPointFile))
  614. {
  615. var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
  616. var textWriter = new JsonTextWriter(sw);
  617. jsonSerializer.Serialize(textWriter, indexCommit);
  618. sw.Flush();
  619. }
  620. var currentSegmentsFileName = indexCommit.SegmentsInfo.SegmentsFileName;
  621. File.Copy(Path.Combine(commitPointDirectory.IndexFullPath, currentSegmentsFileName),
  622. Path.Combine(commitPointDirectory.FullPath, currentSegmentsFileName),
  623. overwrite: true);
  624. var storedCommitPoints = System.IO.Directory.GetDirectories(commitPointDirectory.AllCommitPointsFullPath);
  625. if (storedCommitPoints.Length > configuration.Indexing.MaxNumberOfStoredCommitPoints)
  626. {
  627. foreach (var toDelete in storedCommitPoints.Take(storedCommitPoints.Length - configuration.Indexing.MaxNumberOfStoredCommitPoints))
  628. {
  629. IOExtensions.DeleteDirectory(toDelete);
  630. }
  631. }
  632. }
  633. public void AddDeletedKeysToCommitPoints(IndexDefinition indexDefinition, string[] deletedKeys)
  634. {
  635. var indexFullPath = Path.Combine(path, indexDefinition.IndexId.ToString());
  636. var existingCommitPoints = IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath);
  637. foreach (var commitPointDirectory in existingCommitPoints.Select(commitPoint => new IndexCommitPointDirectory(path, indexDefinition.IndexId.ToString(), commitPoint)))
  638. {
  639. using (var stream = File.Open(commitPointDirectory.DeletedKeysFile, FileMode.OpenOrCreate))
  640. {
  641. stream.Seek(0, SeekOrigin.End);
  642. using (var writer = new StreamWriter(stream))
  643. {
  644. foreach (var deletedKey in deletedKeys)
  645. {
  646. writer.WriteLine(deletedKey);
  647. }
  648. }
  649. }
  650. }
  651. }
  652. private bool TryReusePreviousCommitPointsToRecoverIndex(Directory directory, IndexDefinition indexDefinition, string indexStoragePath, out IndexCommitPoint indexCommit, out string[] keysToDelete)
  653. {
  654. indexCommit = null;
  655. keysToDelete = null;
  656. if (indexDefinition.IsMapReduce)
  657. return false;
  658. var indexFullPath = Path.Combine(indexStoragePath, indexDefinition.IndexId.ToString());
  659. var allCommitPointsFullPath = IndexCommitPointDirectory.GetAllCommitPointsFullPath(indexFullPath);
  660. if (System.IO.Directory.Exists(allCommitPointsFullPath) == false)
  661. return false;
  662. var filesInIndexDirectory = System.IO.Directory.GetFiles(indexFullPath).Select(Path.GetFileName);
  663. var existingCommitPoints =
  664. IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath);
  665. Array.Reverse(existingCommitPoints); // start from the highest generation
  666. foreach (var commitPointDirectoryName in existingCommitPoints)
  667. {
  668. try
  669. {
  670. var commitPointDirectory = new IndexCommitPointDirectory(indexStoragePath, indexDefinition.IndexId.ToString(),
  671. commitPointDirectoryName);
  672. if (TryGetCommitPoint(commitPointDirectory, out indexCommit) == false)
  673. {
  674. IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
  675. continue; // checksum is invalid, try another commit point
  676. }
  677. var missingFile =
  678. indexCommit.SegmentsInfo.ReferencedFiles.Any(
  679. referencedFile => filesInIndexDirectory.Contains(referencedFile) == false);
  680. if (missingFile)
  681. {
  682. IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
  683. continue; // there are some missing files, try another commit point
  684. }
  685. var storedSegmentsFile = indexCommit.SegmentsInfo.SegmentsFileName;
  686. // here there should be only one segments_N file, however remove all if there is more
  687. foreach (var currentSegmentsFile in System.IO.Directory.GetFiles(commitPointDirectory.IndexFullPath, "segments_*"))
  688. {
  689. File.Delete(currentSegmentsFile);
  690. }
  691. // copy old segments_N file
  692. File.Copy(Path.Combine(commitPointDirectory.FullPath, storedSegmentsFile),
  693. Path.Combine(commitPointDirectory.IndexFullPath, storedSegmentsFile), true);
  694. try
  695. {
  696. // update segments.gen file
  697. using (var genOutput = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN))
  698. {
  699. genOutput.WriteInt(SegmentInfos.FORMAT_LOCKLESS);
  700. genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
  701. genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
  702. }
  703. }
  704. catch (Exception)
  705. {
  706. // here we can ignore, segments.gen is used only as fallback
  707. }
  708. if (File.Exists(commitPointDirectory.DeletedKeysFile))
  709. keysToDelete = File.ReadLines(commitPointDirectory.DeletedKeysFile).ToArray();
  710. return true;
  711. }
  712. catch (Exception ex)
  713. {
  714. startupLog.WarnException("Could not recover an index named '" + indexDefinition.IndexId +
  715. "'from segments of the following generation " + commitPointDirectoryName, ex);
  716. }
  717. }
  718. return false;
  719. }
  720. public static IndexSegmentsInfo GetCurrentSegmentsInfo(string indexName, Directory directory)
  721. {
  722. var segmentInfos = new SegmentInfos();
  723. var result = new IndexSegmentsInfo();
  724. try
  725. {
  726. segmentInfos.Read(directory);
  727. result.Generation = segmentInfos.Generation;
  728. result.SegmentsFileName = segmentInfos.GetCurrentSegmentFileName();
  729. result.ReferencedFiles = segmentInfos.Files(directory, false);
  730. }
  731. catch (CorruptIndexException ex)
  732. {
  733. log.WarnException(string.Format("Could not read segment information for an index '{0}'", indexName), ex);
  734. result.IsIndexCorrupted = true;
  735. }
  736. return result;
  737. }
  738. public static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit)
  739. {
  740. using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath))
  741. {
  742. try
  743. {
  744. var textReader = new JsonTextReader(new StreamReader(commitPointFile));
  745. var jsonCommitPoint = RavenJObject.Load(textReader);
  746. var jsonEtag = jsonCommitPoint.Value<RavenJToken>("HighestCommitedETag");
  747. Etag recoveredEtag = null;
  748. if (jsonEtag.Type == JTokenType.Object) // backward compatibility - HighestCommitedETag is written as {"Restarts":123,"Changes":1}
  749. {
  750. jsonCommitPoint.Remove("HighestCommitedETag");
  751. recoveredEtag = new Etag(UuidType.Documents, jsonEtag.Value<long>("Restarts"), jsonEtag.Value<long>("Changes"));
  752. }
  753. indexCommit = jsonCommitPoint.JsonDeserialization<IndexCommitPoint>();
  754. if (indexCommit == null)
  755. return false;
  756. if (recoveredEtag != null)
  757. indexCommit.HighestCommitedETag = recoveredEtag;
  758. if (indexCommit.HighestCommitedETag == null || indexCommit.HighestCommitedETag.CompareTo(Etag.Empty) == 0)
  759. return false;
  760. return true;
  761. }
  762. catch (Exception e)
  763. {
  764. log.Warn("Could not get commit point from the following location {0}. Exception {1}", commitPointDirectory.FileFullPath, e);
  765. indexCommit = null;
  766. return false;
  767. }
  768. }
  769. }
  770. internal Directory MakeRAMDirectoryPhysical(RAMDirectory ramDir, IndexDefinition indexDefinition)
  771. {
  772. var newDir = new LuceneCodecDirectory(Path.Combine(path, indexDefinition.IndexId.ToString()), documentDatabase.IndexCodecs.OfType<AbstractIndexCodec>());
  773. Directory.Copy(ramDir, newDir, false);
  774. return newDir;
  775. }
  776. private Index CreateIndexImplementation(IndexDefinition indexDefinition, Directory directory)
  777. {
  778. var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexDefinition.IndexId);
  779. var indexImplementation = indexDefinition.IsMapReduce
  780. ? (Index)new MapReduceIndex(directory, indexDefinition.IndexId, indexDefinition, viewGenerator, documentDatabase.WorkContext)
  781. : new SimpleIndex(directory, indexDefinition.IndexId, indexDefinition, viewGenerator, documentDatabase.WorkContext);
  782. configuration.Container.SatisfyImportsOnce(indexImplementation);
  783. return indexImplementation;
  784. }
  785. public int[] Indexes
  786. {
  787. get { return indexes.Keys.ToArray(); }
  788. }
  789. public string[] IndexNames
  790. {
  791. get { return indexes.Values.Select(x => x.PublicName).ToArray(); }
  792. }
  793. public bool HasIndex(string index)
  794. {
  795. if (index == null)
  796. return false;
  797. return indexes.Any(x => String.Compare(index, x.Value.PublicName, StringComparison.OrdinalIgnoreCase) == 0);
  798. }
  799. public void Dispose()
  800. {
  801. var exceptionAggregator = new ExceptionAggregator(log, "Could not properly close index storage");
  802. exceptionAggregator.Execute(FlushMapIndexes);
  803. exceptionAggregator.Execute(FlushReduceIndexes);
  804. exceptionAggregator.Execute(() => Parallel.ForEach(indexes.Values, index => exceptionAggregator.Execute(index.Dispose)));
  805. exceptionAggregator.Execute(() => dummyAnalyzer.Close());
  806. exceptionAggregator.Execute(() =>
  807. {
  808. if (crashMarker != null)
  809. crashMarker.Dispose();
  810. });
  811. exceptionAggregator.ThrowIfNeeded();
  812. }
  813. public void DeleteIndex(string name)
  814. {
  815. var value = TryIndexByName(name);
  816. if (value == null)
  817. return;
  818. DeleteIndex(value.indexId);
  819. }
  820. public void DeleteIndex(int id)
  821. {
  822. var value = GetIndexInstance(id);
  823. if (value == null)
  824. {
  825. if (log.IsDebugEnabled)
  826. log.Debug("Ignoring delete for non existing index {0}", id);
  827. return;
  828. }
  829. documentDatabase.TransactionalStorage.Batch(accessor =>
  830. accessor.Lists.Remove("Raven/Indexes/QueryTime", value.PublicName));
  831. if (log.IsDebugEnabled)
  832. log.Debug("Deleting index {0}", value.PublicName);
  833. value.Dispose();
  834. Index ignored;
  835. var dirOnDisk = Path.Combine(path, id.ToString());
  836. if (!indexes.TryRemove(id, out ignored) || !System.IO.Directory.Exists(dirOnDisk))
  837. return;
  838. UpdateIndexMappingFile();
  839. }
  840. public void DeleteIndexData(int id)
  841. {
  842. var dirOnDisk = Path.Combine(path, id.ToString(CultureInfo.InvariantCulture));
  843. IOExtensions.DeleteDirectory(dirOnDisk);
  844. }
  845. public Index ReopenCorruptedIndex(Index index)
  846. {
  847. if (index.Priority != IndexingPriority.Error)
  848. throw new InvalidOperationException(string.Format("Index {0} isn't errored", index.PublicName));
  849. index.Dispose();
  850. var reopened = OpenIndex(index.PublicName, onStartup: false, forceFullIndexCheck: true);
  851. if (reopened == null)
  852. throw new InvalidOperationException("Reopened index cannot be null instance. Index name:" + index.PublicName);
  853. return indexes.AddOrUpdate(reopened.IndexId, n => reopened, (s, existigIndex) => reopened);
  854. }
  855. public void CreateIndexImplementation(IndexDefinition indexDefinition)
  856. {
  857. if (log.IsDebugEnabled)
  858. log.Debug("Creating index {0} with id {1}", indexDefinition.IndexId, indexDefinition.Name);
  859. IndexDefinitionStorage.ResolveAnalyzers(indexDefinition);
  860. if (TryIndexByName(indexDefinition.Name) != null)
  861. {
  862. throw new InvalidOperationException("Index " + indexDefinition.Name + " already exists");
  863. }
  864. var addedIndex = indexes.AddOrUpdate(indexDefinition.IndexId, n =>
  865. {
  866. var directory = OpenOrCreateLuceneDirectory(indexDefinition);
  867. return CreateIndexImplementation(indexDefinition, directory);
  868. }, (s, index) => index);
  869. //prevent corrupted index when creating a map-reduce index
  870. //need to do this for every map reduce index, even when indexing is enabled,
  871. if (addedIndex.IsMapReduce)
  872. {
  873. addedIndex.EnsureIndexWriter();
  874. addedIndex.Flush(Etag.Empty);
  875. }
  876. UpdateIndexMappingFile();
  877. }
  878. public Query GetDocumentQuery(string index, IndexQuery query, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
  879. {
  880. var value = TryIndexByName(index);
  881. if (value == null)
  882. {
  883. if (log.IsDebugEnabled)
  884. log.Debug("Query on non existing index {0}", index);
  885. throw new InvalidOperationException("Index '" + index + "' does not exists");
  886. }
  887. var fieldsToFetch = new FieldsToFetch(new string[0], false, null);
  888. return new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetDocumentQuery();
  889. }
  890. private Index TryIndexByName(string name)
  891. {
  892. return indexes.Where(index => String.Compare(index.Value.PublicName, name, StringComparison.OrdinalIgnoreCase) == 0)
  893. .Select(x => x.Value)
  894. .FirstOrDefault();
  895. }
  896. public IEnumerable<IndexQueryResult> Query(string index,
  897. IndexQuery query,
  898. Func<IndexQueryResult, bool> shouldIncludeInResults,
  899. FieldsToFetch fieldsToFetch,
  900. OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers,
  901. CancellationToken token,
  902. Action<double> parseTiming = null
  903. )
  904. {
  905. Index value = TryIndexByName(index);
  906. if (value == null)
  907. {
  908. if (log.IsDebugEnabled)
  909. log.Debug("Query on non existing index '{0}'", index);
  910. throw new InvalidOperationException("Index '" + index + "' does not exists");
  911. }
  912. if ((value.Priority.HasFlag(IndexingPriority.Idle) || value.Priority.HasFlag(IndexingPriority.Abandoned)) &&
  913. value.Priority.HasFlag(IndexingPriority.Forced) == false)
  914. {
  915. documentDatabase.TransactionalStorage.Batch(accessor =>
  916. {
  917. value.Priority = IndexingPriority.Normal;
  918. try
  919. {
  920. accessor.Indexing.SetIndexPriority(value.indexId, IndexingPriority.Normal);
  921. }
  922. catch (Exception e)
  923. {
  924. if (accessor.IsWriteConflict(e) == false)
  925. throw;
  926. // we explciitly ignore write conflicts here, it is okay if we got set twice (two concurrent queries, or setting while indexing).
  927. }
  928. documentDatabase.WorkContext.ShouldNotifyAboutWork(() => "Idle index queried");
  929. documentDatabase.Notifications.RaiseNotifications(new IndexChangeNotification()
  930. {
  931. Name = value.PublicName,
  932. Type = IndexChangeTypes.IndexPromotedFromIdle
  933. });
  934. });
  935. }
  936. var indexQueryOperation = new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers);
  937. if (parseTiming != null)
  938. parseTiming(indexQueryOperation.QueryParseDuration.TotalMilliseconds);
  939. if (query.Query != null && query.Query.Contains(Constants.IntersectSeparator))
  940. return indexQueryOperation.IntersectionQuery(token);
  941. return indexQueryOperation.Query(token);
  942. }
  943. public IEnumerable<RavenJObject> IndexEntires(
  944. string indexName,
  945. IndexQuery query,
  946. List<string> reduceKeys,
  947. OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers,
  948. Reference<int> totalResults)
  949. {
  950. Index value = TryIndexByName(indexName);
  951. if (value == null)
  952. {
  953. if (log.IsDebugEnabled)
  954. log.Debug("Query on non existing index '{0}'", indexName);
  955. throw new InvalidOperationException("Index '" + indexName + "' does not exists");
  956. }
  957. var indexQueryOperation = new Index.IndexQueryOperation(value, query, null, new FieldsToFetch(null, false, null), indexQueryTriggers, reduceKeys);
  958. return indexQueryOperation.IndexEntries(totalResults);
  959. }
  960. public void RemoveFromIndex(int index, string[] keys, WorkContext context)
  961. {
  962. Index value;
  963. if (indexes.TryGetValue(index, out value) == false)
  964. {
  965. if (log.IsDebugEnabled)
  966. log.Debug("Removing from non existing index '{0}', ignoring", index);
  967. return;
  968. }
  969. value.Remove(keys, context);
  970. context.RaiseIndexChangeNotification(new IndexChangeNotification
  971. {
  972. Name = value.PublicName,
  973. Type = IndexChangeTypes.RemoveFromIndex
  974. });
  975. }
  976. [CLSCompliant(false)]
  977. public IndexingPerformanceStats Index(int index, AbstractViewGenerator viewGenerator, IndexingBatch batch, WorkContext context, IStorageActionsAccessor actions, DateTime minimumTimestamp, CancellationToken token)
  978. {
  979. Index value;
  980. if (indexes.TryGetValue(index, out value) == false)
  981. {
  982. if (log.IsDebugEnabled)
  983. log.Debug("Tried to index on a non existent index {0}, ignoring", index);
  984. return null;
  985. }
  986. using (CultureHelper.EnsureInvariantCulture())
  987. using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
  988. {
  989. var performance = value.IndexDocuments(viewGenerator, batch, actions, minimumTimestamp, token);
  990. context.RaiseIndexChangeNotification(new IndexChangeNotification
  991. {
  992. Name = value.PublicName,
  993. Type = IndexChangeTypes.MapCompleted
  994. });
  995. return performance;
  996. }
  997. }
  998. [CLSCompliant(false)]
  999. public IndexingPerformanceStats Reduce(
  1000. int index,
  1001. AbstractViewGenerator viewGenerator,
  1002. IEnumerable<IGrouping<int, object>> mappedResults,
  1003. int level,
  1004. WorkContext context,
  1005. IStorageActionsAccessor actions,
  1006. HashSet<string> reduceKeys,
  1007. int inputCount)
  1008. {
  1009. Index value;
  1010. if (indexes.TryGetValue(index, out value) == false)
  1011. {
  1012. if (log.IsDebugEnabled)
  1013. log.Debug("Tried to index on a non existent index {0}, ignoring", index);
  1014. return null;
  1015. }
  1016. var mapReduceIndex = value as MapReduceIndex;
  1017. if (mapReduceIndex == null)
  1018. {
  1019. log.Warn("Tried to reduce on an index that is not a map/reduce index: {0}, ignoring", index);
  1020. return null;
  1021. }
  1022. using (CultureHelper.EnsureInvariantCulture())
  1023. {
  1024. var reduceDocuments = new MapReduceIndex.ReduceDocuments(mapReduceIndex, viewGenerator, mappedResults, level, context, actions, reduceKeys, inputCount);
  1025. var performance = reduceDocuments.ExecuteReduction();
  1026. context.RaiseIndexChangeNotification(new IndexChangeNotification
  1027. {
  1028. Name = value.PublicName,
  1029. Type = IndexChangeTypes.ReduceCompleted
  1030. });
  1031. return performance;
  1032. }
  1033. }
  1034. internal IndexSearcherHolder.IndexSearcherHoldingState GetCurrentStateHolder(string indexName)
  1035. {
  1036. return GetIndexByName(indexName).GetCurrentStateHolder();
  1037. }
  1038. public IDisposable GetCurrentIndexSearcher(int indexId, out IndexSearcher searcher)
  1039. {
  1040. return GetIndexInstance(indexId).GetSearcher(out searcher);
  1041. }
  1042. public IDisposable GetCurrentIndexSearcherAndTermDocs(string indexName, out IndexSearcher searcher, out RavenJObject[] termsDocs)
  1043. {
  1044. return GetIndexByName(indexName).GetSearcherAndTermsDocs(out searcher, out termsDocs);
  1045. }
  1046. private Index GetIndexByName(string indexName)
  1047. {
  1048. var result = TryIndexByName(indexName);
  1049. if (result == null)
  1050. throw new InvalidOperationException(string.Format("Index '{0}' does not exist", indexName));
  1051. return result;
  1052. }
  1053. public void RunIdleOperations()
  1054. {
  1055. foreach (var value in indexes.Values)
  1056. {
  1057. if ((SystemTime.UtcNow - value.LastIndexTime).TotalMinutes < 1)
  1058. continue;
  1059. value.Flush(value.GetLastEtagFromStats());
  1060. }
  1061. SetUnusedIndexesToIdle();
  1062. UpdateLatestPersistedQueryTime();
  1063. DeleteSurpassedAutoIndexes();
  1064. }
  1065. public bool IsIndexStale(string indexName, LastCollectionEtags lastCollectionEtags)
  1066. {
  1067. var index = TryIndexByName(indexName);
  1068. if (index == null)
  1069. throw new InvalidOperationException("Could not find index " + indexName);
  1070. return IsIndexStale(index.IndexId, lastCollectionEtags);
  1071. }
  1072. public bool IsIndexStale(int indexId, LastCollectionEtags lastCollectionEtags)
  1073. {
  1074. bool isStale = false;
  1075. documentDatabase.TransactionalStorage.Batch(actions =>
  1076. {
  1077. Index indexInstance = GetIndexInstance(indexId);
  1078. isStale = (indexInstance != null && indexInstance.IsMapIndexingInProgress) || actions.Staleness.IsIndexStale(indexId, null, null);
  1079. if (indexInstance != null && indexInstance.IsTestIndex)
  1080. isStale = false;
  1081. if (isStale && actions.Staleness.IsIndexStaleByTask(indexId, null) == false && actions.Staleness.IsReduceStale(indexId) == false)
  1082. {
  1083. var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexId);
  1084. if (viewGenerator == null)
  1085. return;
  1086. var indexStats = actions.Indexing.GetIndexStats(indexId);
  1087. if (indexStats == null)
  1088. return;
  1089. var lastIndexedEtag = indexStats.LastIndexedEtag;
  1090. var collectionNames = viewGenerator.ForEntityNames.ToList();
  1091. if (lastCollectionEtags.HasEtagGreaterThan(collectionNames, lastIndexedEtag) == false)
  1092. isStale = false;
  1093. }
  1094. });
  1095. return isStale;
  1096. }
  1097. private void DeleteSurpassedAutoIndexes()
  1098. {
  1099. if (indexes.Any(x => x.Value.PublicName.StartsWith("Auto/", StringComparison.InvariantCultureIgnoreCase)) == false)
  1100. return;
  1101. var mergeSuggestions = indexDefinitionStorage.ProposeIndexMergeSuggestions();
  1102. foreach (var mergeSuggestion in mergeSuggestions.Suggestions)
  1103. {
  1104. if (string.IsNullOrEmpty(mergeSuggestion.SurpassingIndex))
  1105. continue;
  1106. if (mergeSuggestion.CanDelete.Any(x => x.StartsWith("Auto/", StringComparison.InvariantCultureIgnoreCase)) == false)
  1107. continue;
  1108. if (IsIndexStale(mergeSuggestion.SurpassingIndex, documentDatabase.LastCollectionEtags))
  1109. continue;
  1110. foreach (var indexToDelete in mergeSuggestion.CanDelete.Where(x => x.StartsWith("Auto/", StringComparison.InvariantCultureIgnoreCase)))
  1111. {
  1112. documentDatabase.Indexes.DeleteIndex(indexToDelete);
  1113. }
  1114. }
  1115. }
  1116. private void UpdateLatestPersistedQueryTime()
  1117. {
  1118. documentDatabase.TransactionalStorage.Batch(accessor =>
  1119. {
  1120. var maxDate = latestPersistedQueryTime;
  1121. foreach (var index in indexes)
  1122. {
  1123. var lastQueryTime = index.Value.LastQueryTime ?? DateTime.MinValue;
  1124. if (lastQueryTime <= latestPersistedQueryTime)
  1125. continue;
  1126. accessor.Lists.Set("Raven/Indexes/QueryTime", index.Value.PublicName, new RavenJObject
  1127. {
  1128. {"LastQueryTime", lastQueryTime}
  1129. }, UuidType.Indexing);
  1130. if (lastQueryTime > maxDate)
  1131. maxDate = lastQueryTime;
  1132. }
  1133. latestPersistedQueryTime = maxDate;
  1134. });
  1135. }
  1136. public class UnusedIndexState
  1137. {
  1138. public DateTime LastQueryTime { get; set; }
  1139. public Index Index { get; set; }
  1140. public string Name { get; set; }
  1141. public IndexingPriority Priority { get; set; }
  1142. public DateTime CreationDate { get; set; }
  1143. }
  1144. private void SetUnusedIndexesToIdle()
  1145. {
  1146. documentDatabase.TransactionalStorage.Batch(accessor =>
  1147. {
  1148. var autoIndexesSortedByLastQueryTime =
  1149. (from index in indexes
  1150. let stats = GetIndexStats(accessor, index.Key)
  1151. where stats != null
  1152. let lastQueryTime = stats.LastQueryTimestamp ?? DateTime.MinValue
  1153. where index.Value.PublicName.StartsWith("Auto/", StringComparison.InvariantCultureIgnoreCase)
  1154. orderby lastQueryTime
  1155. select new UnusedIndexState
  1156. {
  1157. LastQueryTime = lastQueryTime,
  1158. Index = index.Value,
  1159. Name = index.Value.PublicName,
  1160. Priority = stats.Priority,
  1161. CreationDate = stats.CreatedTimestamp
  1162. }).ToArray();
  1163. var timeToWaitBeforeMarkingAutoIndexAsIdle = documentDatabase.Configuration.Indexing.TimeToWaitBeforeMarkingAutoIndexAsIdle.AsTimeSpan;
  1164. var timeToWaitForIdleMinutes = timeToWaitBeforeMarkingAutoIndexAsIdle.TotalMinutes * 10;
  1165. for (var i = 0; i < autoIndexesSortedByLastQueryTime.Length; i++)
  1166. {
  1167. var thisItem = autoIndexesSortedByLastQueryTime[i];
  1168. if (thisItem.Priority.HasFlag(IndexingPriority.Disabled) || // we don't really have much to say about those in here
  1169. thisItem.Priority.HasFlag(IndexingPriority.Error) || // no need to touch erroring indexes
  1170. thisItem.Priority.HasFlag(IndexingPriority.Forced))// if it is forced, we can't change it
  1171. continue;
  1172. var age = (SystemTime.UtcNow - thisItem.CreationDate).TotalMinutes;
  1173. var lastQuery = (SystemTime.UtcNow - thisItem.LastQueryTime).TotalMinutes;
  1174. if (thisItem.Priority.HasFlag(IndexingPriority.Normal))
  1175. {
  1176. if (age < timeToWaitForIdleMinutes)
  1177. {
  1178. HandleActiveIndex(thisItem, age, lastQuery, accessor, timeToWaitForIdleMinutes);
  1179. }
  1180. else
  1181. {
  1182. // If it's a fairly established query then we need to determine whether there is any activity currently
  1183. // If there is activity and this has not been queried against 'recently' it needs idling
  1184. if (i < autoIndexesSortedByLastQueryTime.Length - 1)
  1185. {
  1186. var nextItem = autoIndexesSortedByLastQueryTime[i + 1];
  1187. if ((nextItem.LastQueryTime - thisItem.LastQueryTime).TotalMinutes > timeToWaitForIdleMinutes)
  1188. {
  1189. accessor.Indexing.SetIndexPriority(thisItem.Index.indexId, IndexingPriority.Idle);
  1190. thisItem.Index.Priority = IndexingPriority.Idle;
  1191. documentDatabase.Notifications.RaiseNotifications(new IndexChangeNotification()
  1192. {
  1193. Name = thisItem.Name,
  1194. Type = IndexChangeTypes.IndexDemotedToIdle
  1195. });
  1196. }
  1197. }
  1198. }
  1199. continue;
  1200. }
  1201. if (thisItem.Priority.HasFlag(IndexingPriority.Idle))
  1202. {
  1203. HandleIdleIndex(age, lastQuery, thisItem, accessor);
  1204. continue;
  1205. }
  1206. }
  1207. });
  1208. }
  1209. private IndexStats GetIndexStats(IStorageActionsAccessor accessor, int indexId)
  1210. {
  1211. var indexStats = accessor.Indexing.GetIndexStats(indexId);
  1212. if (indexStats == null)
  1213. return null;
  1214. indexStats.LastQueryTimestamp = GetLastQueryTime(indexId);
  1215. return indexStats;
  1216. }
  1217. private void HandleIdleIndex(double age, double lastQuery, UnusedIndexState thisItem,
  1218. IStorageActionsAccessor accessor)
  1219. {
  1220. // relatively young index, haven't been queried for a while already
  1221. // can be safely removed, probably
  1222. if (age < 90 && lastQuery > 30)
  1223. {
  1224. accessor.Indexing.DeleteIndex(thisItem.Index.indexId, documentDatabase.WorkContext.CancellationToken);
  1225. return;
  1226. }
  1227. if (lastQuery < configuration.Indexing.TimeToWaitBeforeMarkingIdleIndexAsAbandoned.AsTimeSpan.TotalMinutes)
  1228. return;
  1229. // old enough, and haven't been queried for a while, mark it as abandoned
  1230. accessor.Indexing.SetIndexPriority(thisItem.Index.indexId, IndexingPriority.Abandoned);
  1231. thisItem.Index.Priority = IndexingPriority.Abandoned;
  1232. documentDatabase.Notifications.RaiseNotifications(new IndexChangeNotification()
  1233. {
  1234. Name = thisItem.Name,
  1235. Type = IndexChangeTypes.IndexDemotedToAbandoned
  1236. });
  1237. }
  1238. private void HandleActiveIndex(UnusedIndexState thisItem, double age, double lastQuery, IStorageActionsAccessor accessor, double timeToWaitForIdle)
  1239. {
  1240. if (age < (timeToWaitForIdle * 2.5) && lastQuery < (1.5 * timeToWaitForIdle))
  1241. return;
  1242. if (age < (timeToWaitForIdle * 6) && lastQuery < (2.5 * timeToWaitForIdle))
  1243. return;
  1244. accessor.Indexing.SetIndexPriority(thisItem.Index.indexId, IndexingPriority.Idle);
  1245. thisItem.Index.Priority = IndexingPriority.Idle;
  1246. documentDatabase.Notifications.RaiseNotifications(new IndexChangeNotification()
  1247. {
  1248. Name = thisItem.Name,
  1249. Type = IndexChangeTypes.IndexDemotedToIdle
  1250. });
  1251. }
  1252. private void UpdateIndexMappingFile()
  1253. {
  1254. if (configuration.Core.RunInMemory)
  1255. return;
  1256. var sb = new StringBuilder();
  1257. foreach (var index in indexes)
  1258. {
  1259. sb.Append(string.Format("{0} - {1}{2}", index.Value.IndexId, index.Value.PublicName, Environment.NewLine));
  1260. }
  1261. File.WriteAllText(Path.Combine(path, "indexes.txt"), sb.ToString());
  1262. }
  1263. public void FlushMapIndexes()
  1264. {
  1265. if (indexes == null)
  1266. return;
  1267. foreach (var index in indexes)
  1268. {
  1269. if (index.Value.IsMapReduce == false)
  1270. FlushIndex(index.Value);
  1271. }
  1272. }
  1273. public void FlushReduceIndexes()
  1274. {
  1275. if (indexes == null)
  1276. return;
  1277. foreach (var index in indexes)
  1278. {
  1279. if (index.Value.IsMapReduce)
  1280. FlushIndex(index.Value);
  1281. }
  1282. }
  1283. public void FlushIndexes(HashSet<int> indexIds)
  1284. {
  1285. if (indexes == null || indexIds.Count == 0)
  1286. return;
  1287. foreach (var indexId in indexIds)
  1288. {
  1289. FlushIndex(indexId);
  1290. }
  1291. }
  1292. public void FlushIndex(int indexId)
  1293. {
  1294. Index value;
  1295. if (indexes.TryGetValue(indexId, out value))
  1296. FlushIndex(value);
  1297. }
  1298. private static void FlushIndex(Index value)
  1299. {
  1300. var sp = Stopwatch.StartNew();
  1301. try
  1302. {
  1303. value.Flush(value.GetLastEtagFromStats());
  1304. }
  1305. catch (Exception e)
  1306. {
  1307. value.HandleWriteError(e);
  1308. log.WarnException(string.Format("Failed to flush {0} index: {1} (id: {2})",
  1309. GetIndexType(value.IsMapReduce), value.PublicName, value.IndexId), e);
  1310. throw;
  1311. }
  1312. if (log.IsDebugEnabled)
  1313. {
  1314. log.Debug("Flashed {0} index: {1} (id: {2}), took {3}ms",
  1315. GetIndexType(value.IsMapReduce), value.PublicName, value.IndexId, sp.ElapsedMilliseconds);
  1316. }
  1317. }
  1318. private static string GetIndexType(bool isMapReduce)
  1319. {
  1320. return isMapReduce ? "map-reduce" : "simple map";
  1321. }
  1322. public List<int> GetDisabledIndexIds()
  1323. {
  1324. var indexIds = new List<int>();
  1325. foreach (var index in indexes)
  1326. {
  1327. if (index.Value.Priority.HasFlag(IndexingPriority.Disabled))
  1328. indexIds.Add(index.Key);
  1329. }
  1330. return indexIds;
  1331. }
  1332. public IIndexExtension GetIndexExtension(string index, string indexExtensionKey)
  1333. {
  1334. return GetIndexByName(index).GetExtension(indexExtensionKey);
  1335. }
  1336. public IIndexExtension GetIndexExtensionByPrefix(string index, string indexExtensionKeyPrefix)
  1337. {
  1338. return GetIndexByName(index).GetExtensionByPrefix(indexExtensionKeyPrefix);
  1339. }
  1340. public void SetIndexExtension(string indexName, string indexExtensionKey, IIndexExtension suggestionQueryIndexExtension)
  1341. {
  1342. GetIndexByName(indexName).SetExtension(indexExtensionKey, suggestionQueryIndexExtension);
  1343. }
  1344. public Index GetIndexInstance(string indexName)
  1345. {
  1346. return TryIndexByName(indexName);
  1347. }
  1348. public Index GetIndexInstance(int indexId)
  1349. {
  1350. Index value;
  1351. indexes.TryGetValue(indexId, out value);
  1352. return value;
  1353. }
  1354. public void MarkCachedQuery(string indexName)
  1355. {
  1356. GetIndexByName(indexName).MarkQueried();
  1357. }
  1358. internal void SetLastQueryTime(string indexName, DateTime lastQueryTime)
  1359. {
  1360. GetIndexByName(indexName).MarkQueried(lastQueryTime);
  1361. }
  1362. public DateTime? GetLastQueryTime(int index)
  1363. {
  1364. return GetIndexInstance(index).LastQueryTime;
  1365. }
  1366. public DateTime? GetLastQueryTime(string index)
  1367. {
  1368. return GetIndexInstance(index).LastQueryTime;
  1369. }
  1370. public IndexingPerformanceStats[] GetIndexingPerformance(int index)
  1371. {
  1372. return GetIndexInstance(index).GetIndexingPerformance();
  1373. }
  1374. public void Backup(string directory, string incrementalTag = null, Action<string, string, BackupStatus.BackupMessageSeverity> notifyCallback = null)
  1375. {
  1376. Parallel.ForEach(indexes.Values, index =>
  1377. index.Backup(directory, path, incrementalTag, notifyCallback));
  1378. }
  1379. public void MergeAllIndexes()
  1380. {
  1381. Parallel.ForEach(indexes.Values, index =>
  1382. index.MergeSegments());
  1383. }
  1384. public string IndexOnRam(int id)
  1385. {
  1386. return GetIndexInstance(id).IsOnRam;
  1387. }
  1388. public void ForceWriteToDiskAndWriteInMemoryIndexToDiskIfNecessary(string indexName)
  1389. {
  1390. var index = GetIndexByName(indexName);
  1391. index.ForceWriteToDisk();
  1392. index.WriteInMemoryIndexToDiskIfNecessary(Etag.Empty);
  1393. }
  1394. internal bool TryReplaceIndex(string indexName, string indexToReplaceName)
  1395. {
  1396. var indexToReplace = indexDefinitionStorage.GetIndexDefinition(indexToReplaceName);
  1397. var success = indexDefinitionStorage.ReplaceIndex(indexName, indexToReplaceName);
  1398. if (success == false)
  1399. return false;
  1400. if (indexToReplace == null)
  1401. return true;
  1402. documentDatabase.Indexes.DeleteIndex(indexToReplace, removeByNameMapping: false, clearErrors: false, isSideBySideReplacement: true);
  1403. return true;
  1404. }
  1405. }
  1406. }