PageRenderTime 58ms CodeModel.GetById 31ms RepoModel.GetById 0ms app.codeStats 0ms

/ToMigrate/Raven.Database/Indexing/IndexStorage.cs

http://github.com/ayende/ravendb
C# | 1691 lines | 1359 code | 285 blank | 47 comment | 242 complexity | 5b0b2dda17805dbd210707531b5ba65d MD5 | raw file
Possible License(s): GPL-3.0, MPL-2.0-no-copyleft-exception, LGPL-2.1, Apache-2.0, BSD-3-Clause, CC-BY-SA-3.0

Large files files are truncated, but you can click here to view the full file

  1. //-----------------------------------------------------------------------
  2. // <copyright file="IndexStorage.cs" company="Hibernating Rhinos LTD">
  3. // Copyright (c) Hibernating Rhinos LTD. All rights reserved.
  4. // </copyright>
  5. //-----------------------------------------------------------------------
  6. using System;
  7. using System.Collections.Concurrent;
  8. using System.Collections.Generic;
  9. using System.ComponentModel.Composition;
  10. using System.Diagnostics;
  11. using System.Globalization;
  12. using System.IO;
  13. using System.Linq;
  14. using System.Runtime.ConstrainedExecution;
  15. using System.Text;
  16. using System.Threading;
  17. using System.Threading.Tasks;
  18. using Lucene.Net.Analysis;
  19. using Lucene.Net.Index;
  20. using Lucene.Net.Search;
  21. using Lucene.Net.Store;
  22. using Lucene.Net.Util;
  23. using Raven.Abstractions;
  24. using Raven.Abstractions.Data;
  25. using Raven.Abstractions.Extensions;
  26. using Raven.Abstractions.Indexing;
  27. using Raven.Abstractions.Logging;
  28. using Raven.Abstractions.MEF;
  29. using Raven.Abstractions.Util;
  30. using Raven.Database.Actions;
  31. using Raven.Database.Config;
  32. using Raven.Database.Data;
  33. using Raven.Database.Extensions;
  34. using Raven.Database.Impl;
  35. using Raven.Database.Linq;
  36. using Raven.Database.Plugins;
  37. using Raven.Database.Queries;
  38. using Raven.Database.Storage;
  39. using Raven.Database.Util;
  40. using Raven.Imports.Newtonsoft.Json;
  41. using Raven.Imports.Newtonsoft.Json.Linq;
  42. using Raven.Json.Linq;
  43. using Constants = Raven.Abstractions.Data.Constants;
  44. using Directory = Lucene.Net.Store.Directory;
  45. namespace Raven.Database.Indexing
  46. {
  47. /// <summary>
  48. /// Thread safe, single instance for the entire application
  49. /// </summary>
  50. public class IndexStorage : CriticalFinalizerObject, IDisposable
  51. {
  52. private readonly DocumentDatabase documentDatabase;
  53. private const string IndexVersion = "2.0.0.1";
  54. private const string MapReduceIndexVersion = "2.5.0.1";
  55. private readonly IndexDefinitionStorage indexDefinitionStorage;
  56. private readonly RavenConfiguration configuration;
  57. private readonly string path;
  58. private static readonly ILog log = LogManager.GetCurrentClassLogger();
  59. private static readonly ILog startupLog = LogManager.GetLogger(typeof(IndexStorage).FullName + ".Startup");
  60. private readonly Analyzer dummyAnalyzer = new SimpleAnalyzer();
  61. private DateTime latestPersistedQueryTime;
  62. private readonly FileStream crashMarker;
  63. private ConcurrentDictionary<int, Index> indexes =
  64. new ConcurrentDictionary<int, Index>();
  65. public class RegisterLowMemoryHandler : ILowMemoryHandler
  66. {
  67. static RegisterLowMemoryHandler _instance;
  68. public static void Setup()
  69. {
  70. if (_instance != null)
  71. return;
  72. lock (typeof(RegisterLowMemoryHandler))
  73. {
  74. if (_instance != null)
  75. return;
  76. _instance = new RegisterLowMemoryHandler();
  77. MemoryStatistics.RegisterLowMemoryHandler(_instance);
  78. }
  79. }
  80. public void HandleLowMemory()
  81. {
  82. FieldCache_Fields.DEFAULT.PurgeAllCaches();
  83. }
  84. public void SoftMemoryRelease()
  85. {
  86. }
  87. public LowMemoryHandlerStatistics GetStats()
  88. {
  89. var cacheEntries = FieldCache_Fields.DEFAULT.GetCacheEntries();
  90. var memorySum = cacheEntries.Sum(x =>
  91. {
  92. var curEstimator = new RamUsageEstimator(false);
  93. return curEstimator.EstimateRamUsage(x);
  94. });
  95. return new LowMemoryHandlerStatistics
  96. {
  97. Name = "LuceneLowMemoryHandler",
  98. EstimatedUsedMemory = memorySum,
  99. Metadata = new
  100. {
  101. CachedEntriesAmount = cacheEntries.Length
  102. }
  103. };
  104. }
  105. }
  106. public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, RavenConfiguration configuration, DocumentDatabase documentDatabase)
  107. {
  108. try
  109. {
  110. RegisterLowMemoryHandler.Setup();
  111. this.indexDefinitionStorage = indexDefinitionStorage;
  112. this.configuration = configuration;
  113. this.documentDatabase = documentDatabase;
  114. path = configuration.Core.IndexStoragePath;
  115. if (System.IO.Directory.Exists(path) == false && configuration.Core.RunInMemory == false)
  116. System.IO.Directory.CreateDirectory(path);
  117. if (configuration.Core.RunInMemory == false)
  118. {
  119. var crashMarkerPath = Path.Combine(path, "indexing.crash-marker");
  120. if (File.Exists(crashMarkerPath))
  121. {
  122. // the only way this can happen is if we crashed because of a power outage
  123. // in this case, we consider all open indexes to be corrupt and force them
  124. // to be reset. This is because to get better perf, we don't flush the files to disk,
  125. // so in the case of a power outage, we can't be sure that there wasn't still stuff in
  126. // the OS buffer that wasn't written yet.
  127. configuration.Indexing.ResetIndexOnUncleanShutdown = true;
  128. }
  129. // The delete on close ensures that the only way this file will exists is if there was
  130. // a power outage while the server was running.
  131. crashMarker = File.Create(crashMarkerPath, 16, FileOptions.DeleteOnClose);
  132. }
  133. if (log.IsDebugEnabled)
  134. log.Debug("Start opening indexes. There are {0} indexes that need to be loaded", indexDefinitionStorage.IndexNames.Length);
  135. BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(documentDatabase.WorkContext, indexDefinitionStorage.IndexNames,
  136. name =>
  137. {
  138. var index = OpenIndex(name, onStartup: true, forceFullIndexCheck: false);
  139. if (index != null)
  140. indexes.TryAdd(index.IndexId, index);
  141. if (startupLog.IsDebugEnabled)
  142. startupLog.Debug("{0}/{1} indexes loaded", indexes.Count, indexDefinitionStorage.IndexNames.Length);
  143. });
  144. if (log.IsDebugEnabled)
  145. log.Debug("Index storage initialized. All indexes have been opened.");
  146. }
  147. catch (Exception e)
  148. {
  149. log.WarnException("Could not create index storage", e);
  150. try
  151. {
  152. Dispose();
  153. }
  154. catch (Exception ex)
  155. {
  156. log.FatalException("Failed to dispose when already getting an error during ctor", ex);
  157. }
  158. throw;
  159. }
  160. }
  161. private Index OpenIndex(string indexName, bool onStartup, bool forceFullIndexCheck)
  162. {
  163. if (indexName == null)
  164. throw new ArgumentNullException("indexName");
  165. if (startupLog.IsDebugEnabled)
  166. startupLog.Debug("Loading saved index {0}", indexName);
  167. var indexDefinition = indexDefinitionStorage.GetIndexDefinition(indexName);
  168. if (indexDefinition == null)
  169. return null;
  170. Index indexImplementation = null;
  171. bool resetTried = false;
  172. bool recoveryTried = false;
  173. string[] keysToDeleteAfterRecovery = null;
  174. while (true)
  175. {
  176. Directory luceneDirectory = null;
  177. try
  178. {
  179. luceneDirectory = OpenOrCreateLuceneDirectory(indexDefinition, createIfMissing: resetTried, forceFullExistingIndexCheck: forceFullIndexCheck);
  180. indexImplementation = CreateIndexImplementation(indexDefinition, luceneDirectory);
  181. CheckIndexState(luceneDirectory, indexDefinition, indexImplementation, resetTried);
  182. if (forceFullIndexCheck)
  183. {
  184. // the above index check might pass however an index writer creation can still throw an exception
  185. // so we need to check it here to avoid crashing in runtime
  186. new IndexWriter(luceneDirectory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
  187. }
  188. var simpleIndex = indexImplementation as SimpleIndex; // no need to do this on m/r indexes, since we rebuild them from saved data anyway
  189. if (simpleIndex != null && keysToDeleteAfterRecovery != null)
  190. {
  191. // remove keys from index that were deleted after creating commit point
  192. simpleIndex.RemoveDirectlyFromIndex(keysToDeleteAfterRecovery, GetLastEtagForIndex(simpleIndex));
  193. }
  194. LoadExistingSuggestionsExtentions(indexDefinition.Name, indexImplementation);
  195. documentDatabase.TransactionalStorage.Batch(accessor =>
  196. {
  197. IndexStats indexStats = accessor.Indexing.GetIndexStats(indexDefinition.IndexId);
  198. if (indexStats != null)
  199. indexImplementation.Priority = indexStats.Priority;
  200. var read = accessor.Lists.Read("Raven/Indexes/QueryTime", indexName);
  201. if (read == null)
  202. {
  203. if (IsIdleAutoIndex(indexImplementation))
  204. indexImplementation.MarkQueried(); // prevent index abandoning right after startup
  205. return;
  206. }
  207. var dateTime = read.Data.Value<DateTime>("LastQueryTime");
  208. if (IsIdleAutoIndex(indexImplementation) && SystemTime.UtcNow - dateTime > configuration.Indexing.TimeToWaitBeforeRunningAbandonedIndexes.AsTimeSpan)
  209. indexImplementation.MarkQueried(); // prevent index abandoning right after startup
  210. else
  211. indexImplementation.MarkQueried(dateTime);
  212. if (dateTime > latestPersistedQueryTime)
  213. latestPersistedQueryTime = dateTime;
  214. });
  215. break;
  216. }
  217. catch (Exception e)
  218. {
  219. if (resetTried)
  220. throw new InvalidOperationException("Could not open / create index" + indexName + ", reset already tried", e);
  221. if (indexImplementation != null)
  222. indexImplementation.Dispose();
  223. if (recoveryTried == false && luceneDirectory != null)
  224. {
  225. recoveryTried = true;
  226. startupLog.WarnException("Could not open index " + indexName + ". Trying to recover index", e);
  227. keysToDeleteAfterRecovery = TryRecoveringIndex(indexDefinition, luceneDirectory);
  228. }
  229. else
  230. {
  231. resetTried = true;
  232. startupLog.WarnException("Could not open index " + indexName + ". Recovery operation failed, forcibly resetting index", e);
  233. TryResettingIndex(indexName, indexDefinition, onStartup);
  234. }
  235. }
  236. }
  237. return indexImplementation;
  238. }
  239. private void CheckIndexState(Directory directory, IndexDefinition indexDefinition, Index index, bool resetTried)
  240. {
  241. //if (configuration.ResetIndexOnUncleanShutdown == false)
  242. // return;
  243. // 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty
  244. // 2. If no 'LastEtag' in commitData then we consider it an invalid index
  245. // 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag)
  246. var commitData = IndexReader.GetCommitUserData(directory);
  247. if (index.IsMapReduce)
  248. CheckMapReduceIndexState(commitData, resetTried);
  249. else
  250. CheckMapIndexState(commitData, indexDefinition, index);
  251. }
  252. private void CheckMapIndexState(IDictionary<string, string> commitData, IndexDefinition indexDefinition, Index index)
  253. {
  254. string value;
  255. Etag lastEtag = null;
  256. if (commitData != null && commitData.TryGetValue("LastEtag", out value))
  257. Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail
  258. var lastStoredEtag = GetLastEtagForIndex(index) ?? Etag.Empty;
  259. lastEtag = lastEtag ?? Etag.Empty;
  260. if (EtagUtil.IsGreaterThanOrEqual(lastEtag, lastStoredEtag))
  261. return;
  262. log.Info(string.Format("Resetting index '{0} ({1})'. Last stored etag: {2}. Last commit etag: {3}.", indexDefinition.Name, index.indexId, lastStoredEtag, lastEtag));
  263. var now = SystemTime.UtcNow;
  264. ResetLastIndexedEtag(indexDefinition, lastEtag, now);
  265. }
  266. private static void CheckMapReduceIndexState(IDictionary<string, string> commitData, bool resetTried)
  267. {
  268. if (resetTried)
  269. return;
  270. string marker;
  271. long commitMarker;
  272. var valid = commitData != null
  273. && commitData.TryGetValue("Marker", out marker)
  274. && long.TryParse(marker, out commitMarker)
  275. && commitMarker == RavenIndexWriter.CommitMarker;
  276. if (valid == false)
  277. throw new InvalidOperationException("Map-Reduce index corruption detected.");
  278. }
  279. private static bool IsIdleAutoIndex(Index index)
  280. {
  281. return index.PublicName.StartsWith("Auto/") && index.Priority == IndexingPriority.Idle;
  282. }
  283. private void TryResettingIndex(string indexName, IndexDefinition indexDefinition, bool onStartup)
  284. {
  285. try
  286. {
  287. Action reset = () =>
  288. {
  289. try
  290. {
  291. documentDatabase.Indexes.DeleteIndex(indexDefinition, removeIndexReplaceDocument: false);
  292. documentDatabase.Indexes.PutNewIndexIntoStorage(indexName, indexDefinition);
  293. var indexReplaceDocumentKey = Constants.IndexReplacePrefix + indexName;
  294. var indexReplaceDocument = documentDatabase.Documents.Get(indexReplaceDocumentKey);
  295. if (indexReplaceDocument == null)
  296. return;
  297. documentDatabase.Documents.Put(indexReplaceDocumentKey, null, indexReplaceDocument.DataAsJson, indexReplaceDocument.Metadata, null);
  298. }
  299. catch (Exception e)
  300. {
  301. throw new InvalidOperationException("Could not finalize reseting of index: " + indexName, e);
  302. }
  303. };
  304. if (onStartup)
  305. {
  306. // we have to defer the work here until the database is actually ready for work
  307. documentDatabase.OnIndexingWiringComplete += reset;
  308. }
  309. else
  310. {
  311. reset();
  312. }
  313. var indexFullPath = Path.Combine(path, indexDefinition.IndexId.ToString(CultureInfo.InvariantCulture));
  314. IOExtensions.DeleteDirectory(indexFullPath);
  315. var suggestionsForIndex = Path.Combine(configuration.Core.IndexStoragePath, "Raven-Suggestions", indexName);
  316. IOExtensions.DeleteDirectory(suggestionsForIndex);
  317. }
  318. catch (Exception exception)
  319. {
  320. throw new InvalidOperationException("Could not reset index " + indexName, exception);
  321. }
  322. }
  323. private string[] TryRecoveringIndex(IndexDefinition indexDefinition,
  324. Directory luceneDirectory)
  325. {
  326. string[] keysToDeleteAfterRecovery = null;
  327. if (indexDefinition.IsMapReduce == false)
  328. {
  329. IndexCommitPoint commitUsedToRestore;
  330. if (TryReusePreviousCommitPointsToRecoverIndex(luceneDirectory,
  331. indexDefinition, path,
  332. out commitUsedToRestore,
  333. out keysToDeleteAfterRecovery))
  334. {
  335. ResetLastIndexedEtag(indexDefinition, commitUsedToRestore.HighestCommitedETag, commitUsedToRestore.TimeStamp);
  336. }
  337. }
  338. else
  339. {
  340. RegenerateMapReduceIndex(luceneDirectory, indexDefinition);
  341. }
  342. return keysToDeleteAfterRecovery;
  343. }
  344. private void LoadExistingSuggestionsExtentions(string indexName, Index indexImplementation)
  345. {
  346. var suggestionsForIndex = Path.Combine(configuration.Core.IndexStoragePath, "Raven-Suggestions", indexName);
  347. if (!System.IO.Directory.Exists(suggestionsForIndex))
  348. return;
  349. try
  350. {
  351. var directories = System.IO.Directory.GetDirectories(suggestionsForIndex);
  352. if (directories.Any(dir => dir.Contains("-")))
  353. {
  354. // Legacy handling:
  355. // Previously we had separate folder with suggestions for each triple: (field, distanceType, accuracy)
  356. // Now we have field only.
  357. // Legacy naming convention was: field-{distanceType}-{accuracy}
  358. // since when we have - (dash) in SOME folder name it seems to be legacy
  359. HandleLegacySuggestions(directories);
  360. // Refresh directories list as handling legacy might rename or delete some of them.
  361. directories = System.IO.Directory.GetDirectories(suggestionsForIndex);
  362. }
  363. foreach (var directory in directories)
  364. {
  365. IndexSearcher searcher;
  366. using (indexImplementation.GetSearcher(out searcher))
  367. {
  368. var key = Path.GetFileName(directory);
  369. var field = MonoHttpUtility.UrlDecode(key);
  370. var extension = new SuggestionQueryIndexExtension(
  371. indexImplementation,
  372. documentDatabase.WorkContext,
  373. Path.Combine(configuration.Core.IndexStoragePath, "Raven-Suggestions", indexName, key),
  374. searcher.IndexReader.Directory() is RAMDirectory,
  375. field);
  376. indexImplementation.SetExtension(key, extension);
  377. }
  378. }
  379. }
  380. catch (Exception e)
  381. {
  382. log.WarnException("Could not open suggestions for index " + indexName + ", resetting the index", e);
  383. try
  384. {
  385. IOExtensions.DeleteDirectory(suggestionsForIndex);
  386. }
  387. catch (Exception)
  388. {
  389. // ignore the failure
  390. }
  391. throw;
  392. }
  393. }
  394. internal static void HandleLegacySuggestions(string[] directories)
  395. {
  396. var alreadySeenFields = new HashSet<string>();
  397. foreach (var directory in directories)
  398. {
  399. var key = Path.GetFileName(directory);
  400. var parentDir = System.IO.Directory.GetParent(directory).FullName;
  401. if (key.Contains("-"))
  402. {
  403. var tokens = key.Split('-');
  404. var field = tokens[0];
  405. if (alreadySeenFields.Contains(field))
  406. {
  407. log.Info("Removing legacy suggestions: {0}", directory);
  408. IOExtensions.DeleteDirectory(directory);
  409. }
  410. else
  411. {
  412. alreadySeenFields.Add(field);
  413. var newLocation = Path.Combine(parentDir, field);
  414. log.Info("Moving suggestions from: {0} to {1}", directory, newLocation);
  415. System.IO.Directory.Move(directory, newLocation);
  416. }
  417. }
  418. else
  419. {
  420. alreadySeenFields.Add(key);
  421. }
  422. }
  423. }
  424. protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, bool createIfMissing = true, bool forceFullExistingIndexCheck = false)
  425. {
  426. Lucene.Net.Store.Directory directory;
  427. if (configuration.Core.RunInMemory ||
  428. (indexDefinition.IsMapReduce == false && // there is no point in creating map/reduce indexes in memory, we write the intermediate results to disk anyway
  429. indexDefinitionStorage.IsNewThisSession(indexDefinition) &&
  430. indexDefinition.DisableInMemoryIndexing == false &&
  431. configuration.Indexing.DisableInMemoryIndexing == false &&
  432. forceFullExistingIndexCheck == false))
  433. {
  434. directory = new RAMDirectory();
  435. new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose(); // creating index structure
  436. }
  437. else
  438. {
  439. var indexDirectory = indexDefinition.IndexId.ToString();
  440. var indexFullPath = Path.Combine(path, indexDirectory);
  441. directory = new LuceneCodecDirectory(indexFullPath, documentDatabase.IndexCodecs.OfType<AbstractIndexCodec>());
  442. if (!IndexReader.IndexExists(directory))
  443. {
  444. if (createIfMissing == false)
  445. throw new InvalidOperationException(string.Format("Index directory '{0}' does not exists for '{1}' index.", indexFullPath, indexDefinition.Name));
  446. WriteIndexVersion(directory, indexDefinition);
  447. //creating index structure if we need to
  448. new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
  449. }
  450. else
  451. {
  452. EnsureIndexVersionMatches(directory, indexDefinition);
  453. if (forceFullExistingIndexCheck == false)
  454. {
  455. if (directory.FileExists("write.lock")) // force lock release, because it was still open when we shut down
  456. {
  457. IndexWriter.Unlock(directory);
  458. // for some reason, just calling unlock doesn't remove this file
  459. directory.DeleteFile("write.lock");
  460. }
  461. if (directory.FileExists("writing-to-index.lock")) // we had an unclean shutdown
  462. {
  463. if (configuration.Indexing.ResetIndexOnUncleanShutdown)
  464. throw new InvalidOperationException(string.Format("Rude shutdown detected on '{0}' index in '{1}' directory.", indexDefinition.Name, indexFullPath));
  465. CheckIndexAndTryToFix(directory, indexDefinition);
  466. directory.DeleteFile("writing-to-index.lock");
  467. }
  468. }
  469. else
  470. {
  471. IndexWriter.Unlock(directory);
  472. if (directory.FileExists("write.lock"))
  473. directory.DeleteFile("write.lock");
  474. CheckIndexAndTryToFix(directory, indexDefinition);
  475. if (directory.FileExists("writing-to-index.lock"))
  476. directory.DeleteFile("writing-to-index.lock");
  477. }
  478. }
  479. }
  480. return directory;
  481. }
  482. private void RegenerateMapReduceIndex(Directory directory, IndexDefinition indexDefinition)
  483. {
  484. // remove old index data
  485. var dirOnDisk = Path.Combine(path, indexDefinition.IndexId.ToString());
  486. IOExtensions.DeleteDirectory(dirOnDisk);
  487. // initialize by new index
  488. System.IO.Directory.CreateDirectory(dirOnDisk);
  489. WriteIndexVersion(directory, indexDefinition);
  490. new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Dispose();
  491. var start = 0;
  492. const int take = 100;
  493. documentDatabase.TransactionalStorage.Batch(actions =>
  494. {
  495. IList<ReduceTypePerKey> reduceKeysAndTypes;
  496. do
  497. {
  498. reduceKeysAndTypes = actions.MapReduce.GetReduceKeysAndTypes(indexDefinition.IndexId, start, take).ToList();
  499. start += take;
  500. var keysToScheduleOnLevel2 =
  501. reduceKeysAndTypes.Where(x => x.OperationTypeToPerform == ReduceType.MultiStep).ToList();
  502. var keysToScheduleOnLevel0 =
  503. reduceKeysAndTypes.Where(x => x.OperationTypeToPerform == ReduceType.SingleStep).ToList();
  504. var itemsToScheduleOnLevel2 = keysToScheduleOnLevel2.Select(x => new ReduceKeyAndBucket(0, x.ReduceKey)).ToList();
  505. var itemsToScheduleOnLevel0 = new List<ReduceKeyAndBucket>();
  506. foreach (var reduceKey in keysToScheduleOnLevel0.Select(x => x.ReduceKey))
  507. {
  508. var mappedBuckets = actions.MapReduce.GetMappedBuckets(indexDefinition.IndexId, reduceKey, CancellationToken.None).Distinct();
  509. itemsToScheduleOnLevel0.AddRange(mappedBuckets.Select(x => new ReduceKeyAndBucket(x, reduceKey)));
  510. }
  511. foreach (var itemToReduce in itemsToScheduleOnLevel2)
  512. {
  513. actions.MapReduce.ScheduleReductions(indexDefinition.IndexId, 2, itemToReduce);
  514. actions.General.MaybePulseTransaction();
  515. }
  516. foreach (var itemToReduce in itemsToScheduleOnLevel0)
  517. {
  518. actions.MapReduce.ScheduleReductions(indexDefinition.IndexId, 0, itemToReduce);
  519. actions.General.MaybePulseTransaction();
  520. }
  521. } while (reduceKeysAndTypes.Count > 0);
  522. });
  523. }
  524. private void ResetLastIndexedEtag(IndexDefinition indexDefinition, Etag lastIndexedEtag, DateTime timestamp)
  525. {
  526. documentDatabase.TransactionalStorage.Batch(
  527. accessor =>
  528. accessor.Indexing.UpdateLastIndexed(indexDefinition.IndexId, lastIndexedEtag, timestamp));
  529. }
  530. internal Etag GetLastEtagForIndex(Index index)
  531. {
  532. if (index.IsMapReduce)
  533. return null;
  534. IndexStats stats = null;
  535. documentDatabase.TransactionalStorage.Batch(accessor => stats = accessor.Indexing.GetIndexStats(index.IndexId));
  536. return stats != null ? stats.LastIndexedEtag : Etag.Empty;
  537. }
  538. public static string IndexVersionFileName(IndexDefinition indexDefinition)
  539. {
  540. if (indexDefinition.IsMapReduce)
  541. return "mapReduce.version";
  542. return "index.version";
  543. }
  544. public static void WriteIndexVersion(Directory directory, IndexDefinition indexDefinition)
  545. {
  546. var version = IndexVersion;
  547. if (indexDefinition.IsMapReduce)
  548. {
  549. version = MapReduceIndexVersion;
  550. }
  551. using (var indexOutput = directory.CreateOutput(IndexVersionFileName(indexDefinition)))
  552. {
  553. indexOutput.WriteString(version);
  554. indexOutput.Flush();
  555. }
  556. }
  557. private static void EnsureIndexVersionMatches(Directory directory, IndexDefinition indexDefinition)
  558. {
  559. var versionToCheck = IndexVersion;
  560. if (indexDefinition.IsMapReduce)
  561. {
  562. versionToCheck = MapReduceIndexVersion;
  563. }
  564. var indexVersion = IndexVersionFileName(indexDefinition);
  565. if (directory.FileExists(indexVersion) == false)
  566. {
  567. throw new InvalidOperationException("Could not find " + indexVersion + " " + indexDefinition.IndexId + ", resetting index");
  568. }
  569. using (var indexInput = directory.OpenInput(indexVersion))
  570. {
  571. var versionFromDisk = indexInput.ReadString();
  572. if (versionFromDisk != versionToCheck)
  573. throw new InvalidOperationException("Index " + indexDefinition.IndexId + " is of version " + versionFromDisk +
  574. " which is not compatible with " + versionToCheck + ", resetting index");
  575. }
  576. }
  577. private static void CheckIndexAndTryToFix(Directory directory, IndexDefinition indexDefinition)
  578. {
  579. startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDefinition.Name);
  580. var memoryStream = new MemoryStream();
  581. var stringWriter = new StreamWriter(memoryStream);
  582. var checkIndex = new CheckIndex(directory);
  583. if (startupLog.IsWarnEnabled)
  584. checkIndex.SetInfoStream(stringWriter);
  585. var sp = Stopwatch.StartNew();
  586. var status = checkIndex.CheckIndex_Renamed_Method();
  587. sp.Stop();
  588. if (startupLog.IsWarnEnabled)
  589. {
  590. startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDefinition.Name, sp.Elapsed, status.clean);
  591. memoryStream.Position = 0;
  592. log.Warn(new StreamReader(memoryStream).ReadToEnd());
  593. }
  594. if (status.clean)
  595. return;
  596. startupLog.Warn("Attempting to fix index: {0}", indexDefinition.Name);
  597. sp.Restart();
  598. checkIndex.FixIndex(status);
  599. startupLog.Warn("Fixed index {0} in {1}", indexDefinition.Name, sp.Elapsed);
  600. }
  601. public void StoreCommitPoint(string indexName, IndexCommitPoint indexCommit)
  602. {
  603. if (indexCommit.SegmentsInfo == null || indexCommit.SegmentsInfo.IsIndexCorrupted)
  604. return;
  605. var directoryName = indexCommit.SegmentsInfo.Generation.ToString("0000000000000000000", CultureInfo.InvariantCulture);
  606. var commitPointDirectory = new IndexCommitPointDirectory(path, indexName, directoryName);
  607. if (System.IO.Directory.Exists(commitPointDirectory.AllCommitPointsFullPath) == false)
  608. {
  609. System.IO.Directory.CreateDirectory(commitPointDirectory.AllCommitPointsFullPath);
  610. }
  611. System.IO.Directory.CreateDirectory(commitPointDirectory.FullPath);
  612. using (var commitPointFile = File.Create(commitPointDirectory.FileFullPath))
  613. using (var sw = new StreamWriter(commitPointFile))
  614. {
  615. var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
  616. var textWriter = new JsonTextWriter(sw);
  617. jsonSerializer.Serialize(textWriter, indexCommit);
  618. sw.Flush();
  619. }
  620. var currentSegmentsFileName = indexCommit.SegmentsInfo.SegmentsFileName;
  621. File.Copy(Path.Combine(commitPointDirectory.IndexFullPath, currentSegmentsFileName),
  622. Path.Combine(commitPointDirectory.FullPath, currentSegmentsFileName),
  623. overwrite: true);
  624. var storedCommitPoints = System.IO.Directory.GetDirectories(commitPointDirectory.AllCommitPointsFullPath);
  625. if (storedCommitPoints.Length > configuration.Indexing.MaxNumberOfStoredCommitPoints)
  626. {
  627. foreach (var toDelete in storedCommitPoints.Take(storedCommitPoints.Length - configuration.Indexing.MaxNumberOfStoredCommitPoints))
  628. {
  629. IOExtensions.DeleteDirectory(toDelete);
  630. }
  631. }
  632. }
  633. public void AddDeletedKeysToCommitPoints(IndexDefinition indexDefinition, string[] deletedKeys)
  634. {
  635. var indexFullPath = Path.Combine(path, indexDefinition.IndexId.ToString());
  636. var existingCommitPoints = IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath);
  637. foreach (var commitPointDirectory in existingCommitPoints.Select(commitPoint => new IndexCommitPointDirectory(path, indexDefinition.IndexId.ToString(), commitPoint)))
  638. {
  639. using (var stream = File.Open(commitPointDirectory.DeletedKeysFile, FileMode.OpenOrCreate))
  640. {
  641. stream.Seek(0, SeekOrigin.End);
  642. using (var writer = new StreamWriter(stream))
  643. {
  644. foreach (var deletedKey in deletedKeys)
  645. {
  646. writer.WriteLine(deletedKey);
  647. }
  648. }
  649. }
  650. }
  651. }
  652. private bool TryReusePreviousCommitPointsToRecoverIndex(Directory directory, IndexDefinition indexDefinition, string indexStoragePath, out IndexCommitPoint indexCommit, out string[] keysToDelete)
  653. {
  654. indexCommit = null;
  655. keysToDelete = null;
  656. if (indexDefinition.IsMapReduce)
  657. return false;
  658. var indexFullPath = Path.Combine(indexStoragePath, indexDefinition.IndexId.ToString());
  659. var allCommitPointsFullPath = IndexCommitPointDirectory.GetAllCommitPointsFullPath(indexFullPath);
  660. if (System.IO.Directory.Exists(allCommitPointsFullPath) == false)
  661. return false;
  662. var filesInIndexDirectory = System.IO.Directory.GetFiles(indexFullPath).Select(Path.GetFileName);
  663. var existingCommitPoints =
  664. IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath);
  665. Array.Reverse(existingCommitPoints); // start from the highest generation
  666. foreach (var commitPointDirectoryName in existingCommitPoints)
  667. {
  668. try
  669. {
  670. var commitPointDirectory = new IndexCommitPointDirectory(indexStoragePath, indexDefinition.IndexId.ToString(),
  671. commitPointDirectoryName);
  672. if (TryGetCommitPoint(commitPointDirectory, out indexCommit) == false)
  673. {
  674. IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
  675. continue; // checksum is invalid, try another commit point
  676. }
  677. var missingFile =
  678. indexCommit.SegmentsInfo.ReferencedFiles.Any(
  679. referencedFile => filesInIndexDirectory.Contains(referencedFile) == false);
  680. if (missingFile)
  681. {
  682. IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
  683. continue; // there are some missing files, try another commit point
  684. }
  685. var storedSegmentsFile = indexCommit.SegmentsInfo.SegmentsFileName;
  686. // here there should be only one segments_N file, however remove all if there is more
  687. foreach (var currentSegmentsFile in System.IO.Directory.GetFiles(commitPointDirectory.IndexFullPath, "segments_*"))
  688. {
  689. File.Delete(currentSegmentsFile);
  690. }
  691. // copy old segments_N file
  692. File.Copy(Path.Combine(commitPointDirectory.FullPath, storedSegmentsFile),
  693. Path.Combine(commitPointDirectory.IndexFullPath, storedSegmentsFile), true);
  694. try
  695. {
  696. // update segments.gen file
  697. using (var genOutput = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN))
  698. {
  699. genOutput.WriteInt(SegmentInfos.FORMAT_LOCKLESS);
  700. genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
  701. genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
  702. }
  703. }
  704. catch (Exception)
  705. {
  706. // here we can ignore, segments.gen is used only as fallback
  707. }
  708. if (File.Exists(commitPointDirectory.DeletedKeysFile))
  709. keysToDelete = File.ReadLines(commitPointDirectory.DeletedKeysFile).ToArray();
  710. return true;
  711. }
  712. catch (Exception ex)
  713. {
  714. startupLog.WarnException("Could not recover an index named '" + indexDefinition.IndexId +
  715. "'from segments of the following generation " + commitPointDirectoryName, ex);
  716. }
  717. }
  718. return false;
  719. }
  720. public static IndexSegmentsInfo GetCurrentSegmentsInfo(string indexName, Directory directory)
  721. {
  722. var segmentInfos = new SegmentInfos();
  723. var result = new IndexSegmentsInfo();
  724. try
  725. {
  726. segmentInfos.Read(directory);
  727. result.Generation = segmentInfos.Generation;
  728. result.SegmentsFileName = segmentInfos.GetCurrentSegmentFileName();
  729. result.ReferencedFiles = segmentInfos.Files(directory, false);
  730. }
  731. catch (CorruptIndexException ex)
  732. {
  733. log.WarnException(string.Format("Could not read segment information for an index '{0}'", indexName), ex);
  734. result.IsIndexCorrupted = true;
  735. }
  736. return result;
  737. }
  738. public static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit)
  739. {
  740. using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath))
  741. {
  742. try
  743. {
  744. var textReader = new JsonTextReader(new StreamReader(commitPointFile));
  745. var jsonCommitPoint = RavenJObject.Load(textReader);
  746. var jsonEtag = jsonCommitPoint.Value<RavenJToken>("HighestCommitedETag");
  747. Etag recoveredEtag = null;
  748. if (jsonEtag.Type == JTokenType.Object) // backward compatibility - HighestCommitedETag is written as {"Restarts":123,"Changes":1}
  749. {
  750. jsonCommitPoint.Remove("HighestCommitedETag");
  751. recoveredEtag = new Etag(UuidType.Documents, jsonEtag.Value<long>("Restarts"), jsonEtag.Value<long>("Changes"));
  752. }
  753. indexCommit = jsonCommitPoint.JsonDeserialization<IndexCommitPoint>();
  754. if (indexCommit == null)
  755. return false;
  756. if (recoveredEtag != null)
  757. indexCommit.HighestCommitedETag = recoveredEtag;
  758. if (indexCommit.HighestCommitedETag == null || indexCommit.HighestCommitedETag.CompareTo(Etag.Empty) == 0)
  759. return false;
  760. return true;
  761. }
  762. catch (Exception e)
  763. {
  764. log.Warn("Could not get commit point from the following location {0}. Exception {1}", commitPointDirectory.FileFullPath, e);
  765. indexCommit = null;
  766. return false;
  767. }
  768. }
  769. }
  770. internal Directory MakeRAMDirectoryPhysical(RAMDirectory ramDir, IndexDefinition indexDefinition)
  771. {
  772. var newDir = new LuceneCodecDirectory(Path.Combine(path, indexDefinition.IndexId.ToString()), documentDatabase.IndexCodecs.OfType<AbstractIndexCodec>());
  773. Directory.Copy(ramDir, newDir, false);
  774. return newDir;
  775. }
  776. private Index CreateIndexImplementation(IndexDefinition indexDefinition, Directory directory)
  777. {
  778. var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexDefinition.IndexId);
  779. var indexImplementation = indexDefinition.IsMapReduce
  780. ? (Index)new MapReduceIndex(directory, indexDefinition.IndexId, indexDefinition, viewGenerator, documentDatabase.WorkContext)
  781. : new SimpleIndex(directory, indexDefinition.IndexId, indexDefinition, viewGenerator, documentDatabase.WorkContext);
  782. configuration.Container.SatisfyImportsOnce(indexImplementation);
  783. return indexImplementation;
  784. }
  785. public int[] Indexes
  786. {
  787. get { return indexes.Keys.ToArray(); }
  788. }
  789. public string[] IndexNames
  790. {
  791. get { return indexes.Values.Select(x => x.PublicName).ToArray(); }
  792. }
  793. public bool HasIndex(string index)
  794. {
  795. if (index == null)
  796. return false;
  797. return indexes.Any(x => String.Compare(index, x.Value.PublicName, StringComparison.OrdinalIgnoreCase) == 0);
  798. }
  799. public void Dispose()
  800. {
  801. var exceptionAggregator = new ExceptionAggregator(log, "Could not properly close index storage");
  802. exceptionAggregator.Execute(FlushMapIndexes);
  803. exceptionAggregator.Execute(FlushReduceIndexes);
  804. exceptionAggregator.Execute(() => Parallel.ForEach(indexes.Values, index => exceptionAggregator.Execute(index.Dispose)));
  805. exceptionAggregator.Execute(() => dummyAnalyzer.Close());
  806. exceptionAggregator.Execute(() =>
  807. {
  808. if (crashMarker != null)
  809. crashMarker.Dispose();
  810. });
  811. exceptionAggregator.ThrowIfNeeded();
  812. }
  813. public void DeleteIndex(string name)
  814. {
  815. var value = TryIndexByName(name);
  816. if (value == null)
  817. return;
  818. DeleteIndex(value.indexId);
  819. }
  820. public void DeleteIndex(int id)
  821. {
  822. var value = GetIndexInstance(id);
  823. if (value == null)
  824. {
  825. if (log.IsDebugEnabled)
  826. log.Debug("Ignoring delete for non existing index {0}", id);
  827. return;
  828. }
  829. documentDatabase.TransactionalStorage.Batch(accessor =>
  830. accessor.Lists.Remove("Raven/Indexes/QueryTime", value.PublicName));
  831. if (log.IsDebugEnabled)
  832. log.Debug("Deleting index {0}", value.PublicName);
  833. value.Dispose();
  834. Index ignored;
  835. var dirOnDisk = Path.Combine(path, id.ToString());
  836. if (!indexes.TryRemove(id, out ignored) || !System.IO.Directory.Exists(dirOnDisk))
  837. return;
  838. UpdateIndexMappingFile();
  839. }
  840. public void DeleteIndexData(int id)
  841. {
  842. var dirOnDisk = Path.Combine(path, id.ToString(CultureInfo.InvariantCulture));
  843. IOExtensions.DeleteDirectory(dirOnDisk);
  844. }
  845. public Index ReopenCorruptedIndex(Index index)
  846. {
  847. if (index.Priority != IndexingPriority.Error)
  848. throw new InvalidOperationException(string.Format("Index {0} isn't errored", index.PublicName));
  849. index.Dispose();
  850. var reopened = OpenIndex(index.PublicName, onStartup: false, forceFullIndexCheck: true);
  851. if (reopened == null)
  852. throw new InvalidOperationException("Reopened index cannot be null instance. Index name:" + index.PublicName);
  853. return indexes.AddOrUpdate(reopened.IndexId, n => reopened, (s, existigIndex) => reopened);
  854. }
  855. public void CreateIndexImplementation(IndexDefinition indexDefinition)
  856. {
  857. if (log.IsDebugEnabled)
  858. log.Debug("Creating index {0} with id {1}", indexDefinition.IndexId, indexDefinition.Name);
  859. IndexDefinitionStorage.ResolveAnalyzers(indexDefinition);
  860. if (TryIndexByName(indexDefinition.Name) != null)
  861. {
  862. throw new InvalidOperationException("Index " + indexDefinition.Name + " already exists");
  863. }
  864. var addedIndex = indexes.AddOrUpdate(indexDefinition.IndexId, n =>
  865. {
  866. var directory = OpenOrCreateLuceneDirectory(indexDefinition);
  867. return CreateIndexImplementation(indexDefinition, directory);
  868. }, (s, index) => index);
  869. //prevent corrupted index when creating a map-reduce index
  870. //need to do this for every map reduce index, even when indexing is enabled,
  871. if (addedIndex.IsMapReduce)
  872. {
  873. addedIndex.EnsureIndexWriter();
  874. addedIndex.Flush(Etag.Empty);
  875. }
  876. UpdateIndexMappingFile();
  877. }
  878. public Query GetDocumentQuery(string index, IndexQuery query, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
  879. {
  880. var value = TryIndexByName(index);
  881. if (value == null)
  882. {
  883. if (log.IsDebugEnabled)
  884. log.Debug("Query on non existing index {0}", index);
  885. throw new InvalidOperationException("Index '" + index + "' does not exists");
  886. }
  887. var fieldsToFetch = new FieldsToFetch(new string[0], false, null);
  888. return new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetDocumentQuery();
  889. }
  890. private Index TryIndexByName(string name)
  891. {
  892. return indexes.Where(index => String.Compare(index.Value.PublicName, name, StringComparison.OrdinalIgnoreCase) == 0)
  893. .Select(x => x.Value)
  894. .FirstOrDefault();
  895. }
  896. public IEnumerable<IndexQueryResult> Query(string index,
  897. IndexQuery query,
  898. Func<IndexQueryResult, bool> shouldIncludeInResults,
  899. FieldsToFetch fieldsToFetch,
  900. OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers,
  901. CancellationToken token,
  902. Action<double> parseTiming = null
  903. )
  904. {
  905. Index value = TryIndexByName(index);
  906. if (value == null)
  907. {
  908. if (log.IsDebugEnabled)
  909. log.Debug("Query on non existing index '{0}'", index);
  910. throw new InvalidOperationException("Index '" + index + "' does not exists");
  911. }
  912. if ((value.Priority.HasFlag(IndexingPriority.Idle) || value.Priority.HasFlag(IndexingPriority.Abandoned)) &&
  913. value.Priority.HasFlag(IndexingPriority.Forced) == false)
  914. {
  915. documentDatabase.TransactionalStorage.Batch(accessor =>
  916. {
  917. value.Priority = IndexingPriority.Normal;
  918. try
  919. {
  920. accessor.Indexing.SetIndexPriority(value.indexId, IndexingPriority.Normal);
  921. }
  922. catch (Exception e)
  923. {
  924. if (accessor.IsWriteConflict(e) == false)
  925. throw;
  926. // we explciitly ignore write conflicts here, it is okay if we got set twice (two concurrent queries, or setting while indexing).
  927. }
  928. documentDatabase.WorkContext.ShouldNotifyAboutWork(() => "Idle index q…

Large files files are truncated, but you can click here to view the full file