PageRenderTime 56ms CodeModel.GetById 20ms RepoModel.GetById 0ms app.codeStats 1ms

/library/Zend/Search/Lucene/Index/SegmentInfo.php

https://bitbucket.org/fabiancarlos/feature_seguimentos
PHP | 2132 lines | 1207 code | 309 blank | 616 comment | 337 complexity | 41fb01fe1260ce2dcc3aaac88d116a76 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

  1. <?php
  2. /**
  3. * Zend Framework
  4. *
  5. * LICENSE
  6. *
  7. * This source file is subject to the new BSD license that is bundled
  8. * with this package in the file LICENSE.txt.
  9. * It is also available through the world-wide-web at this URL:
  10. * http://framework.zend.com/license/new-bsd
  11. * If you did not receive a copy of the license and are unable to
  12. * obtain it through the world-wide-web, please send an email
  13. * to license@zend.com so we can send you a copy immediately.
  14. *
  15. * @category Zend
  16. * @package Zend_Search_Lucene
  17. * @subpackage Index
  18. * @copyright Copyright (c) 2005-2011 Zend Technologies USA Inc. (http://www.zend.com)
  19. * @license http://framework.zend.com/license/new-bsd New BSD License
  20. * @version $Id: SegmentInfo.php 23775 2011-03-01 17:25:24Z ralph $
  21. */
  22. /** Zend_Search_Lucene_Index_TermsStream_Interface */
  23. require_once 'Zend/Search/Lucene/Index/TermsStream/Interface.php';
  24. /** Zend_Search_Lucene_Search_Similarity */
  25. require_once 'Zend/Search/Lucene/Search/Similarity.php';
  26. /** Zend_Search_Lucene_Index_FieldInfo */
  27. require_once 'Zend/Search/Lucene/Index/FieldInfo.php';
  28. /** Zend_Search_Lucene_Index_Term */
  29. require_once 'Zend/Search/Lucene/Index/Term.php';
  30. /** Zend_Search_Lucene_Index_TermInfo */
  31. require_once 'Zend/Search/Lucene/Index/TermInfo.php';
  32. /**
  33. * @category Zend
  34. * @package Zend_Search_Lucene
  35. * @subpackage Index
  36. * @copyright Copyright (c) 2005-2011 Zend Technologies USA Inc. (http://www.zend.com)
  37. * @license http://framework.zend.com/license/new-bsd New BSD License
  38. */
  39. class Zend_Search_Lucene_Index_SegmentInfo implements Zend_Search_Lucene_Index_TermsStream_Interface
  40. {
  41. /**
  42. * "Full scan vs fetch" boundary.
  43. *
  44. * If filter selectivity is less than this value, then full scan is performed
  45. * (since term entries fetching has some additional overhead).
  46. */
  47. const FULL_SCAN_VS_FETCH_BOUNDARY = 5;
  48. /**
  49. * Number of docs in a segment
  50. *
  51. * @var integer
  52. */
  53. private $_docCount;
  54. /**
  55. * Segment name
  56. *
  57. * @var string
  58. */
  59. private $_name;
  60. /**
  61. * Term Dictionary Index
  62. *
  63. * Array of arrays (Zend_Search_Lucene_Index_Term objects are represented as arrays because
  64. * of performance considerations)
  65. * [0] -> $termValue
  66. * [1] -> $termFieldNum
  67. *
  68. * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos
  69. *
  70. * @var array
  71. */
  72. private $_termDictionary;
  73. /**
  74. * Term Dictionary Index TermInfos
  75. *
  76. * Array of arrays (Zend_Search_Lucene_Index_TermInfo objects are represented as arrays because
  77. * of performance considerations)
  78. * [0] -> $docFreq
  79. * [1] -> $freqPointer
  80. * [2] -> $proxPointer
  81. * [3] -> $skipOffset
  82. * [4] -> $indexPointer
  83. *
  84. * @var array
  85. */
  86. private $_termDictionaryInfos;
  87. /**
  88. * Segment fields. Array of Zend_Search_Lucene_Index_FieldInfo objects for this segment
  89. *
  90. * @var array
  91. */
  92. private $_fields;
  93. /**
  94. * Field positions in a dictionary.
  95. * (Term dictionary contains filelds ordered by names)
  96. *
  97. * @var array
  98. */
  99. private $_fieldsDicPositions;
  100. /**
  101. * Associative array where the key is the file name and the value is data offset
  102. * in a compound segment file (.csf).
  103. *
  104. * @var array
  105. */
  106. private $_segFiles;
  107. /**
  108. * Associative array where the key is the file name and the value is file size (.csf).
  109. *
  110. * @var array
  111. */
  112. private $_segFileSizes;
  113. /**
  114. * Delete file generation number
  115. *
  116. * -2 means autodetect latest delete generation
  117. * -1 means 'there is no delete file'
  118. * 0 means pre-2.1 format delete file
  119. * X specifies used delete file
  120. *
  121. * @var integer
  122. */
  123. private $_delGen;
  124. /**
  125. * Segment has single norms file
  126. *
  127. * If true then one .nrm file is used for all fields
  128. * Otherwise .fN files are used
  129. *
  130. * @var boolean
  131. */
  132. private $_hasSingleNormFile;
  133. /**
  134. * Use compound segment file (*.cfs) to collect all other segment files
  135. * (excluding .del files)
  136. *
  137. * @var boolean
  138. */
  139. private $_isCompound;
  140. /**
  141. * File system adapter.
  142. *
  143. * @var Zend_Search_Lucene_Storage_Directory_Filesystem
  144. */
  145. private $_directory;
  146. /**
  147. * Normalization factors.
  148. * An array fieldName => normVector
  149. * normVector is a binary string.
  150. * Each byte corresponds to an indexed document in a segment and
  151. * encodes normalization factor (float value, encoded by
  152. * Zend_Search_Lucene_Search_Similarity::encodeNorm())
  153. *
  154. * @var array
  155. */
  156. private $_norms = array();
  157. /**
  158. * List of deleted documents.
  159. * bitset if bitset extension is loaded or array otherwise.
  160. *
  161. * @var mixed
  162. */
  163. private $_deleted = null;
  164. /**
  165. * $this->_deleted update flag
  166. *
  167. * @var boolean
  168. */
  169. private $_deletedDirty = false;
  170. /**
  171. * True if segment uses shared doc store
  172. *
  173. * @var boolean
  174. */
  175. private $_usesSharedDocStore;
  176. /*
  177. * Shared doc store options.
  178. * It's an assotiative array with the following items:
  179. * - 'offset' => $docStoreOffset The starting document in the shared doc store files where this segment's documents begin
  180. * - 'segment' => $docStoreSegment The name of the segment that has the shared doc store files.
  181. * - 'isCompound' => $docStoreIsCompoundFile True, if compound file format is used for the shared doc store files (.cfx file).
  182. */
  183. private $_sharedDocStoreOptions;
  184. /**
  185. * Zend_Search_Lucene_Index_SegmentInfo constructor
  186. *
  187. * @param Zend_Search_Lucene_Storage_Directory $directory
  188. * @param string $name
  189. * @param integer $docCount
  190. * @param integer $delGen
  191. * @param array|null $docStoreOptions
  192. * @param boolean $hasSingleNormFile
  193. * @param boolean $isCompound
  194. */
  195. public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name, $docCount, $delGen = 0, $docStoreOptions = null, $hasSingleNormFile = false, $isCompound = null)
  196. {
  197. $this->_directory = $directory;
  198. $this->_name = $name;
  199. $this->_docCount = $docCount;
  200. if ($docStoreOptions !== null) {
  201. $this->_usesSharedDocStore = true;
  202. $this->_sharedDocStoreOptions = $docStoreOptions;
  203. if ($docStoreOptions['isCompound']) {
  204. $cfxFile = $this->_directory->getFileObject($docStoreOptions['segment'] . '.cfx');
  205. $cfxFilesCount = $cfxFile->readVInt();
  206. $cfxFiles = array();
  207. $cfxFileSizes = array();
  208. for ($count = 0; $count < $cfxFilesCount; $count++) {
  209. $dataOffset = $cfxFile->readLong();
  210. if ($count != 0) {
  211. $cfxFileSizes[$fileName] = $dataOffset - end($cfxFiles);
  212. }
  213. $fileName = $cfxFile->readString();
  214. $cfxFiles[$fileName] = $dataOffset;
  215. }
  216. if ($count != 0) {
  217. $cfxFileSizes[$fileName] = $this->_directory->fileLength($docStoreOptions['segment'] . '.cfx') - $dataOffset;
  218. }
  219. $this->_sharedDocStoreOptions['files'] = $cfxFiles;
  220. $this->_sharedDocStoreOptions['fileSizes'] = $cfxFileSizes;
  221. }
  222. }
  223. $this->_hasSingleNormFile = $hasSingleNormFile;
  224. $this->_delGen = $delGen;
  225. $this->_termDictionary = null;
  226. if ($isCompound !== null) {
  227. $this->_isCompound = $isCompound;
  228. } else {
  229. // It's a pre-2.1 segment or isCompound is set to 'unknown'
  230. // Detect if segment uses compound file
  231. require_once 'Zend/Search/Lucene/Exception.php';
  232. try {
  233. // Try to open compound file
  234. $this->_directory->getFileObject($name . '.cfs');
  235. // Compound file is found
  236. $this->_isCompound = true;
  237. } catch (Zend_Search_Lucene_Exception $e) {
  238. if (strpos($e->getMessage(), 'is not readable') !== false) {
  239. // Compound file is not found or is not readable
  240. $this->_isCompound = false;
  241. } else {
  242. throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
  243. }
  244. }
  245. }
  246. $this->_segFiles = array();
  247. if ($this->_isCompound) {
  248. $cfsFile = $this->_directory->getFileObject($name . '.cfs');
  249. $segFilesCount = $cfsFile->readVInt();
  250. for ($count = 0; $count < $segFilesCount; $count++) {
  251. $dataOffset = $cfsFile->readLong();
  252. if ($count != 0) {
  253. $this->_segFileSizes[$fileName] = $dataOffset - end($this->_segFiles);
  254. }
  255. $fileName = $cfsFile->readString();
  256. $this->_segFiles[$fileName] = $dataOffset;
  257. }
  258. if ($count != 0) {
  259. $this->_segFileSizes[$fileName] = $this->_directory->fileLength($name . '.cfs') - $dataOffset;
  260. }
  261. }
  262. $fnmFile = $this->openCompoundFile('.fnm');
  263. $fieldsCount = $fnmFile->readVInt();
  264. $fieldNames = array();
  265. $fieldNums = array();
  266. $this->_fields = array();
  267. for ($count=0; $count < $fieldsCount; $count++) {
  268. $fieldName = $fnmFile->readString();
  269. $fieldBits = $fnmFile->readByte();
  270. $this->_fields[$count] = new Zend_Search_Lucene_Index_FieldInfo($fieldName,
  271. $fieldBits & 0x01 /* field is indexed */,
  272. $count,
  273. $fieldBits & 0x02 /* termvectors are stored */,
  274. $fieldBits & 0x10 /* norms are omitted */,
  275. $fieldBits & 0x20 /* payloads are stored */);
  276. if ($fieldBits & 0x10) {
  277. // norms are omitted for the indexed field
  278. $this->_norms[$count] = str_repeat(chr(Zend_Search_Lucene_Search_Similarity::encodeNorm(1.0)), $docCount);
  279. }
  280. $fieldNums[$count] = $count;
  281. $fieldNames[$count] = $fieldName;
  282. }
  283. array_multisort($fieldNames, SORT_ASC, SORT_REGULAR, $fieldNums);
  284. $this->_fieldsDicPositions = array_flip($fieldNums);
  285. if ($this->_delGen == -2) {
  286. // SegmentInfo constructor is invoked from index writer
  287. // Autodetect current delete file generation number
  288. $this->_delGen = $this->_detectLatestDelGen();
  289. }
  290. // Load deletions
  291. $this->_deleted = $this->_loadDelFile();
  292. }
  293. /**
  294. * Load detetions file
  295. *
  296. * Returns bitset or an array depending on bitset extension availability
  297. *
  298. * @return mixed
  299. * @throws Zend_Search_Lucene_Exception
  300. */
  301. private function _loadDelFile()
  302. {
  303. if ($this->_delGen == -1) {
  304. // There is no delete file for this segment
  305. return null;
  306. } else if ($this->_delGen == 0) {
  307. // It's a segment with pre-2.1 format delete file
  308. // Try to load deletions file
  309. return $this->_loadPre21DelFile();
  310. } else {
  311. // It's 2.1+ format deleteions file
  312. return $this->_load21DelFile();
  313. }
  314. }
  315. /**
  316. * Load pre-2.1 detetions file
  317. *
  318. * Returns bitset or an array depending on bitset extension availability
  319. *
  320. * @return mixed
  321. * @throws Zend_Search_Lucene_Exception
  322. */
  323. private function _loadPre21DelFile()
  324. {
  325. require_once 'Zend/Search/Lucene/Exception.php';
  326. try {
  327. // '.del' files always stored in a separate file
  328. // Segment compound is not used
  329. $delFile = $this->_directory->getFileObject($this->_name . '.del');
  330. $byteCount = $delFile->readInt();
  331. $byteCount = ceil($byteCount/8);
  332. $bitCount = $delFile->readInt();
  333. if ($bitCount == 0) {
  334. $delBytes = '';
  335. } else {
  336. $delBytes = $delFile->readBytes($byteCount);
  337. }
  338. if (extension_loaded('bitset')) {
  339. return $delBytes;
  340. } else {
  341. $deletions = array();
  342. for ($count = 0; $count < $byteCount; $count++) {
  343. $byte = ord($delBytes[$count]);
  344. for ($bit = 0; $bit < 8; $bit++) {
  345. if ($byte & (1<<$bit)) {
  346. $deletions[$count*8 + $bit] = 1;
  347. }
  348. }
  349. }
  350. return $deletions;
  351. }
  352. } catch(Zend_Search_Lucene_Exception $e) {
  353. if (strpos($e->getMessage(), 'is not readable') === false) {
  354. throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
  355. }
  356. // There is no deletion file
  357. $this->_delGen = -1;
  358. return null;
  359. }
  360. }
  361. /**
  362. * Load 2.1+ format detetions file
  363. *
  364. * Returns bitset or an array depending on bitset extension availability
  365. *
  366. * @return mixed
  367. */
  368. private function _load21DelFile()
  369. {
  370. $delFile = $this->_directory->getFileObject($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');
  371. $format = $delFile->readInt();
  372. if ($format == (int)0xFFFFFFFF) {
  373. if (extension_loaded('bitset')) {
  374. $deletions = bitset_empty();
  375. } else {
  376. $deletions = array();
  377. }
  378. $byteCount = $delFile->readInt();
  379. $bitCount = $delFile->readInt();
  380. $delFileSize = $this->_directory->fileLength($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');
  381. $byteNum = 0;
  382. do {
  383. $dgap = $delFile->readVInt();
  384. $nonZeroByte = $delFile->readByte();
  385. $byteNum += $dgap;
  386. if (extension_loaded('bitset')) {
  387. for ($bit = 0; $bit < 8; $bit++) {
  388. if ($nonZeroByte & (1<<$bit)) {
  389. bitset_incl($deletions, $byteNum*8 + $bit);
  390. }
  391. }
  392. return $deletions;
  393. } else {
  394. for ($bit = 0; $bit < 8; $bit++) {
  395. if ($nonZeroByte & (1<<$bit)) {
  396. $deletions[$byteNum*8 + $bit] = 1;
  397. }
  398. }
  399. return (count($deletions) > 0) ? $deletions : null;
  400. }
  401. } while ($delFile->tell() < $delFileSize);
  402. } else {
  403. // $format is actually byte count
  404. $byteCount = ceil($format/8);
  405. $bitCount = $delFile->readInt();
  406. if ($bitCount == 0) {
  407. $delBytes = '';
  408. } else {
  409. $delBytes = $delFile->readBytes($byteCount);
  410. }
  411. if (extension_loaded('bitset')) {
  412. return $delBytes;
  413. } else {
  414. $deletions = array();
  415. for ($count = 0; $count < $byteCount; $count++) {
  416. $byte = ord($delBytes[$count]);
  417. for ($bit = 0; $bit < 8; $bit++) {
  418. if ($byte & (1<<$bit)) {
  419. $deletions[$count*8 + $bit] = 1;
  420. }
  421. }
  422. }
  423. return (count($deletions) > 0) ? $deletions : null;
  424. }
  425. }
  426. }
  427. /**
  428. * Opens index file stoted within compound index file
  429. *
  430. * @param string $extension
  431. * @param boolean $shareHandler
  432. * @throws Zend_Search_Lucene_Exception
  433. * @return Zend_Search_Lucene_Storage_File
  434. */
  435. public function openCompoundFile($extension, $shareHandler = true)
  436. {
  437. if (($extension == '.fdx' || $extension == '.fdt') && $this->_usesSharedDocStore) {
  438. $fdxFName = $this->_sharedDocStoreOptions['segment'] . '.fdx';
  439. $fdtFName = $this->_sharedDocStoreOptions['segment'] . '.fdt';
  440. if (!$this->_sharedDocStoreOptions['isCompound']) {
  441. $fdxFile = $this->_directory->getFileObject($fdxFName, $shareHandler);
  442. $fdxFile->seek($this->_sharedDocStoreOptions['offset']*8, SEEK_CUR);
  443. if ($extension == '.fdx') {
  444. // '.fdx' file is requested
  445. return $fdxFile;
  446. } else {
  447. // '.fdt' file is requested
  448. $fdtStartOffset = $fdxFile->readLong();
  449. $fdtFile = $this->_directory->getFileObject($fdtFName, $shareHandler);
  450. $fdtFile->seek($fdtStartOffset, SEEK_CUR);
  451. return $fdtFile;
  452. }
  453. }
  454. if( !isset($this->_sharedDocStoreOptions['files'][$fdxFName]) ) {
  455. require_once 'Zend/Search/Lucene/Exception.php';
  456. throw new Zend_Search_Lucene_Exception('Shared doc storage segment compound file doesn\'t contain '
  457. . $fdxFName . ' file.' );
  458. }
  459. if( !isset($this->_sharedDocStoreOptions['files'][$fdtFName]) ) {
  460. require_once 'Zend/Search/Lucene/Exception.php';
  461. throw new Zend_Search_Lucene_Exception('Shared doc storage segment compound file doesn\'t contain '
  462. . $fdtFName . ' file.' );
  463. }
  464. // Open shared docstore segment file
  465. $cfxFile = $this->_directory->getFileObject($this->_sharedDocStoreOptions['segment'] . '.cfx', $shareHandler);
  466. // Seek to the start of '.fdx' file within compound file
  467. $cfxFile->seek($this->_sharedDocStoreOptions['files'][$fdxFName]);
  468. // Seek to the start of current segment documents section
  469. $cfxFile->seek($this->_sharedDocStoreOptions['offset']*8, SEEK_CUR);
  470. if ($extension == '.fdx') {
  471. // '.fdx' file is requested
  472. return $cfxFile;
  473. } else {
  474. // '.fdt' file is requested
  475. $fdtStartOffset = $cfxFile->readLong();
  476. // Seek to the start of '.fdt' file within compound file
  477. $cfxFile->seek($this->_sharedDocStoreOptions['files'][$fdtFName]);
  478. // Seek to the start of current segment documents section
  479. $cfxFile->seek($fdtStartOffset, SEEK_CUR);
  480. return $fdtFile;
  481. }
  482. }
  483. $filename = $this->_name . $extension;
  484. if (!$this->_isCompound) {
  485. return $this->_directory->getFileObject($filename, $shareHandler);
  486. }
  487. if( !isset($this->_segFiles[$filename]) ) {
  488. require_once 'Zend/Search/Lucene/Exception.php';
  489. throw new Zend_Search_Lucene_Exception('Segment compound file doesn\'t contain '
  490. . $filename . ' file.' );
  491. }
  492. $file = $this->_directory->getFileObject($this->_name . '.cfs', $shareHandler);
  493. $file->seek($this->_segFiles[$filename]);
  494. return $file;
  495. }
  496. /**
  497. * Get compound file length
  498. *
  499. * @param string $extension
  500. * @return integer
  501. */
  502. public function compoundFileLength($extension)
  503. {
  504. if (($extension == '.fdx' || $extension == '.fdt') && $this->_usesSharedDocStore) {
  505. $filename = $this->_sharedDocStoreOptions['segment'] . $extension;
  506. if (!$this->_sharedDocStoreOptions['isCompound']) {
  507. return $this->_directory->fileLength($filename);
  508. }
  509. if( !isset($this->_sharedDocStoreOptions['fileSizes'][$filename]) ) {
  510. require_once 'Zend/Search/Lucene/Exception.php';
  511. throw new Zend_Search_Lucene_Exception('Shared doc store compound file doesn\'t contain '
  512. . $filename . ' file.' );
  513. }
  514. return $this->_sharedDocStoreOptions['fileSizes'][$filename];
  515. }
  516. $filename = $this->_name . $extension;
  517. // Try to get common file first
  518. if ($this->_directory->fileExists($filename)) {
  519. return $this->_directory->fileLength($filename);
  520. }
  521. if( !isset($this->_segFileSizes[$filename]) ) {
  522. require_once 'Zend/Search/Lucene/Exception.php';
  523. throw new Zend_Search_Lucene_Exception('Index compound file doesn\'t contain '
  524. . $filename . ' file.' );
  525. }
  526. return $this->_segFileSizes[$filename];
  527. }
  528. /**
  529. * Returns field index or -1 if field is not found
  530. *
  531. * @param string $fieldName
  532. * @return integer
  533. */
  534. public function getFieldNum($fieldName)
  535. {
  536. foreach( $this->_fields as $field ) {
  537. if( $field->name == $fieldName ) {
  538. return $field->number;
  539. }
  540. }
  541. return -1;
  542. }
  543. /**
  544. * Returns field info for specified field
  545. *
  546. * @param integer $fieldNum
  547. * @return Zend_Search_Lucene_Index_FieldInfo
  548. */
  549. public function getField($fieldNum)
  550. {
  551. return $this->_fields[$fieldNum];
  552. }
  553. /**
  554. * Returns array of fields.
  555. * if $indexed parameter is true, then returns only indexed fields.
  556. *
  557. * @param boolean $indexed
  558. * @return array
  559. */
  560. public function getFields($indexed = false)
  561. {
  562. $result = array();
  563. foreach( $this->_fields as $field ) {
  564. if( (!$indexed) || $field->isIndexed ) {
  565. $result[ $field->name ] = $field->name;
  566. }
  567. }
  568. return $result;
  569. }
  570. /**
  571. * Returns array of FieldInfo objects.
  572. *
  573. * @return array
  574. */
  575. public function getFieldInfos()
  576. {
  577. return $this->_fields;
  578. }
  579. /**
  580. * Returns actual deletions file generation number.
  581. *
  582. * @return integer
  583. */
  584. public function getDelGen()
  585. {
  586. return $this->_delGen;
  587. }
  588. /**
  589. * Returns the total number of documents in this segment (including deleted documents).
  590. *
  591. * @return integer
  592. */
  593. public function count()
  594. {
  595. return $this->_docCount;
  596. }
  597. /**
  598. * Returns number of deleted documents.
  599. *
  600. * @return integer
  601. */
  602. private function _deletedCount()
  603. {
  604. if ($this->_deleted === null) {
  605. return 0;
  606. }
  607. if (extension_loaded('bitset')) {
  608. return count(bitset_to_array($this->_deleted));
  609. } else {
  610. return count($this->_deleted);
  611. }
  612. }
  613. /**
  614. * Returns the total number of non-deleted documents in this segment.
  615. *
  616. * @return integer
  617. */
  618. public function numDocs()
  619. {
  620. if ($this->hasDeletions()) {
  621. return $this->_docCount - $this->_deletedCount();
  622. } else {
  623. return $this->_docCount;
  624. }
  625. }
  626. /**
  627. * Get field position in a fields dictionary
  628. *
  629. * @param integer $fieldNum
  630. * @return integer
  631. */
  632. private function _getFieldPosition($fieldNum) {
  633. // Treat values which are not in a translation table as a 'direct value'
  634. return isset($this->_fieldsDicPositions[$fieldNum]) ?
  635. $this->_fieldsDicPositions[$fieldNum] : $fieldNum;
  636. }
  637. /**
  638. * Return segment name
  639. *
  640. * @return string
  641. */
  642. public function getName()
  643. {
  644. return $this->_name;
  645. }
  646. /**
  647. * TermInfo cache
  648. *
  649. * Size is 1024.
  650. * Numbers are used instead of class constants because of performance considerations
  651. *
  652. * @var array
  653. */
  654. private $_termInfoCache = array();
  655. private function _cleanUpTermInfoCache()
  656. {
  657. // Clean 256 term infos
  658. foreach ($this->_termInfoCache as $key => $termInfo) {
  659. unset($this->_termInfoCache[$key]);
  660. // leave 768 last used term infos
  661. if (count($this->_termInfoCache) == 768) {
  662. break;
  663. }
  664. }
  665. }
  666. /**
  667. * Load terms dictionary index
  668. *
  669. * @throws Zend_Search_Lucene_Exception
  670. */
  671. private function _loadDictionaryIndex()
  672. {
  673. // Check, if index is already serialized
  674. if ($this->_directory->fileExists($this->_name . '.sti')) {
  675. // Load serialized dictionary index data
  676. $stiFile = $this->_directory->getFileObject($this->_name . '.sti');
  677. $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti'));
  678. // Load dictionary index data
  679. if (($unserializedData = @unserialize($stiFileData)) !== false) {
  680. list($this->_termDictionary, $this->_termDictionaryInfos) = $unserializedData;
  681. return;
  682. }
  683. }
  684. // Load data from .tii file and generate .sti file
  685. // Prefetch dictionary index data
  686. $tiiFile = $this->openCompoundFile('.tii');
  687. $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii'));
  688. /** Zend_Search_Lucene_Index_DictionaryLoader */
  689. require_once 'Zend/Search/Lucene/Index/DictionaryLoader.php';
  690. // Load dictionary index data
  691. list($this->_termDictionary, $this->_termDictionaryInfos) =
  692. Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData);
  693. $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos));
  694. $stiFile = $this->_directory->createFile($this->_name . '.sti');
  695. $stiFile->writeBytes($stiFileData);
  696. }
  697. /**
  698. * Scans terms dictionary and returns term info
  699. *
  700. * @param Zend_Search_Lucene_Index_Term $term
  701. * @return Zend_Search_Lucene_Index_TermInfo
  702. */
  703. public function getTermInfo(Zend_Search_Lucene_Index_Term $term)
  704. {
  705. $termKey = $term->key();
  706. if (isset($this->_termInfoCache[$termKey])) {
  707. $termInfo = $this->_termInfoCache[$termKey];
  708. // Move termInfo to the end of cache
  709. unset($this->_termInfoCache[$termKey]);
  710. $this->_termInfoCache[$termKey] = $termInfo;
  711. return $termInfo;
  712. }
  713. if ($this->_termDictionary === null) {
  714. $this->_loadDictionaryIndex();
  715. }
  716. $searchField = $this->getFieldNum($term->field);
  717. if ($searchField == -1) {
  718. return null;
  719. }
  720. $searchDicField = $this->_getFieldPosition($searchField);
  721. // search for appropriate value in dictionary
  722. $lowIndex = 0;
  723. $highIndex = count($this->_termDictionary)-1;
  724. while ($highIndex >= $lowIndex) {
  725. // $mid = ($highIndex - $lowIndex)/2;
  726. $mid = ($highIndex + $lowIndex) >> 1;
  727. $midTerm = $this->_termDictionary[$mid];
  728. $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */);
  729. $delta = $searchDicField - $fieldNum;
  730. if ($delta == 0) {
  731. $delta = strcmp($term->text, $midTerm[1] /* text */);
  732. }
  733. if ($delta < 0) {
  734. $highIndex = $mid-1;
  735. } elseif ($delta > 0) {
  736. $lowIndex = $mid+1;
  737. } else {
  738. // return $this->_termDictionaryInfos[$mid]; // We got it!
  739. $a = $this->_termDictionaryInfos[$mid];
  740. $termInfo = new Zend_Search_Lucene_Index_TermInfo($a[0], $a[1], $a[2], $a[3], $a[4]);
  741. // Put loaded termInfo into cache
  742. $this->_termInfoCache[$termKey] = $termInfo;
  743. return $termInfo;
  744. }
  745. }
  746. if ($highIndex == -1) {
  747. // Term is out of the dictionary range
  748. return null;
  749. }
  750. $prevPosition = $highIndex;
  751. $prevTerm = $this->_termDictionary[$prevPosition];
  752. $prevTermInfo = $this->_termDictionaryInfos[$prevPosition];
  753. $tisFile = $this->openCompoundFile('.tis');
  754. $tiVersion = $tisFile->readInt();
  755. if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */ &&
  756. $tiVersion != (int)0xFFFFFFFD /* 2.1+ format */) {
  757. require_once 'Zend/Search/Lucene/Exception.php';
  758. throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format');
  759. }
  760. $termCount = $tisFile->readLong();
  761. $indexInterval = $tisFile->readInt();
  762. $skipInterval = $tisFile->readInt();
  763. if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) {
  764. $maxSkipLevels = $tisFile->readInt();
  765. }
  766. $tisFile->seek($prevTermInfo[4] /* indexPointer */ - (($tiVersion == (int)0xFFFFFFFD)? 24 : 20) /* header size*/, SEEK_CUR);
  767. $termValue = $prevTerm[1] /* text */;
  768. $termFieldNum = $prevTerm[0] /* field */;
  769. $freqPointer = $prevTermInfo[1] /* freqPointer */;
  770. $proxPointer = $prevTermInfo[2] /* proxPointer */;
  771. for ($count = $prevPosition*$indexInterval + 1;
  772. $count <= $termCount &&
  773. ( $this->_getFieldPosition($termFieldNum) < $searchDicField ||
  774. ($this->_getFieldPosition($termFieldNum) == $searchDicField &&
  775. strcmp($termValue, $term->text) < 0) );
  776. $count++) {
  777. $termPrefixLength = $tisFile->readVInt();
  778. $termSuffix = $tisFile->readString();
  779. $termFieldNum = $tisFile->readVInt();
  780. $termValue = Zend_Search_Lucene_Index_Term::getPrefix($termValue, $termPrefixLength) . $termSuffix;
  781. $docFreq = $tisFile->readVInt();
  782. $freqPointer += $tisFile->readVInt();
  783. $proxPointer += $tisFile->readVInt();
  784. if( $docFreq >= $skipInterval ) {
  785. $skipOffset = $tisFile->readVInt();
  786. } else {
  787. $skipOffset = 0;
  788. }
  789. }
  790. if ($termFieldNum == $searchField && $termValue == $term->text) {
  791. $termInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);
  792. } else {
  793. $termInfo = null;
  794. }
  795. // Put loaded termInfo into cache
  796. $this->_termInfoCache[$termKey] = $termInfo;
  797. if (count($this->_termInfoCache) == 1024) {
  798. $this->_cleanUpTermInfoCache();
  799. }
  800. return $termInfo;
  801. }
  802. /**
  803. * Returns IDs of all the documents containing term.
  804. *
  805. * @param Zend_Search_Lucene_Index_Term $term
  806. * @param integer $shift
  807. * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
  808. * @return array
  809. */
  810. public function termDocs(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
  811. {
  812. $termInfo = $this->getTermInfo($term);
  813. if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
  814. if ($docsFilter !== null && $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
  815. $docsFilter->segmentFilters[$this->_name] = array();
  816. }
  817. return array();
  818. }
  819. $frqFile = $this->openCompoundFile('.frq');
  820. $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
  821. $docId = 0;
  822. $result = array();
  823. if ($docsFilter !== null) {
  824. if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
  825. require_once 'Zend/Search/Lucene/Exception.php';
  826. throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
  827. }
  828. if (isset($docsFilter->segmentFilters[$this->_name])) {
  829. // Filter already has some data for the current segment
  830. // Make short name for the filter (which doesn't need additional dereferencing)
  831. $filter = &$docsFilter->segmentFilters[$this->_name];
  832. // Check if filter is not empty
  833. if (count($filter) == 0) {
  834. return array();
  835. }
  836. if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
  837. // Perform fetching
  838. // ---------------------------------------------------------------
  839. $updatedFilterData = array();
  840. for( $count=0; $count < $termInfo->docFreq; $count++ ) {
  841. $docDelta = $frqFile->readVInt();
  842. if( $docDelta % 2 == 1 ) {
  843. $docId += ($docDelta-1)/2;
  844. } else {
  845. $docId += $docDelta/2;
  846. // read freq
  847. $frqFile->readVInt();
  848. }
  849. if (isset($filter[$docId])) {
  850. $result[] = $shift + $docId;
  851. $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  852. }
  853. }
  854. $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
  855. // ---------------------------------------------------------------
  856. } else {
  857. // Perform full scan
  858. $updatedFilterData = array();
  859. for( $count=0; $count < $termInfo->docFreq; $count++ ) {
  860. $docDelta = $frqFile->readVInt();
  861. if( $docDelta % 2 == 1 ) {
  862. $docId += ($docDelta-1)/2;
  863. } else {
  864. $docId += $docDelta/2;
  865. // read freq
  866. $frqFile->readVInt();
  867. }
  868. if (isset($filter[$docId])) {
  869. $result[] = $shift + $docId;
  870. $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  871. }
  872. }
  873. $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
  874. }
  875. } else {
  876. // Filter is present, but doesn't has data for the current segment yet
  877. $filterData = array();
  878. for( $count=0; $count < $termInfo->docFreq; $count++ ) {
  879. $docDelta = $frqFile->readVInt();
  880. if( $docDelta % 2 == 1 ) {
  881. $docId += ($docDelta-1)/2;
  882. } else {
  883. $docId += $docDelta/2;
  884. // read freq
  885. $frqFile->readVInt();
  886. }
  887. $result[] = $shift + $docId;
  888. $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  889. }
  890. $docsFilter->segmentFilters[$this->_name] = $filterData;
  891. }
  892. } else {
  893. for( $count=0; $count < $termInfo->docFreq; $count++ ) {
  894. $docDelta = $frqFile->readVInt();
  895. if( $docDelta % 2 == 1 ) {
  896. $docId += ($docDelta-1)/2;
  897. } else {
  898. $docId += $docDelta/2;
  899. // read freq
  900. $frqFile->readVInt();
  901. }
  902. $result[] = $shift + $docId;
  903. }
  904. }
  905. return $result;
  906. }
  907. /**
  908. * Returns term freqs array.
  909. * Result array structure: array(docId => freq, ...)
  910. *
  911. * @param Zend_Search_Lucene_Index_Term $term
  912. * @param integer $shift
  913. * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
  914. * @return Zend_Search_Lucene_Index_TermInfo
  915. */
  916. public function termFreqs(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
  917. {
  918. $termInfo = $this->getTermInfo($term);
  919. if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
  920. if ($docsFilter !== null && $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
  921. $docsFilter->segmentFilters[$this->_name] = array();
  922. }
  923. return array();
  924. }
  925. $frqFile = $this->openCompoundFile('.frq');
  926. $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
  927. $result = array();
  928. $docId = 0;
  929. $result = array();
  930. if ($docsFilter !== null) {
  931. if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
  932. require_once 'Zend/Search/Lucene/Exception.php';
  933. throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
  934. }
  935. if (isset($docsFilter->segmentFilters[$this->_name])) {
  936. // Filter already has some data for the current segment
  937. // Make short name for the filter (which doesn't need additional dereferencing)
  938. $filter = &$docsFilter->segmentFilters[$this->_name];
  939. // Check if filter is not empty
  940. if (count($filter) == 0) {
  941. return array();
  942. }
  943. if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
  944. // Perform fetching
  945. // ---------------------------------------------------------------
  946. $updatedFilterData = array();
  947. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  948. $docDelta = $frqFile->readVInt();
  949. if ($docDelta % 2 == 1) {
  950. $docId += ($docDelta-1)/2;
  951. if (isset($filter[$docId])) {
  952. $result[$shift + $docId] = 1;
  953. $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  954. }
  955. } else {
  956. $docId += $docDelta/2;
  957. $freq = $frqFile->readVInt();
  958. if (isset($filter[$docId])) {
  959. $result[$shift + $docId] = $freq;
  960. $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  961. }
  962. }
  963. }
  964. $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
  965. // ---------------------------------------------------------------
  966. } else {
  967. // Perform full scan
  968. $updatedFilterData = array();
  969. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  970. $docDelta = $frqFile->readVInt();
  971. if ($docDelta % 2 == 1) {
  972. $docId += ($docDelta-1)/2;
  973. if (isset($filter[$docId])) {
  974. $result[$shift + $docId] = 1;
  975. $updatedFilterData[$docId] = 1; // 1 is just some constant value, so we don't need additional var dereference here
  976. }
  977. } else {
  978. $docId += $docDelta/2;
  979. $freq = $frqFile->readVInt();
  980. if (isset($filter[$docId])) {
  981. $result[$shift + $docId] = $freq;
  982. $updatedFilterData[$docId] = 1; // 1 is just some constant value, so we don't need additional var dereference here
  983. }
  984. }
  985. }
  986. $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
  987. }
  988. } else {
  989. // Filter doesn't has data for current segment
  990. $filterData = array();
  991. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  992. $docDelta = $frqFile->readVInt();
  993. if ($docDelta % 2 == 1) {
  994. $docId += ($docDelta-1)/2;
  995. $result[$shift + $docId] = 1;
  996. $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  997. } else {
  998. $docId += $docDelta/2;
  999. $result[$shift + $docId] = $frqFile->readVInt();
  1000. $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  1001. }
  1002. }
  1003. $docsFilter->segmentFilters[$this->_name] = $filterData;
  1004. }
  1005. } else {
  1006. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  1007. $docDelta = $frqFile->readVInt();
  1008. if ($docDelta % 2 == 1) {
  1009. $docId += ($docDelta-1)/2;
  1010. $result[$shift + $docId] = 1;
  1011. } else {
  1012. $docId += $docDelta/2;
  1013. $result[$shift + $docId] = $frqFile->readVInt();
  1014. }
  1015. }
  1016. }
  1017. return $result;
  1018. }
  1019. /**
  1020. * Returns term positions array.
  1021. * Result array structure: array(docId => array(pos1, pos2, ...), ...)
  1022. *
  1023. * @param Zend_Search_Lucene_Index_Term $term
  1024. * @param integer $shift
  1025. * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
  1026. * @return Zend_Search_Lucene_Index_TermInfo
  1027. */
  1028. public function termPositions(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
  1029. {
  1030. $termInfo = $this->getTermInfo($term);
  1031. if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
  1032. if ($docsFilter !== null && $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
  1033. $docsFilter->segmentFilters[$this->_name] = array();
  1034. }
  1035. return array();
  1036. }
  1037. $frqFile = $this->openCompoundFile('.frq');
  1038. $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
  1039. $docId = 0;
  1040. $freqs = array();
  1041. if ($docsFilter !== null) {
  1042. if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
  1043. require_once 'Zend/Search/Lucene/Exception.php';
  1044. throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
  1045. }
  1046. if (isset($docsFilter->segmentFilters[$this->_name])) {
  1047. // Filter already has some data for the current segment
  1048. // Make short name for the filter (which doesn't need additional dereferencing)
  1049. $filter = &$docsFilter->segmentFilters[$this->_name];
  1050. // Check if filter is not empty
  1051. if (count($filter) == 0) {
  1052. return array();
  1053. }
  1054. if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
  1055. // Perform fetching
  1056. // ---------------------------------------------------------------
  1057. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  1058. $docDelta = $frqFile->readVInt();
  1059. if ($docDelta % 2 == 1) {
  1060. $docId += ($docDelta-1)/2;
  1061. $freqs[$docId] = 1;
  1062. } else {
  1063. $docId += $docDelta/2;
  1064. $freqs[$docId] = $frqFile->readVInt();
  1065. }
  1066. }
  1067. $updatedFilterData = array();
  1068. $result = array();
  1069. $prxFile = $this->openCompoundFile('.prx');
  1070. $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
  1071. foreach ($freqs as $docId => $freq) {
  1072. $termPosition = 0;
  1073. $positions = array();
  1074. // we have to read .prx file to get right position for next doc
  1075. // even filter doesn't match current document
  1076. for ($count = 0; $count < $freq; $count++ ) {
  1077. $termPosition += $prxFile->readVInt();
  1078. $positions[] = $termPosition;
  1079. }
  1080. // Include into updated filter and into result only if doc is matched by filter
  1081. if (isset($filter[$docId])) {
  1082. $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  1083. $result[$shift + $docId] = $positions;
  1084. }
  1085. }
  1086. $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
  1087. // ---------------------------------------------------------------
  1088. } else {
  1089. // Perform full scan
  1090. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  1091. $docDelta = $frqFile->readVInt();
  1092. if ($docDelta % 2 == 1) {
  1093. $docId += ($docDelta-1)/2;
  1094. $freqs[$docId] = 1;
  1095. } else {
  1096. $docId += $docDelta/2;
  1097. $freqs[$docId] = $frqFile->readVInt();
  1098. }
  1099. }
  1100. $updatedFilterData = array();
  1101. $result = array();
  1102. $prxFile = $this->openCompoundFile('.prx');
  1103. $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
  1104. foreach ($freqs as $docId => $freq) {
  1105. $termPosition = 0;
  1106. $positions = array();
  1107. // we have to read .prx file to get right position for next doc
  1108. // even filter doesn't match current document
  1109. for ($count = 0; $count < $freq; $count++ ) {
  1110. $termPosition += $prxFile->readVInt();
  1111. $positions[] = $termPosition;
  1112. }
  1113. // Include into updated filter and into result only if doc is matched by filter
  1114. if (isset($filter[$docId])) {
  1115. $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  1116. $result[$shift + $docId] = $positions;
  1117. }
  1118. }
  1119. $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
  1120. }
  1121. } else {
  1122. // Filter doesn't has data for current segment
  1123. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  1124. $docDelta = $frqFile->readVInt();
  1125. if ($docDelta % 2 == 1) {
  1126. $docId += ($docDelta-1)/2;
  1127. $freqs[$docId] = 1;
  1128. } else {
  1129. $docId += $docDelta/2;
  1130. $freqs[$docId] = $frqFile->readVInt();
  1131. }
  1132. }
  1133. $filterData = array();
  1134. $result = array();
  1135. $prxFile = $this->openCompoundFile('.prx');
  1136. $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
  1137. foreach ($freqs as $docId => $freq) {
  1138. $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
  1139. $termPosition = 0;
  1140. $positions = array();
  1141. for ($count = 0; $count < $freq; $count++ ) {
  1142. $termPosition += $prxFile->readVInt();
  1143. $positions[] = $termPosition;
  1144. }
  1145. $result[$shift + $docId] = $positions;
  1146. }
  1147. $docsFilter->segmentFilters[$this->_name] = $filterData;
  1148. }
  1149. } else {
  1150. for ($count = 0; $count < $termInfo->docFreq; $count++) {
  1151. $docDelta = $frqFile->readVInt();
  1152. if ($docDelta % 2 == 1) {
  1153. $docId += ($docDelta-1)/2;
  1154. $freqs[$docId] = 1;
  1155. } else {
  1156. $docId += $docDelta/2;
  1157. $freqs[$docId] = $frqFile->readVInt();
  1158. }
  1159. }
  1160. $result = array();
  1161. $prxFile = $this->openCompoundFile('.prx');
  1162. $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
  1163. foreach ($freqs as $docId => $freq) {
  1164. $termPosition = 0;
  1165. $positions = array();
  1166. for ($count = 0; $count < $freq; $count++ ) {
  1167. $termPosition += $prxFile->readVInt();
  1168. $positions[] = $termPosition;
  1169. }
  1170. $result[$shift + $docId] = $positions;
  1171. }
  1172. }
  1173. return $result;
  1174. }
  1175. /**
  1176. * Load normalizatin factors from an index file
  1177. *
  1178. * @param integer $fieldNum
  1179. * @throws Zend_Search_Lucene_Exception

Large files files are truncated, but you can click here to view the full file