/lib/vendor/Zend/Search/Lucene/Index/SegmentInfo.php
PHP | 2132 lines | 1207 code | 309 blank | 616 comment | 337 complexity | b76c6224ed609046a4ab15bc49309d89 MD5 | raw file
Possible License(s): BSD-3-Clause
Large files files are truncated, but you can click here to view the full file
- <?php
- /**
- * Zend Framework
- *
- * LICENSE
- *
- * This source file is subject to the new BSD license that is bundled
- * with this package in the file LICENSE.txt.
- * It is also available through the world-wide-web at this URL:
- * http://framework.zend.com/license/new-bsd
- * If you did not receive a copy of the license and are unable to
- * obtain it through the world-wide-web, please send an email
- * to license@zend.com so we can send you a copy immediately.
- *
- * @category Zend
- * @package Zend_Search_Lucene
- * @subpackage Index
- * @copyright Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
- * @license http://framework.zend.com/license/new-bsd New BSD License
- * @version $Id: SegmentInfo.php 22987 2010-09-21 10:39:53Z alexander $
- */
- /** Zend_Search_Lucene_Index_TermsStream_Interface */
- require_once 'Zend/Search/Lucene/Index/TermsStream/Interface.php';
- /** Zend_Search_Lucene_Search_Similarity */
- require_once 'Zend/Search/Lucene/Search/Similarity.php';
- /** Zend_Search_Lucene_Index_FieldInfo */
- require_once 'Zend/Search/Lucene/Index/FieldInfo.php';
- /** Zend_Search_Lucene_Index_Term */
- require_once 'Zend/Search/Lucene/Index/Term.php';
- /** Zend_Search_Lucene_Index_TermInfo */
- require_once 'Zend/Search/Lucene/Index/TermInfo.php';
- /**
- * @category Zend
- * @package Zend_Search_Lucene
- * @subpackage Index
- * @copyright Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
- * @license http://framework.zend.com/license/new-bsd New BSD License
- */
- class Zend_Search_Lucene_Index_SegmentInfo implements Zend_Search_Lucene_Index_TermsStream_Interface
- {
- /**
- * "Full scan vs fetch" boundary.
- *
- * If filter selectivity is less than this value, then full scan is performed
- * (since term entries fetching has some additional overhead).
- */
- const FULL_SCAN_VS_FETCH_BOUNDARY = 5;
- /**
- * Number of docs in a segment
- *
- * @var integer
- */
- private $_docCount;
- /**
- * Segment name
- *
- * @var string
- */
- private $_name;
- /**
- * Term Dictionary Index
- *
- * Array of arrays (Zend_Search_Lucene_Index_Term objects are represented as arrays because
- * of performance considerations)
- * [0] -> $termValue
- * [1] -> $termFieldNum
- *
- * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos
- *
- * @var array
- */
- private $_termDictionary;
- /**
- * Term Dictionary Index TermInfos
- *
- * Array of arrays (Zend_Search_Lucene_Index_TermInfo objects are represented as arrays because
- * of performance considerations)
- * [0] -> $docFreq
- * [1] -> $freqPointer
- * [2] -> $proxPointer
- * [3] -> $skipOffset
- * [4] -> $indexPointer
- *
- * @var array
- */
- private $_termDictionaryInfos;
- /**
- * Segment fields. Array of Zend_Search_Lucene_Index_FieldInfo objects for this segment
- *
- * @var array
- */
- private $_fields;
- /**
- * Field positions in a dictionary.
- * (Term dictionary contains filelds ordered by names)
- *
- * @var array
- */
- private $_fieldsDicPositions;
- /**
- * Associative array where the key is the file name and the value is data offset
- * in a compound segment file (.csf).
- *
- * @var array
- */
- private $_segFiles;
- /**
- * Associative array where the key is the file name and the value is file size (.csf).
- *
- * @var array
- */
- private $_segFileSizes;
- /**
- * Delete file generation number
- *
- * -2 means autodetect latest delete generation
- * -1 means 'there is no delete file'
- * 0 means pre-2.1 format delete file
- * X specifies used delete file
- *
- * @var integer
- */
- private $_delGen;
- /**
- * Segment has single norms file
- *
- * If true then one .nrm file is used for all fields
- * Otherwise .fN files are used
- *
- * @var boolean
- */
- private $_hasSingleNormFile;
- /**
- * Use compound segment file (*.cfs) to collect all other segment files
- * (excluding .del files)
- *
- * @var boolean
- */
- private $_isCompound;
- /**
- * File system adapter.
- *
- * @var Zend_Search_Lucene_Storage_Directory_Filesystem
- */
- private $_directory;
- /**
- * Normalization factors.
- * An array fieldName => normVector
- * normVector is a binary string.
- * Each byte corresponds to an indexed document in a segment and
- * encodes normalization factor (float value, encoded by
- * Zend_Search_Lucene_Search_Similarity::encodeNorm())
- *
- * @var array
- */
- private $_norms = array();
- /**
- * List of deleted documents.
- * bitset if bitset extension is loaded or array otherwise.
- *
- * @var mixed
- */
- private $_deleted = null;
- /**
- * $this->_deleted update flag
- *
- * @var boolean
- */
- private $_deletedDirty = false;
- /**
- * True if segment uses shared doc store
- *
- * @var boolean
- */
- private $_usesSharedDocStore;
- /*
- * Shared doc store options.
- * It's an assotiative array with the following items:
- * - 'offset' => $docStoreOffset The starting document in the shared doc store files where this segment's documents begin
- * - 'segment' => $docStoreSegment The name of the segment that has the shared doc store files.
- * - 'isCompound' => $docStoreIsCompoundFile True, if compound file format is used for the shared doc store files (.cfx file).
- */
- private $_sharedDocStoreOptions;
- /**
- * Zend_Search_Lucene_Index_SegmentInfo constructor
- *
- * @param Zend_Search_Lucene_Storage_Directory $directory
- * @param string $name
- * @param integer $docCount
- * @param integer $delGen
- * @param array|null $docStoreOptions
- * @param boolean $hasSingleNormFile
- * @param boolean $isCompound
- */
- public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name, $docCount, $delGen = 0, $docStoreOptions = null, $hasSingleNormFile = false, $isCompound = null)
- {
- $this->_directory = $directory;
- $this->_name = $name;
- $this->_docCount = $docCount;
- if ($docStoreOptions !== null) {
- $this->_usesSharedDocStore = true;
- $this->_sharedDocStoreOptions = $docStoreOptions;
- if ($docStoreOptions['isCompound']) {
- $cfxFile = $this->_directory->getFileObject($docStoreOptions['segment'] . '.cfx');
- $cfxFilesCount = $cfxFile->readVInt();
- $cfxFiles = array();
- $cfxFileSizes = array();
- for ($count = 0; $count < $cfxFilesCount; $count++) {
- $dataOffset = $cfxFile->readLong();
- if ($count != 0) {
- $cfxFileSizes[$fileName] = $dataOffset - end($cfxFiles);
- }
- $fileName = $cfxFile->readString();
- $cfxFiles[$fileName] = $dataOffset;
- }
- if ($count != 0) {
- $cfxFileSizes[$fileName] = $this->_directory->fileLength($docStoreOptions['segment'] . '.cfx') - $dataOffset;
- }
- $this->_sharedDocStoreOptions['files'] = $cfxFiles;
- $this->_sharedDocStoreOptions['fileSizes'] = $cfxFileSizes;
- }
- }
- $this->_hasSingleNormFile = $hasSingleNormFile;
- $this->_delGen = $delGen;
- $this->_termDictionary = null;
- if ($isCompound !== null) {
- $this->_isCompound = $isCompound;
- } else {
- // It's a pre-2.1 segment or isCompound is set to 'unknown'
- // Detect if segment uses compound file
- require_once 'Zend/Search/Lucene/Exception.php';
- try {
- // Try to open compound file
- $this->_directory->getFileObject($name . '.cfs');
- // Compound file is found
- $this->_isCompound = true;
- } catch (Zend_Search_Lucene_Exception $e) {
- if (strpos($e->getMessage(), 'is not readable') !== false) {
- // Compound file is not found or is not readable
- $this->_isCompound = false;
- } else {
- throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
- }
- }
- }
- $this->_segFiles = array();
- if ($this->_isCompound) {
- $cfsFile = $this->_directory->getFileObject($name . '.cfs');
- $segFilesCount = $cfsFile->readVInt();
- for ($count = 0; $count < $segFilesCount; $count++) {
- $dataOffset = $cfsFile->readLong();
- if ($count != 0) {
- $this->_segFileSizes[$fileName] = $dataOffset - end($this->_segFiles);
- }
- $fileName = $cfsFile->readString();
- $this->_segFiles[$fileName] = $dataOffset;
- }
- if ($count != 0) {
- $this->_segFileSizes[$fileName] = $this->_directory->fileLength($name . '.cfs') - $dataOffset;
- }
- }
- $fnmFile = $this->openCompoundFile('.fnm');
- $fieldsCount = $fnmFile->readVInt();
- $fieldNames = array();
- $fieldNums = array();
- $this->_fields = array();
- for ($count=0; $count < $fieldsCount; $count++) {
- $fieldName = $fnmFile->readString();
- $fieldBits = $fnmFile->readByte();
- $this->_fields[$count] = new Zend_Search_Lucene_Index_FieldInfo($fieldName,
- $fieldBits & 0x01 /* field is indexed */,
- $count,
- $fieldBits & 0x02 /* termvectors are stored */,
- $fieldBits & 0x10 /* norms are omitted */,
- $fieldBits & 0x20 /* payloads are stored */);
- if ($fieldBits & 0x10) {
- // norms are omitted for the indexed field
- $this->_norms[$count] = str_repeat(chr(Zend_Search_Lucene_Search_Similarity::encodeNorm(1.0)), $docCount);
- }
- $fieldNums[$count] = $count;
- $fieldNames[$count] = $fieldName;
- }
- array_multisort($fieldNames, SORT_ASC, SORT_REGULAR, $fieldNums);
- $this->_fieldsDicPositions = array_flip($fieldNums);
- if ($this->_delGen == -2) {
- // SegmentInfo constructor is invoked from index writer
- // Autodetect current delete file generation number
- $this->_delGen = $this->_detectLatestDelGen();
- }
- // Load deletions
- $this->_deleted = $this->_loadDelFile();
- }
- /**
- * Load detetions file
- *
- * Returns bitset or an array depending on bitset extension availability
- *
- * @return mixed
- * @throws Zend_Search_Lucene_Exception
- */
- private function _loadDelFile()
- {
- if ($this->_delGen == -1) {
- // There is no delete file for this segment
- return null;
- } else if ($this->_delGen == 0) {
- // It's a segment with pre-2.1 format delete file
- // Try to load deletions file
- return $this->_loadPre21DelFile();
- } else {
- // It's 2.1+ format deleteions file
- return $this->_load21DelFile();
- }
- }
- /**
- * Load pre-2.1 detetions file
- *
- * Returns bitset or an array depending on bitset extension availability
- *
- * @return mixed
- * @throws Zend_Search_Lucene_Exception
- */
- private function _loadPre21DelFile()
- {
- require_once 'Zend/Search/Lucene/Exception.php';
- try {
- // '.del' files always stored in a separate file
- // Segment compound is not used
- $delFile = $this->_directory->getFileObject($this->_name . '.del');
- $byteCount = $delFile->readInt();
- $byteCount = ceil($byteCount/8);
- $bitCount = $delFile->readInt();
- if ($bitCount == 0) {
- $delBytes = '';
- } else {
- $delBytes = $delFile->readBytes($byteCount);
- }
- if (extension_loaded('bitset')) {
- return $delBytes;
- } else {
- $deletions = array();
- for ($count = 0; $count < $byteCount; $count++) {
- $byte = ord($delBytes[$count]);
- for ($bit = 0; $bit < 8; $bit++) {
- if ($byte & (1<<$bit)) {
- $deletions[$count*8 + $bit] = 1;
- }
- }
- }
- return $deletions;
- }
- } catch(Zend_Search_Lucene_Exception $e) {
- if (strpos($e->getMessage(), 'is not readable') === false) {
- throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
- }
- // There is no deletion file
- $this->_delGen = -1;
- return null;
- }
- }
- /**
- * Load 2.1+ format detetions file
- *
- * Returns bitset or an array depending on bitset extension availability
- *
- * @return mixed
- */
- private function _load21DelFile()
- {
- $delFile = $this->_directory->getFileObject($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');
- $format = $delFile->readInt();
- if ($format == (int)0xFFFFFFFF) {
- if (extension_loaded('bitset')) {
- $deletions = bitset_empty();
- } else {
- $deletions = array();
- }
- $byteCount = $delFile->readInt();
- $bitCount = $delFile->readInt();
- $delFileSize = $this->_directory->fileLength($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');
- $byteNum = 0;
- do {
- $dgap = $delFile->readVInt();
- $nonZeroByte = $delFile->readByte();
- $byteNum += $dgap;
- if (extension_loaded('bitset')) {
- for ($bit = 0; $bit < 8; $bit++) {
- if ($nonZeroByte & (1<<$bit)) {
- bitset_incl($deletions, $byteNum*8 + $bit);
- }
- }
- return $deletions;
- } else {
- for ($bit = 0; $bit < 8; $bit++) {
- if ($nonZeroByte & (1<<$bit)) {
- $deletions[$byteNum*8 + $bit] = 1;
- }
- }
- return (count($deletions) > 0) ? $deletions : null;
- }
- } while ($delFile->tell() < $delFileSize);
- } else {
- // $format is actually byte count
- $byteCount = ceil($format/8);
- $bitCount = $delFile->readInt();
- if ($bitCount == 0) {
- $delBytes = '';
- } else {
- $delBytes = $delFile->readBytes($byteCount);
- }
- if (extension_loaded('bitset')) {
- return $delBytes;
- } else {
- $deletions = array();
- for ($count = 0; $count < $byteCount; $count++) {
- $byte = ord($delBytes[$count]);
- for ($bit = 0; $bit < 8; $bit++) {
- if ($byte & (1<<$bit)) {
- $deletions[$count*8 + $bit] = 1;
- }
- }
- }
- return (count($deletions) > 0) ? $deletions : null;
- }
- }
- }
- /**
- * Opens index file stoted within compound index file
- *
- * @param string $extension
- * @param boolean $shareHandler
- * @throws Zend_Search_Lucene_Exception
- * @return Zend_Search_Lucene_Storage_File
- */
- public function openCompoundFile($extension, $shareHandler = true)
- {
- if (($extension == '.fdx' || $extension == '.fdt') && $this->_usesSharedDocStore) {
- $fdxFName = $this->_sharedDocStoreOptions['segment'] . '.fdx';
- $fdtFName = $this->_sharedDocStoreOptions['segment'] . '.fdt';
- if (!$this->_sharedDocStoreOptions['isCompound']) {
- $fdxFile = $this->_directory->getFileObject($fdxFName, $shareHandler);
- $fdxFile->seek($this->_sharedDocStoreOptions['offset']*8, SEEK_CUR);
- if ($extension == '.fdx') {
- // '.fdx' file is requested
- return $fdxFile;
- } else {
- // '.fdt' file is requested
- $fdtStartOffset = $fdxFile->readLong();
- $fdtFile = $this->_directory->getFileObject($fdtFName, $shareHandler);
- $fdtFile->seek($fdtStartOffset, SEEK_CUR);
- return $fdtFile;
- }
- }
- if( !isset($this->_sharedDocStoreOptions['files'][$fdxFName]) ) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Shared doc storage segment compound file doesn\'t contain '
- . $fdxFName . ' file.' );
- }
- if( !isset($this->_sharedDocStoreOptions['files'][$fdtFName]) ) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Shared doc storage segment compound file doesn\'t contain '
- . $fdtFName . ' file.' );
- }
- // Open shared docstore segment file
- $cfxFile = $this->_directory->getFileObject($this->_sharedDocStoreOptions['segment'] . '.cfx', $shareHandler);
- // Seek to the start of '.fdx' file within compound file
- $cfxFile->seek($this->_sharedDocStoreOptions['files'][$fdxFName]);
- // Seek to the start of current segment documents section
- $cfxFile->seek($this->_sharedDocStoreOptions['offset']*8, SEEK_CUR);
- if ($extension == '.fdx') {
- // '.fdx' file is requested
- return $cfxFile;
- } else {
- // '.fdt' file is requested
- $fdtStartOffset = $cfxFile->readLong();
- // Seek to the start of '.fdt' file within compound file
- $cfxFile->seek($this->_sharedDocStoreOptions['files'][$fdtFName]);
- // Seek to the start of current segment documents section
- $cfxFile->seek($fdtStartOffset, SEEK_CUR);
- return $fdtFile;
- }
- }
- $filename = $this->_name . $extension;
- if (!$this->_isCompound) {
- return $this->_directory->getFileObject($filename, $shareHandler);
- }
- if( !isset($this->_segFiles[$filename]) ) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Segment compound file doesn\'t contain '
- . $filename . ' file.' );
- }
- $file = $this->_directory->getFileObject($this->_name . '.cfs', $shareHandler);
- $file->seek($this->_segFiles[$filename]);
- return $file;
- }
- /**
- * Get compound file length
- *
- * @param string $extension
- * @return integer
- */
- public function compoundFileLength($extension)
- {
- if (($extension == '.fdx' || $extension == '.fdt') && $this->_usesSharedDocStore) {
- $filename = $this->_sharedDocStoreOptions['segment'] . $extension;
- if (!$this->_sharedDocStoreOptions['isCompound']) {
- return $this->_directory->fileLength($filename);
- }
- if( !isset($this->_sharedDocStoreOptions['fileSizes'][$filename]) ) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Shared doc store compound file doesn\'t contain '
- . $filename . ' file.' );
- }
- return $this->_sharedDocStoreOptions['fileSizes'][$filename];
- }
- $filename = $this->_name . $extension;
- // Try to get common file first
- if ($this->_directory->fileExists($filename)) {
- return $this->_directory->fileLength($filename);
- }
- if( !isset($this->_segFileSizes[$filename]) ) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Index compound file doesn\'t contain '
- . $filename . ' file.' );
- }
- return $this->_segFileSizes[$filename];
- }
- /**
- * Returns field index or -1 if field is not found
- *
- * @param string $fieldName
- * @return integer
- */
- public function getFieldNum($fieldName)
- {
- foreach( $this->_fields as $field ) {
- if( $field->name == $fieldName ) {
- return $field->number;
- }
- }
- return -1;
- }
- /**
- * Returns field info for specified field
- *
- * @param integer $fieldNum
- * @return Zend_Search_Lucene_Index_FieldInfo
- */
- public function getField($fieldNum)
- {
- return $this->_fields[$fieldNum];
- }
- /**
- * Returns array of fields.
- * if $indexed parameter is true, then returns only indexed fields.
- *
- * @param boolean $indexed
- * @return array
- */
- public function getFields($indexed = false)
- {
- $result = array();
- foreach( $this->_fields as $field ) {
- if( (!$indexed) || $field->isIndexed ) {
- $result[ $field->name ] = $field->name;
- }
- }
- return $result;
- }
- /**
- * Returns array of FieldInfo objects.
- *
- * @return array
- */
- public function getFieldInfos()
- {
- return $this->_fields;
- }
- /**
- * Returns actual deletions file generation number.
- *
- * @return integer
- */
- public function getDelGen()
- {
- return $this->_delGen;
- }
- /**
- * Returns the total number of documents in this segment (including deleted documents).
- *
- * @return integer
- */
- public function count()
- {
- return $this->_docCount;
- }
- /**
- * Returns number of deleted documents.
- *
- * @return integer
- */
- private function _deletedCount()
- {
- if ($this->_deleted === null) {
- return 0;
- }
- if (extension_loaded('bitset')) {
- return count(bitset_to_array($this->_deleted));
- } else {
- return count($this->_deleted);
- }
- }
- /**
- * Returns the total number of non-deleted documents in this segment.
- *
- * @return integer
- */
- public function numDocs()
- {
- if ($this->hasDeletions()) {
- return $this->_docCount - $this->_deletedCount();
- } else {
- return $this->_docCount;
- }
- }
- /**
- * Get field position in a fields dictionary
- *
- * @param integer $fieldNum
- * @return integer
- */
- private function _getFieldPosition($fieldNum) {
- // Treat values which are not in a translation table as a 'direct value'
- return isset($this->_fieldsDicPositions[$fieldNum]) ?
- $this->_fieldsDicPositions[$fieldNum] : $fieldNum;
- }
- /**
- * Return segment name
- *
- * @return string
- */
- public function getName()
- {
- return $this->_name;
- }
- /**
- * TermInfo cache
- *
- * Size is 1024.
- * Numbers are used instead of class constants because of performance considerations
- *
- * @var array
- */
- private $_termInfoCache = array();
- private function _cleanUpTermInfoCache()
- {
- // Clean 256 term infos
- foreach ($this->_termInfoCache as $key => $termInfo) {
- unset($this->_termInfoCache[$key]);
- // leave 768 last used term infos
- if (count($this->_termInfoCache) == 768) {
- break;
- }
- }
- }
- /**
- * Load terms dictionary index
- *
- * @throws Zend_Search_Lucene_Exception
- */
- private function _loadDictionaryIndex()
- {
- // Check, if index is already serialized
- if ($this->_directory->fileExists($this->_name . '.sti')) {
- // Load serialized dictionary index data
- $stiFile = $this->_directory->getFileObject($this->_name . '.sti');
- $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti'));
- // Load dictionary index data
- if (($unserializedData = @unserialize($stiFileData)) !== false) {
- list($this->_termDictionary, $this->_termDictionaryInfos) = $unserializedData;
- return;
- }
- }
- // Load data from .tii file and generate .sti file
- // Prefetch dictionary index data
- $tiiFile = $this->openCompoundFile('.tii');
- $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii'));
- /** Zend_Search_Lucene_Index_DictionaryLoader */
- require_once 'Zend/Search/Lucene/Index/DictionaryLoader.php';
- // Load dictionary index data
- list($this->_termDictionary, $this->_termDictionaryInfos) =
- Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData);
- $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos));
- $stiFile = $this->_directory->createFile($this->_name . '.sti');
- $stiFile->writeBytes($stiFileData);
- }
- /**
- * Scans terms dictionary and returns term info
- *
- * @param Zend_Search_Lucene_Index_Term $term
- * @return Zend_Search_Lucene_Index_TermInfo
- */
- public function getTermInfo(Zend_Search_Lucene_Index_Term $term)
- {
- $termKey = $term->key();
- if (isset($this->_termInfoCache[$termKey])) {
- $termInfo = $this->_termInfoCache[$termKey];
- // Move termInfo to the end of cache
- unset($this->_termInfoCache[$termKey]);
- $this->_termInfoCache[$termKey] = $termInfo;
- return $termInfo;
- }
- if ($this->_termDictionary === null) {
- $this->_loadDictionaryIndex();
- }
- $searchField = $this->getFieldNum($term->field);
- if ($searchField == -1) {
- return null;
- }
- $searchDicField = $this->_getFieldPosition($searchField);
- // search for appropriate value in dictionary
- $lowIndex = 0;
- $highIndex = count($this->_termDictionary)-1;
- while ($highIndex >= $lowIndex) {
- // $mid = ($highIndex - $lowIndex)/2;
- $mid = ($highIndex + $lowIndex) >> 1;
- $midTerm = $this->_termDictionary[$mid];
- $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */);
- $delta = $searchDicField - $fieldNum;
- if ($delta == 0) {
- $delta = strcmp($term->text, $midTerm[1] /* text */);
- }
- if ($delta < 0) {
- $highIndex = $mid-1;
- } elseif ($delta > 0) {
- $lowIndex = $mid+1;
- } else {
- // return $this->_termDictionaryInfos[$mid]; // We got it!
- $a = $this->_termDictionaryInfos[$mid];
- $termInfo = new Zend_Search_Lucene_Index_TermInfo($a[0], $a[1], $a[2], $a[3], $a[4]);
- // Put loaded termInfo into cache
- $this->_termInfoCache[$termKey] = $termInfo;
- return $termInfo;
- }
- }
- if ($highIndex == -1) {
- // Term is out of the dictionary range
- return null;
- }
- $prevPosition = $highIndex;
- $prevTerm = $this->_termDictionary[$prevPosition];
- $prevTermInfo = $this->_termDictionaryInfos[$prevPosition];
- $tisFile = $this->openCompoundFile('.tis');
- $tiVersion = $tisFile->readInt();
- if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */ &&
- $tiVersion != (int)0xFFFFFFFD /* 2.1+ format */) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format');
- }
- $termCount = $tisFile->readLong();
- $indexInterval = $tisFile->readInt();
- $skipInterval = $tisFile->readInt();
- if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) {
- $maxSkipLevels = $tisFile->readInt();
- }
- $tisFile->seek($prevTermInfo[4] /* indexPointer */ - (($tiVersion == (int)0xFFFFFFFD)? 24 : 20) /* header size*/, SEEK_CUR);
- $termValue = $prevTerm[1] /* text */;
- $termFieldNum = $prevTerm[0] /* field */;
- $freqPointer = $prevTermInfo[1] /* freqPointer */;
- $proxPointer = $prevTermInfo[2] /* proxPointer */;
- for ($count = $prevPosition*$indexInterval + 1;
- $count <= $termCount &&
- ( $this->_getFieldPosition($termFieldNum) < $searchDicField ||
- ($this->_getFieldPosition($termFieldNum) == $searchDicField &&
- strcmp($termValue, $term->text) < 0) );
- $count++) {
- $termPrefixLength = $tisFile->readVInt();
- $termSuffix = $tisFile->readString();
- $termFieldNum = $tisFile->readVInt();
- $termValue = Zend_Search_Lucene_Index_Term::getPrefix($termValue, $termPrefixLength) . $termSuffix;
- $docFreq = $tisFile->readVInt();
- $freqPointer += $tisFile->readVInt();
- $proxPointer += $tisFile->readVInt();
- if( $docFreq >= $skipInterval ) {
- $skipOffset = $tisFile->readVInt();
- } else {
- $skipOffset = 0;
- }
- }
- if ($termFieldNum == $searchField && $termValue == $term->text) {
- $termInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);
- } else {
- $termInfo = null;
- }
- // Put loaded termInfo into cache
- $this->_termInfoCache[$termKey] = $termInfo;
- if (count($this->_termInfoCache) == 1024) {
- $this->_cleanUpTermInfoCache();
- }
- return $termInfo;
- }
- /**
- * Returns IDs of all the documents containing term.
- *
- * @param Zend_Search_Lucene_Index_Term $term
- * @param integer $shift
- * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
- * @return array
- */
- public function termDocs(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
- {
- $termInfo = $this->getTermInfo($term);
- if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
- if ($docsFilter !== null && $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
- $docsFilter->segmentFilters[$this->_name] = array();
- }
- return array();
- }
- $frqFile = $this->openCompoundFile('.frq');
- $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
- $docId = 0;
- $result = array();
- if ($docsFilter !== null) {
- if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
- }
- if (isset($docsFilter->segmentFilters[$this->_name])) {
- // Filter already has some data for the current segment
- // Make short name for the filter (which doesn't need additional dereferencing)
- $filter = &$docsFilter->segmentFilters[$this->_name];
- // Check if filter is not empty
- if (count($filter) == 0) {
- return array();
- }
- if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
- // Perform fetching
- // ---------------------------------------------------------------
- $updatedFilterData = array();
- for( $count=0; $count < $termInfo->docFreq; $count++ ) {
- $docDelta = $frqFile->readVInt();
- if( $docDelta % 2 == 1 ) {
- $docId += ($docDelta-1)/2;
- } else {
- $docId += $docDelta/2;
- // read freq
- $frqFile->readVInt();
- }
- if (isset($filter[$docId])) {
- $result[] = $shift + $docId;
- $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
- // ---------------------------------------------------------------
- } else {
- // Perform full scan
- $updatedFilterData = array();
- for( $count=0; $count < $termInfo->docFreq; $count++ ) {
- $docDelta = $frqFile->readVInt();
- if( $docDelta % 2 == 1 ) {
- $docId += ($docDelta-1)/2;
- } else {
- $docId += $docDelta/2;
- // read freq
- $frqFile->readVInt();
- }
- if (isset($filter[$docId])) {
- $result[] = $shift + $docId;
- $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
- }
- } else {
- // Filter is present, but doesn't has data for the current segment yet
- $filterData = array();
- for( $count=0; $count < $termInfo->docFreq; $count++ ) {
- $docDelta = $frqFile->readVInt();
- if( $docDelta % 2 == 1 ) {
- $docId += ($docDelta-1)/2;
- } else {
- $docId += $docDelta/2;
- // read freq
- $frqFile->readVInt();
- }
- $result[] = $shift + $docId;
- $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- }
- $docsFilter->segmentFilters[$this->_name] = $filterData;
- }
- } else {
- for( $count=0; $count < $termInfo->docFreq; $count++ ) {
- $docDelta = $frqFile->readVInt();
- if( $docDelta % 2 == 1 ) {
- $docId += ($docDelta-1)/2;
- } else {
- $docId += $docDelta/2;
- // read freq
- $frqFile->readVInt();
- }
- $result[] = $shift + $docId;
- }
- }
- return $result;
- }
- /**
- * Returns term freqs array.
- * Result array structure: array(docId => freq, ...)
- *
- * @param Zend_Search_Lucene_Index_Term $term
- * @param integer $shift
- * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
- * @return Zend_Search_Lucene_Index_TermInfo
- */
- public function termFreqs(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
- {
- $termInfo = $this->getTermInfo($term);
- if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
- if ($docsFilter !== null && $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
- $docsFilter->segmentFilters[$this->_name] = array();
- }
- return array();
- }
- $frqFile = $this->openCompoundFile('.frq');
- $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
- $result = array();
- $docId = 0;
- $result = array();
- if ($docsFilter !== null) {
- if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
- }
- if (isset($docsFilter->segmentFilters[$this->_name])) {
- // Filter already has some data for the current segment
- // Make short name for the filter (which doesn't need additional dereferencing)
- $filter = &$docsFilter->segmentFilters[$this->_name];
- // Check if filter is not empty
- if (count($filter) == 0) {
- return array();
- }
- if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
- // Perform fetching
- // ---------------------------------------------------------------
- $updatedFilterData = array();
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- if (isset($filter[$docId])) {
- $result[$shift + $docId] = 1;
- $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- }
- } else {
- $docId += $docDelta/2;
- $freq = $frqFile->readVInt();
- if (isset($filter[$docId])) {
- $result[$shift + $docId] = $freq;
- $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- }
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
- // ---------------------------------------------------------------
- } else {
- // Perform full scan
- $updatedFilterData = array();
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- if (isset($filter[$docId])) {
- $result[$shift + $docId] = 1;
- $updatedFilterData[$docId] = 1; // 1 is just some constant value, so we don't need additional var dereference here
- }
- } else {
- $docId += $docDelta/2;
- $freq = $frqFile->readVInt();
- if (isset($filter[$docId])) {
- $result[$shift + $docId] = $freq;
- $updatedFilterData[$docId] = 1; // 1 is just some constant value, so we don't need additional var dereference here
- }
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
- }
- } else {
- // Filter doesn't has data for current segment
- $filterData = array();
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- $result[$shift + $docId] = 1;
- $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- } else {
- $docId += $docDelta/2;
- $result[$shift + $docId] = $frqFile->readVInt();
- $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $filterData;
- }
- } else {
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- $result[$shift + $docId] = 1;
- } else {
- $docId += $docDelta/2;
- $result[$shift + $docId] = $frqFile->readVInt();
- }
- }
- }
- return $result;
- }
- /**
- * Returns term positions array.
- * Result array structure: array(docId => array(pos1, pos2, ...), ...)
- *
- * @param Zend_Search_Lucene_Index_Term $term
- * @param integer $shift
- * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
- * @return Zend_Search_Lucene_Index_TermInfo
- */
- public function termPositions(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
- {
- $termInfo = $this->getTermInfo($term);
- if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
- if ($docsFilter !== null && $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
- $docsFilter->segmentFilters[$this->_name] = array();
- }
- return array();
- }
- $frqFile = $this->openCompoundFile('.frq');
- $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
- $docId = 0;
- $freqs = array();
- if ($docsFilter !== null) {
- if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
- require_once 'Zend/Search/Lucene/Exception.php';
- throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
- }
- if (isset($docsFilter->segmentFilters[$this->_name])) {
- // Filter already has some data for the current segment
- // Make short name for the filter (which doesn't need additional dereferencing)
- $filter = &$docsFilter->segmentFilters[$this->_name];
- // Check if filter is not empty
- if (count($filter) == 0) {
- return array();
- }
- if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
- // Perform fetching
- // ---------------------------------------------------------------
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- $freqs[$docId] = 1;
- } else {
- $docId += $docDelta/2;
- $freqs[$docId] = $frqFile->readVInt();
- }
- }
- $updatedFilterData = array();
- $result = array();
- $prxFile = $this->openCompoundFile('.prx');
- $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
- foreach ($freqs as $docId => $freq) {
- $termPosition = 0;
- $positions = array();
- // we have to read .prx file to get right position for next doc
- // even filter doesn't match current document
- for ($count = 0; $count < $freq; $count++ ) {
- $termPosition += $prxFile->readVInt();
- $positions[] = $termPosition;
- }
- // Include into updated filter and into result only if doc is matched by filter
- if (isset($filter[$docId])) {
- $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- $result[$shift + $docId] = $positions;
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
- // ---------------------------------------------------------------
- } else {
- // Perform full scan
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- $freqs[$docId] = 1;
- } else {
- $docId += $docDelta/2;
- $freqs[$docId] = $frqFile->readVInt();
- }
- }
- $updatedFilterData = array();
- $result = array();
- $prxFile = $this->openCompoundFile('.prx');
- $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
- foreach ($freqs as $docId => $freq) {
- $termPosition = 0;
- $positions = array();
- // we have to read .prx file to get right position for next doc
- // even filter doesn't match current document
- for ($count = 0; $count < $freq; $count++ ) {
- $termPosition += $prxFile->readVInt();
- $positions[] = $termPosition;
- }
- // Include into updated filter and into result only if doc is matched by filter
- if (isset($filter[$docId])) {
- $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- $result[$shift + $docId] = $positions;
- }
- }
- $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
- }
- } else {
- // Filter doesn't has data for current segment
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- $freqs[$docId] = 1;
- } else {
- $docId += $docDelta/2;
- $freqs[$docId] = $frqFile->readVInt();
- }
- }
- $filterData = array();
- $result = array();
- $prxFile = $this->openCompoundFile('.prx');
- $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
- foreach ($freqs as $docId => $freq) {
- $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
- $termPosition = 0;
- $positions = array();
- for ($count = 0; $count < $freq; $count++ ) {
- $termPosition += $prxFile->readVInt();
- $positions[] = $termPosition;
- }
- $result[$shift + $docId] = $positions;
- }
- $docsFilter->segmentFilters[$this->_name] = $filterData;
- }
- } else {
- for ($count = 0; $count < $termInfo->docFreq; $count++) {
- $docDelta = $frqFile->readVInt();
- if ($docDelta % 2 == 1) {
- $docId += ($docDelta-1)/2;
- $freqs[$docId] = 1;
- } else {
- $docId += $docDelta/2;
- $freqs[$docId] = $frqFile->readVInt();
- }
- }
- $result = array();
- $prxFile = $this->openCompoundFile('.prx');
- $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
- foreach ($freqs as $docId => $freq) {
- $termPosition = 0;
- $positions = array();
- for ($count = 0; $count < $freq; $count++ ) {
- $termPosition += $prxFile->readVInt();
- $positions[] = $termPosition;
- }
- $result[$shift + $docId] = $positions;
- }
- }
- return $result;
- }
- /**
- * Load normalizatin factors from an index file
- *
- * @param integer $fieldNum
- * @throws Zend_Search_Lucene_Exceptio…
Large files files are truncated, but you can click here to view the full file