silverstripe-docsviewer/thirdparty/Zend/Search/Lucene/Index/SegmentInfo.php

<?php
/**
 * Zend Framework
 *
 * LICENSE
 *
 * This source file is subject to the new BSD license that is bundled
 * with this package in the file LICENSE.txt.
 * It is also available through the world-wide-web at this URL:
 * http://framework.zend.com/license/new-bsd
 * If you did not receive a copy of the license and are unable to
 * obtain it through the world-wide-web, please send an email
 * to license@zend.com so we can send you a copy immediately.
 *
 * @category   Zend
 * @package    Zend_Search_Lucene
 * @subpackage Index
 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
 * @license    http://framework.zend.com/license/new-bsd     New BSD License
 * @version    $Id: SegmentInfo.php 20096 2010-01-06 02:05:09Z bkarwin $
 */

/** Zend_Search_Lucene_Index_TermsStream_Interface */
require_once 'Zend/Search/Lucene/Index/TermsStream/Interface.php';


/** Zend_Search_Lucene_Search_Similarity */
require_once 'Zend/Search/Lucene/Search/Similarity.php';

/** Zend_Search_Lucene_Index_FieldInfo */
require_once 'Zend/Search/Lucene/Index/FieldInfo.php';

/** Zend_Search_Lucene_Index_Term */
require_once 'Zend/Search/Lucene/Index/Term.php';

/** Zend_Search_Lucene_Index_TermInfo */
require_once 'Zend/Search/Lucene/Index/TermInfo.php';

/**
 * @category   Zend
 * @package    Zend_Search_Lucene
 * @subpackage Index
 * @copyright  Copyright (c) 2005-2010 Zend Technologies USA Inc. (http://www.zend.com)
 * @license    http://framework.zend.com/license/new-bsd     New BSD License
 */
class Zend_Search_Lucene_Index_SegmentInfo implements Zend_Search_Lucene_Index_TermsStream_Interface
{
    /**
     * "Full scan vs fetch" boundary.
     *
     * If filter selectivity is less than this value, then full scan is performed
     * (since term entries fetching has some additional overhead).
     */
    const FULL_SCAN_VS_FETCH_BOUNDARY = 5;

    /**
     * Number of docs in a segment
     *
     * @var integer
     */
    private $_docCount;

    /**
     * Segment name
     *
     * @var string
     */
    private $_name;

    /**
     * Term Dictionary Index
     *
     * Array of arrays (Zend_Search_Lucene_Index_Term objects are represented as arrays because
     * of performance considerations)
     * [0] -> $termValue
     * [1] -> $termFieldNum
     *
     * Corresponding Zend_Search_Lucene_Index_TermInfo object stored in the $_termDictionaryInfos
     *
     * @var array
     */
    private $_termDictionary;

    /**
     * Term Dictionary Index TermInfos
     *
     * Array of arrays (Zend_Search_Lucene_Index_TermInfo objects are represented as arrays because
     * of performance considerations)
     * [0] -> $docFreq
     * [1] -> $freqPointer
     * [2] -> $proxPointer
     * [3] -> $skipOffset
     * [4] -> $indexPointer
     *
     * @var array
     */
    private $_termDictionaryInfos;

    /**
     * Segment fields. Array of Zend_Search_Lucene_Index_FieldInfo objects for this segment
     *
     * @var array
     */
    private $_fields;

    /**
     * Field positions in a dictionary.
     * (Term dictionary contains filelds ordered by names)
     *
     * @var array
     */
    private $_fieldsDicPositions;


    /**
     * Associative array where the key is the file name and the value is data offset
     * in a compound segment file (.csf).
     *
     * @var array
     */
    private $_segFiles;

    /**
     * Associative array where the key is the file name and the value is file size (.csf).
     *
     * @var array
     */
    private $_segFileSizes;

    /**
     * Delete file generation number
     *
     * -2 means autodetect latest delete generation
     * -1 means 'there is no delete file'
     *  0 means pre-2.1 format delete file
     *  X specifies used delete file
     *
     * @var integer
     */
    private $_delGen;

    /**
     * Segment has single norms file
     *
     * If true then one .nrm file is used for all fields
     * Otherwise .fN files are used
     *
     * @var boolean
     */
    private $_hasSingleNormFile;

    /**
     * Use compound segment file (*.cfs) to collect all other segment files
     * (excluding .del files)
     *
     * @var boolean
     */
    private $_isCompound;


    /**
     * File system adapter.
     *
     * @var Zend_Search_Lucene_Storage_Directory_Filesystem
     */
    private $_directory;

    /**
     * Normalization factors.
     * An array fieldName => normVector
     * normVector is a binary string.
     * Each byte corresponds to an indexed document in a segment and
     * encodes normalization factor (float value, encoded by
     * Zend_Search_Lucene_Search_Similarity::encodeNorm())
     *
     * @var array
     */
    private $_norms = array();

    /**
     * List of deleted documents.
     * bitset if bitset extension is loaded or array otherwise.
     *
     * @var mixed
     */
    private $_deleted = null;

    /**
     * $this->_deleted update flag
     *
     * @var boolean
     */
    private $_deletedDirty = false;

    /**
     * True if segment uses shared doc store
     *
     * @var boolean
     */
    private $_usesSharedDocStore;

    /*
     * Shared doc store options.
     * It's an assotiative array with the following items:
     * - 'offset'     => $docStoreOffset           The starting document in the shared doc store files where this segment's documents begin
     * - 'segment'    => $docStoreSegment          The name of the segment that has the shared doc store files.
     * - 'isCompound' => $docStoreIsCompoundFile   True, if compound file format is used for the shared doc store files (.cfx file).
     */
    private $_sharedDocStoreOptions;


    /**
     * Zend_Search_Lucene_Index_SegmentInfo constructor
     *
     * @param Zend_Search_Lucene_Storage_Directory $directory
     * @param string     $name
     * @param integer    $docCount
     * @param integer    $delGen
     * @param array|null $docStoreOptions
     * @param boolean    $hasSingleNormFile
     * @param boolean    $isCompound
     */
    public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name, $docCount, $delGen = 0, $docStoreOptions = null, $hasSingleNormFile = false, $isCompound = null)
    {
        $this->_directory = $directory;
        $this->_name      = $name;
        $this->_docCount  = $docCount;

        if ($docStoreOptions !== null) {
            $this->_usesSharedDocStore    = true;
            $this->_sharedDocStoreOptions = $docStoreOptions;

            if ($docStoreOptions['isCompound']) {
                $cfxFile       = $this->_directory->getFileObject($docStoreOptions['segment'] . '.cfx');
                $cfxFilesCount = $cfxFile->readVInt();

                $cfxFiles     = array();
                $cfxFileSizes = array();

                for ($count = 0; $count < $cfxFilesCount; $count++) {
                    $dataOffset = $cfxFile->readLong();
                    if ($count != 0) {
                        $cfxFileSizes[$fileName] = $dataOffset - end($cfxFiles);
                    }
                    $fileName            = $cfxFile->readString();
                    $cfxFiles[$fileName] = $dataOffset;
                }
                if ($count != 0) {
                    $cfxFileSizes[$fileName] = $this->_directory->fileLength($docStoreOptions['segment'] . '.cfx') - $dataOffset;
                }

                $this->_sharedDocStoreOptions['files']     = $cfxFiles;
                $this->_sharedDocStoreOptions['fileSizes'] = $cfxFileSizes;
            }
        }

        $this->_hasSingleNormFile = $hasSingleNormFile;
        $this->_delGen            = $delGen;
        $this->_termDictionary    = null;


        if ($isCompound !== null) {
            $this->_isCompound    = $isCompound;
        } else {
            // It's a pre-2.1 segment or isCompound is set to 'unknown'
            // Detect if segment uses compound file
            require_once 'Zend/Search/Lucene/Exception.php';
            try {
                // Try to open compound file
                $this->_directory->getFileObject($name . '.cfs');

                // Compound file is found
                $this->_isCompound = true;
            } catch (Zend_Search_Lucene_Exception $e) {
                if (strpos($e->getMessage(), 'is not readable') !== false) {
                    // Compound file is not found or is not readable
                    $this->_isCompound = false;
                } else {
                    throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
                }
            }
        }

        $this->_segFiles = array();
        if ($this->_isCompound) {
            $cfsFile = $this->_directory->getFileObject($name . '.cfs');
            $segFilesCount = $cfsFile->readVInt();

            for ($count = 0; $count < $segFilesCount; $count++) {
                $dataOffset = $cfsFile->readLong();
                if ($count != 0) {
                    $this->_segFileSizes[$fileName] = $dataOffset - end($this->_segFiles);
                }
                $fileName = $cfsFile->readString();
                $this->_segFiles[$fileName] = $dataOffset;
            }
            if ($count != 0) {
                $this->_segFileSizes[$fileName] = $this->_directory->fileLength($name . '.cfs') - $dataOffset;
            }
        }

        $fnmFile = $this->openCompoundFile('.fnm');
        $fieldsCount = $fnmFile->readVInt();
        $fieldNames = array();
        $fieldNums  = array();
        $this->_fields = array();

        for ($count=0; $count < $fieldsCount; $count++) {
            $fieldName = $fnmFile->readString();
            $fieldBits = $fnmFile->readByte();
            $this->_fields[$count] = new Zend_Search_Lucene_Index_FieldInfo($fieldName,
                                                                            $fieldBits & 0x01 /* field is indexed */,
                                                                            $count,
                                                                            $fieldBits & 0x02 /* termvectors are stored */,
                                                                            $fieldBits & 0x10 /* norms are omitted */,
                                                                            $fieldBits & 0x20 /* payloads are stored */);
            if ($fieldBits & 0x10) {
                // norms are omitted for the indexed field
                $this->_norms[$count] = str_repeat(chr(Zend_Search_Lucene_Search_Similarity::encodeNorm(1.0)), $docCount);
            }

            $fieldNums[$count]  = $count;
            $fieldNames[$count] = $fieldName;
        }
        array_multisort($fieldNames, SORT_ASC, SORT_REGULAR, $fieldNums);
        $this->_fieldsDicPositions = array_flip($fieldNums);

        if ($this->_delGen == -2) {
            // SegmentInfo constructor is invoked from index writer
            // Autodetect current delete file generation number
            $this->_delGen = $this->_detectLatestDelGen();
        }

        // Load deletions
        $this->_deleted = $this->_loadDelFile();
    }

    /**
     * Load detetions file
     *
     * Returns bitset or an array depending on bitset extension availability
     *
     * @return mixed
     * @throws Zend_Search_Lucene_Exception
     */
    private function _loadDelFile()
    {
        if ($this->_delGen == -1) {
            // There is no delete file for this segment
            return null;
        } else if ($this->_delGen == 0) {
            // It's a segment with pre-2.1 format delete file
            // Try to load deletions file
            return $this->_loadPre21DelFile();
        } else {
            // It's 2.1+ format deleteions file
            return $this->_load21DelFile();
        }
    }

    /**
     * Load pre-2.1 detetions file
     *
     * Returns bitset or an array depending on bitset extension availability
     *
     * @return mixed
     * @throws Zend_Search_Lucene_Exception
     */
    private function _loadPre21DelFile()
    {
        require_once 'Zend/Search/Lucene/Exception.php';
        try {
            // '.del' files always stored in a separate file
            // Segment compound is not used
            $delFile = $this->_directory->getFileObject($this->_name . '.del');

            $byteCount = $delFile->readInt();
            $byteCount = ceil($byteCount/8);
            $bitCount  = $delFile->readInt();

            if ($bitCount == 0) {
                $delBytes = '';
            } else {
                $delBytes = $delFile->readBytes($byteCount);
            }

            if (extension_loaded('bitset')) {
                return $delBytes;
            } else {
                $deletions = array();
                for ($count = 0; $count < $byteCount; $count++) {
                    $byte = ord($delBytes[$count]);
                    for ($bit = 0; $bit < 8; $bit++) {
                        if ($byte & (1<<$bit)) {
                            $deletions[$count*8 + $bit] = 1;
                        }
                    }
                }

                return $deletions;
            }
        } catch(Zend_Search_Lucene_Exception $e) {
            if (strpos($e->getMessage(), 'is not readable') === false) {
                throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
            }
            // There is no deletion file
            $this->_delGen = -1;

            return null;
        }
    }

    /**
     * Load 2.1+ format detetions file
     *
     * Returns bitset or an array depending on bitset extension availability
     *
     * @return mixed
     */
    private function _load21DelFile()
    {
        $delFile = $this->_directory->getFileObject($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');

        $format = $delFile->readInt();

        if ($format == (int)0xFFFFFFFF) {
            if (extension_loaded('bitset')) {
                $deletions = bitset_empty();
            } else {
                $deletions = array();
            }

            $byteCount = $delFile->readInt();
            $bitCount  = $delFile->readInt();

            $delFileSize = $this->_directory->fileLength($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');
            $byteNum = 0;

            do {
                $dgap = $delFile->readVInt();
                $nonZeroByte = $delFile->readByte();

                $byteNum += $dgap;


                if (extension_loaded('bitset')) {
                    for ($bit = 0; $bit < 8; $bit++) {
                        if ($nonZeroByte & (1<<$bit)) {
                            bitset_incl($deletions, $byteNum*8 + $bit);
                        }
                    }
                    return $deletions;
                } else {
                    for ($bit = 0; $bit < 8; $bit++) {
                        if ($nonZeroByte & (1<<$bit)) {
                            $deletions[$byteNum*8 + $bit] = 1;
                        }
                    }
                    return (count($deletions) > 0) ? $deletions : null;
                }

            } while ($delFile->tell() < $delFileSize);
        } else {
            // $format is actually byte count
            $byteCount = ceil($format/8);
            $bitCount  = $delFile->readInt();

            if ($bitCount == 0) {
                $delBytes = '';
            } else {
                $delBytes = $delFile->readBytes($byteCount);
            }

            if (extension_loaded('bitset')) {
                return $delBytes;
            } else {
                $deletions = array();
                for ($count = 0; $count < $byteCount; $count++) {
                    $byte = ord($delBytes[$count]);
                    for ($bit = 0; $bit < 8; $bit++) {
                        if ($byte & (1<<$bit)) {
                            $deletions[$count*8 + $bit] = 1;
                        }
                    }
                }

                return (count($deletions) > 0) ? $deletions : null;
            }
        }
    }

    /**
     * Opens index file stoted within compound index file
     *
     * @param string $extension
     * @param boolean $shareHandler
     * @throws Zend_Search_Lucene_Exception
     * @return Zend_Search_Lucene_Storage_File
     */
    public function openCompoundFile($extension, $shareHandler = true)
    {
        if (($extension == '.fdx'  || $extension == '.fdt')  &&  $this->_usesSharedDocStore) {
            $fdxFName = $this->_sharedDocStoreOptions['segment'] . '.fdx';
            $fdtFName = $this->_sharedDocStoreOptions['segment'] . '.fdt';

            if (!$this->_sharedDocStoreOptions['isCompound']) {
                $fdxFile = $this->_directory->getFileObject($fdxFName, $shareHandler);
                $fdxFile->seek($this->_sharedDocStoreOptions['offset']*8, SEEK_CUR);

                if ($extension == '.fdx') {
                    // '.fdx' file is requested
                    return $fdxFile;
                } else {
                    // '.fdt' file is requested
                    $fdtStartOffset = $fdxFile->readLong();

                    $fdtFile = $this->_directory->getFileObject($fdtFName, $shareHandler);
                    $fdtFile->seek($fdtStartOffset, SEEK_CUR);

                    return $fdtFile;
                }
            }

            if( !isset($this->_sharedDocStoreOptions['files'][$fdxFName]) ) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Shared doc storage segment compound file doesn\'t contain '
                                       . $fdxFName . ' file.' );
            }
            if( !isset($this->_sharedDocStoreOptions['files'][$fdtFName]) ) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Shared doc storage segment compound file doesn\'t contain '
                                       . $fdtFName . ' file.' );
            }

            // Open shared docstore segment file
            $cfxFile = $this->_directory->getFileObject($this->_sharedDocStoreOptions['segment'] . '.cfx', $shareHandler);
            // Seek to the start of '.fdx' file within compound file
            $cfxFile->seek($this->_sharedDocStoreOptions['files'][$fdxFName]);
            // Seek to the start of current segment documents section
            $cfxFile->seek($this->_sharedDocStoreOptions['offset']*8, SEEK_CUR);

            if ($extension == '.fdx') {
                // '.fdx' file is requested
                return $cfxFile;
            } else {
                // '.fdt' file is requested
                $fdtStartOffset = $cfxFile->readLong();

                // Seek to the start of '.fdt' file within compound file
                $cfxFile->seek($this->_sharedDocStoreOptions['files'][$fdtFName]);
                // Seek to the start of current segment documents section
                $cfxFile->seek($fdtStartOffset, SEEK_CUR);

                return $fdtFile;
            }
        }

        $filename = $this->_name . $extension;

        if (!$this->_isCompound) {
            return $this->_directory->getFileObject($filename, $shareHandler);
        }

        if( !isset($this->_segFiles[$filename]) ) {
            require_once 'Zend/Search/Lucene/Exception.php';
            throw new Zend_Search_Lucene_Exception('Segment compound file doesn\'t contain '
                                       . $filename . ' file.' );
        }

        $file = $this->_directory->getFileObject($this->_name . '.cfs', $shareHandler);
        $file->seek($this->_segFiles[$filename]);
        return $file;
    }

    /**
     * Get compound file length
     *
     * @param string $extension
     * @return integer
     */
    public function compoundFileLength($extension)
    {
        if (($extension == '.fdx'  || $extension == '.fdt')  &&  $this->_usesSharedDocStore) {
            $filename = $this->_sharedDocStoreOptions['segment'] . $extension;

            if (!$this->_sharedDocStoreOptions['isCompound']) {
                return $this->_directory->fileLength($filename);
            }

            if( !isset($this->_sharedDocStoreOptions['fileSizes'][$filename]) ) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Shared doc store compound file doesn\'t contain '
                                           . $filename . ' file.' );
            }

            return $this->_sharedDocStoreOptions['fileSizes'][$filename];
        }


        $filename = $this->_name . $extension;

        // Try to get common file first
        if ($this->_directory->fileExists($filename)) {
            return $this->_directory->fileLength($filename);
        }

        if( !isset($this->_segFileSizes[$filename]) ) {
            require_once 'Zend/Search/Lucene/Exception.php';
            throw new Zend_Search_Lucene_Exception('Index compound file doesn\'t contain '
                                       . $filename . ' file.' );
        }

        return $this->_segFileSizes[$filename];
    }

    /**
     * Returns field index or -1 if field is not found
     *
     * @param string $fieldName
     * @return integer
     */
    public function getFieldNum($fieldName)
    {
        foreach( $this->_fields as $field ) {
            if( $field->name == $fieldName ) {
                return $field->number;
            }
        }

        return -1;
    }

    /**
     * Returns field info for specified field
     *
     * @param integer $fieldNum
     * @return Zend_Search_Lucene_Index_FieldInfo
     */
    public function getField($fieldNum)
    {
        return $this->_fields[$fieldNum];
    }

    /**
     * Returns array of fields.
     * if $indexed parameter is true, then returns only indexed fields.
     *
     * @param boolean $indexed
     * @return array
     */
    public function getFields($indexed = false)
    {
        $result = array();
        foreach( $this->_fields as $field ) {
            if( (!$indexed) || $field->isIndexed ) {
                $result[ $field->name ] = $field->name;
            }
        }
        return $result;
    }

    /**
     * Returns array of FieldInfo objects.
     *
     * @return array
     */
    public function getFieldInfos()
    {
        return $this->_fields;
    }

    /**
     * Returns actual deletions file generation number.
     *
     * @return integer
     */
    public function getDelGen()
    {
        return $this->_delGen;
    }

    /**
     * Returns the total number of documents in this segment (including deleted documents).
     *
     * @return integer
     */
    public function count()
    {
        return $this->_docCount;
    }

    /**
     * Returns number of deleted documents.
     *
     * @return integer
     */
    private function _deletedCount()
    {
        if ($this->_deleted === null) {
            return 0;
        }

        if (extension_loaded('bitset')) {
            return count(bitset_to_array($this->_deleted));
        } else {
            return count($this->_deleted);
        }
    }

    /**
     * Returns the total number of non-deleted documents in this segment.
     *
     * @return integer
     */
    public function numDocs()
    {
        if ($this->hasDeletions()) {
            return $this->_docCount - $this->_deletedCount();
        } else {
            return $this->_docCount;
        }
    }

    /**
     * Get field position in a fields dictionary
     *
     * @param integer $fieldNum
     * @return integer
     */
    private function _getFieldPosition($fieldNum) {
        // Treat values which are not in a translation table as a 'direct value'
        return isset($this->_fieldsDicPositions[$fieldNum]) ?
                           $this->_fieldsDicPositions[$fieldNum] : $fieldNum;
    }

    /**
     * Return segment name
     *
     * @return string
     */
    public function getName()
    {
        return $this->_name;
    }


    /**
     * TermInfo cache
     *
     * Size is 1024.
     * Numbers are used instead of class constants because of performance considerations
     *
     * @var array
     */
    private $_termInfoCache = array();

    private function _cleanUpTermInfoCache()
    {
        // Clean 256 term infos
        foreach ($this->_termInfoCache as $key => $termInfo) {
            unset($this->_termInfoCache[$key]);

            // leave 768 last used term infos
            if (count($this->_termInfoCache) == 768) {
                break;
            }
        }
    }

    /**
     * Load terms dictionary index
     *
     * @throws Zend_Search_Lucene_Exception
     */
    private function _loadDictionaryIndex()
    {
        // Check, if index is already serialized
        if ($this->_directory->fileExists($this->_name . '.sti')) {
            // Load serialized dictionary index data
            $stiFile = $this->_directory->getFileObject($this->_name . '.sti');
            $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti'));

            // Load dictionary index data
            if (($unserializedData = @unserialize($stiFileData)) !== false) {
                list($this->_termDictionary, $this->_termDictionaryInfos) = $unserializedData;
                return;
            }
        }

        // Load data from .tii file and generate .sti file

        // Prefetch dictionary index data
        $tiiFile = $this->openCompoundFile('.tii');
        $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii'));

        /** Zend_Search_Lucene_Index_DictionaryLoader */
        require_once 'Zend/Search/Lucene/Index/DictionaryLoader.php';

        // Load dictionary index data
        list($this->_termDictionary, $this->_termDictionaryInfos) =
                    Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData);

        $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos));
        $stiFile = $this->_directory->createFile($this->_name . '.sti');
        $stiFile->writeBytes($stiFileData);
    }

    /**
     * Scans terms dictionary and returns term info
     *
     * @param Zend_Search_Lucene_Index_Term $term
     * @return Zend_Search_Lucene_Index_TermInfo
     */
    public function getTermInfo(Zend_Search_Lucene_Index_Term $term)
    {
        $termKey = $term->key();
        if (isset($this->_termInfoCache[$termKey])) {
            $termInfo = $this->_termInfoCache[$termKey];

            // Move termInfo to the end of cache
            unset($this->_termInfoCache[$termKey]);
            $this->_termInfoCache[$termKey] = $termInfo;

            return $termInfo;
        }


        if ($this->_termDictionary === null) {
            $this->_loadDictionaryIndex();
        }

        $searchField = $this->getFieldNum($term->field);

        if ($searchField == -1) {
            return null;
        }
        $searchDicField = $this->_getFieldPosition($searchField);

        // search for appropriate value in dictionary
        $lowIndex = 0;
        $highIndex = count($this->_termDictionary)-1;
        while ($highIndex >= $lowIndex) {
            // $mid = ($highIndex - $lowIndex)/2;
            $mid = ($highIndex + $lowIndex) >> 1;
            $midTerm = $this->_termDictionary[$mid];

            $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */);
            $delta = $searchDicField - $fieldNum;
            if ($delta == 0) {
                $delta = strcmp($term->text, $midTerm[1] /* text */);
            }

            if ($delta < 0) {
                $highIndex = $mid-1;
            } elseif ($delta > 0) {
                $lowIndex  = $mid+1;
            } else {
                // return $this->_termDictionaryInfos[$mid]; // We got it!
                $a = $this->_termDictionaryInfos[$mid];
                $termInfo = new Zend_Search_Lucene_Index_TermInfo($a[0], $a[1], $a[2], $a[3], $a[4]);

                // Put loaded termInfo into cache
                $this->_termInfoCache[$termKey] = $termInfo;

                return $termInfo;
            }
        }

        if ($highIndex == -1) {
            // Term is out of the dictionary range
            return null;
        }

        $prevPosition = $highIndex;
        $prevTerm = $this->_termDictionary[$prevPosition];
        $prevTermInfo = $this->_termDictionaryInfos[$prevPosition];

        $tisFile = $this->openCompoundFile('.tis');
        $tiVersion = $tisFile->readInt();
        if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */  &&
            $tiVersion != (int)0xFFFFFFFD /* 2.1+ format    */) {
            require_once 'Zend/Search/Lucene/Exception.php';
            throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format');
        }

        $termCount     = $tisFile->readLong();
        $indexInterval = $tisFile->readInt();
        $skipInterval  = $tisFile->readInt();
        if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) {
            $maxSkipLevels = $tisFile->readInt();
        }

        $tisFile->seek($prevTermInfo[4] /* indexPointer */ - (($tiVersion == (int)0xFFFFFFFD)? 24 : 20) /* header size*/, SEEK_CUR);

        $termValue    = $prevTerm[1] /* text */;
        $termFieldNum = $prevTerm[0] /* field */;
        $freqPointer = $prevTermInfo[1] /* freqPointer */;
        $proxPointer = $prevTermInfo[2] /* proxPointer */;
        for ($count = $prevPosition*$indexInterval + 1;
             $count <= $termCount &&
             ( $this->_getFieldPosition($termFieldNum) < $searchDicField ||
              ($this->_getFieldPosition($termFieldNum) == $searchDicField &&
               strcmp($termValue, $term->text) < 0) );
             $count++) {
            $termPrefixLength = $tisFile->readVInt();
            $termSuffix       = $tisFile->readString();
            $termFieldNum     = $tisFile->readVInt();
            $termValue        = Zend_Search_Lucene_Index_Term::getPrefix($termValue, $termPrefixLength) . $termSuffix;

            $docFreq      = $tisFile->readVInt();
            $freqPointer += $tisFile->readVInt();
            $proxPointer += $tisFile->readVInt();
            if( $docFreq >= $skipInterval ) {
                $skipOffset = $tisFile->readVInt();
            } else {
                $skipOffset = 0;
            }
        }

        if ($termFieldNum == $searchField && $termValue == $term->text) {
            $termInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);
        } else {
            $termInfo = null;
        }

        // Put loaded termInfo into cache
        $this->_termInfoCache[$termKey] = $termInfo;

        if (count($this->_termInfoCache) == 1024) {
            $this->_cleanUpTermInfoCache();
        }

        return $termInfo;
    }

    /**
     * Returns IDs of all the documents containing term.
     *
     * @param Zend_Search_Lucene_Index_Term $term
     * @param integer $shift
     * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
     * @return array
     */
    public function termDocs(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
    {
        $termInfo = $this->getTermInfo($term);

        if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
            if ($docsFilter !== null  &&  $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
                $docsFilter->segmentFilters[$this->_name] = array();
            }
            return array();
        }

        $frqFile = $this->openCompoundFile('.frq');
        $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
        $docId  = 0;
        $result = array();

        if ($docsFilter !== null) {
            if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
            }

            if (isset($docsFilter->segmentFilters[$this->_name])) {
                // Filter already has some data for the current segment

                // Make short name for the filter (which doesn't need additional dereferencing)
                $filter = &$docsFilter->segmentFilters[$this->_name];

                // Check if filter is not empty
                if (count($filter) == 0) {
                    return array();
                }

                if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
                    // Perform fetching
// ---------------------------------------------------------------
                    $updatedFilterData = array();

                    for( $count=0; $count < $termInfo->docFreq; $count++ ) {
                        $docDelta = $frqFile->readVInt();
                        if( $docDelta % 2 == 1 ) {
                            $docId += ($docDelta-1)/2;
                        } else {
                            $docId += $docDelta/2;
                            // read freq
                            $frqFile->readVInt();
                        }

                        if (isset($filter[$docId])) {
                           $result[] = $shift + $docId;
                           $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                        }
                    }
                    $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
// ---------------------------------------------------------------
                } else {
                    // Perform full scan
                    $updatedFilterData = array();

                    for( $count=0; $count < $termInfo->docFreq; $count++ ) {
                        $docDelta = $frqFile->readVInt();
                        if( $docDelta % 2 == 1 ) {
                            $docId += ($docDelta-1)/2;
                        } else {
                            $docId += $docDelta/2;
                            // read freq
                            $frqFile->readVInt();
                        }

                        if (isset($filter[$docId])) {
                           $result[] = $shift + $docId;
                           $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                        }
                    }
                    $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
                }
            } else {
                // Filter is present, but doesn't has data for the current segment yet
                $filterData = array();
                for( $count=0; $count < $termInfo->docFreq; $count++ ) {
                    $docDelta = $frqFile->readVInt();
                    if( $docDelta % 2 == 1 ) {
                        $docId += ($docDelta-1)/2;
                    } else {
                        $docId += $docDelta/2;
                        // read freq
                        $frqFile->readVInt();
                    }

                    $result[] = $shift + $docId;
                    $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                }
                $docsFilter->segmentFilters[$this->_name] = $filterData;
            }
        } else {
            for( $count=0; $count < $termInfo->docFreq; $count++ ) {
                $docDelta = $frqFile->readVInt();
                if( $docDelta % 2 == 1 ) {
                    $docId += ($docDelta-1)/2;
                } else {
                    $docId += $docDelta/2;
                    // read freq
                    $frqFile->readVInt();
                }

                $result[] = $shift + $docId;
            }
        }

        return $result;
    }

    /**
     * Returns term freqs array.
     * Result array structure: array(docId => freq, ...)
     *
     * @param Zend_Search_Lucene_Index_Term $term
     * @param integer $shift
     * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
     * @return Zend_Search_Lucene_Index_TermInfo
     */
    public function termFreqs(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
    {
        $termInfo = $this->getTermInfo($term);

        if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
            if ($docsFilter !== null  &&  $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
                $docsFilter->segmentFilters[$this->_name] = array();
            }
            return array();
        }

        $frqFile = $this->openCompoundFile('.frq');
        $frqFile->seek($termInfo->freqPointer,SEEK_CUR);
        $result = array();
        $docId = 0;

        $result = array();

        if ($docsFilter !== null) {
            if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
            }

            if (isset($docsFilter->segmentFilters[$this->_name])) {
                // Filter already has some data for the current segment

                // Make short name for the filter (which doesn't need additional dereferencing)
                $filter = &$docsFilter->segmentFilters[$this->_name];

                // Check if filter is not empty
                if (count($filter) == 0) {
                    return array();
                }


                if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
                    // Perform fetching
// ---------------------------------------------------------------
                    $updatedFilterData = array();

                    for ($count = 0; $count < $termInfo->docFreq; $count++) {
                        $docDelta = $frqFile->readVInt();
                        if ($docDelta % 2 == 1) {
                            $docId += ($docDelta-1)/2;
                            if (isset($filter[$docId])) {
                                $result[$shift + $docId] = 1;
                                $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                            }
                        } else {
                            $docId += $docDelta/2;
                            if (isset($filter[$docId])) {
                                $result[$shift + $docId] = $frqFile->readVInt();
                                $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                            }
                        }
                    }
                    $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
// ---------------------------------------------------------------
                } else {
                    // Perform full scan
                    $updatedFilterData = array();

                    for ($count = 0; $count < $termInfo->docFreq; $count++) {
                        $docDelta = $frqFile->readVInt();
                        if ($docDelta % 2 == 1) {
                            $docId += ($docDelta-1)/2;
                            if (isset($filter[$docId])) {
                                $result[$shift + $docId] = 1;
                                $updatedFilterData[$docId] = 1; // 1 is just some constant value, so we don't need additional var dereference here
                            }
                        } else {
                            $docId += $docDelta/2;
                            if (isset($filter[$docId])) {
                                $result[$shift + $docId] = $frqFile->readVInt();
                                $updatedFilterData[$docId] = 1; // 1 is just some constant value, so we don't need additional var dereference here
                            }
                        }
                    }
                    $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
                }
            } else {
                // Filter doesn't has data for current segment
                $filterData = array();

                for ($count = 0; $count < $termInfo->docFreq; $count++) {
                    $docDelta = $frqFile->readVInt();
                    if ($docDelta % 2 == 1) {
                        $docId += ($docDelta-1)/2;
                        $result[$shift + $docId] = 1;
                        $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                    } else {
                        $docId += $docDelta/2;
                        $result[$shift + $docId] = $frqFile->readVInt();
                        $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                    }
                }

                $docsFilter->segmentFilters[$this->_name] = $filterData;
            }
        } else {
            for ($count = 0; $count < $termInfo->docFreq; $count++) {
                $docDelta = $frqFile->readVInt();
                if ($docDelta % 2 == 1) {
                    $docId += ($docDelta-1)/2;
                    $result[$shift + $docId] = 1;
                } else {
                    $docId += $docDelta/2;
                    $result[$shift + $docId] = $frqFile->readVInt();
                }
            }
        }

        return $result;
    }

    /**
     * Returns term positions array.
     * Result array structure: array(docId => array(pos1, pos2, ...), ...)
     *
     * @param Zend_Search_Lucene_Index_Term $term
     * @param integer $shift
     * @param Zend_Search_Lucene_Index_DocsFilter|null $docsFilter
     * @return Zend_Search_Lucene_Index_TermInfo
     */
    public function termPositions(Zend_Search_Lucene_Index_Term $term, $shift = 0, $docsFilter = null)
    {
        $termInfo = $this->getTermInfo($term);

        if (!$termInfo instanceof Zend_Search_Lucene_Index_TermInfo) {
            if ($docsFilter !== null  &&  $docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
                $docsFilter->segmentFilters[$this->_name] = array();
            }
            return array();
        }

        $frqFile = $this->openCompoundFile('.frq');
        $frqFile->seek($termInfo->freqPointer,SEEK_CUR);

        $docId = 0;
        $freqs = array();


        if ($docsFilter !== null) {
            if (!$docsFilter instanceof Zend_Search_Lucene_Index_DocsFilter) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Documents filter must be an instance of Zend_Search_Lucene_Index_DocsFilter or null.');
            }

            if (isset($docsFilter->segmentFilters[$this->_name])) {
                // Filter already has some data for the current segment

                // Make short name for the filter (which doesn't need additional dereferencing)
                $filter = &$docsFilter->segmentFilters[$this->_name];

                // Check if filter is not empty
                if (count($filter) == 0) {
                    return array();
                }

                if ($this->_docCount/count($filter) < self::FULL_SCAN_VS_FETCH_BOUNDARY) {
                    // Perform fetching
// ---------------------------------------------------------------
                    for ($count = 0; $count < $termInfo->docFreq; $count++) {
                        $docDelta = $frqFile->readVInt();
                        if ($docDelta % 2 == 1) {
                            $docId += ($docDelta-1)/2;
                            $freqs[$docId] = 1;
                        } else {
                            $docId += $docDelta/2;
                            $freqs[$docId] = $frqFile->readVInt();
                        }
                    }

                    $updatedFilterData = array();
                    $result = array();
                    $prxFile = $this->openCompoundFile('.prx');
                    $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
                    foreach ($freqs as $docId => $freq) {
                        $termPosition = 0;
                        $positions = array();

                        // we have to read .prx file to get right position for next doc
                        // even filter doesn't match current document
                        for ($count = 0; $count < $freq; $count++ ) {
                            $termPosition += $prxFile->readVInt();
                            $positions[] = $termPosition;
                        }

                        // Include into updated filter and into result only if doc is matched by filter
                        if (isset($filter[$docId])) {
                            $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                            $result[$shift + $docId] = $positions;
                        }
                    }

                    $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
// ---------------------------------------------------------------
                } else {
                    // Perform full scan
                    for ($count = 0; $count < $termInfo->docFreq; $count++) {
                        $docDelta = $frqFile->readVInt();
                        if ($docDelta % 2 == 1) {
                            $docId += ($docDelta-1)/2;
                            $freqs[$docId] = 1;
                        } else {
                            $docId += $docDelta/2;
                            $freqs[$docId] = $frqFile->readVInt();
                        }
                    }

                    $updatedFilterData = array();
                    $result = array();
                    $prxFile = $this->openCompoundFile('.prx');
                    $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
                    foreach ($freqs as $docId => $freq) {
                        $termPosition = 0;
                        $positions = array();

                        // we have to read .prx file to get right position for next doc
                        // even filter doesn't match current document
                        for ($count = 0; $count < $freq; $count++ ) {
                            $termPosition += $prxFile->readVInt();
                            $positions[] = $termPosition;
                        }

                        // Include into updated filter and into result only if doc is matched by filter
                        if (isset($filter[$docId])) {
                            $updatedFilterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here
                            $result[$shift + $docId] = $positions;
                        }
                    }

                    $docsFilter->segmentFilters[$this->_name] = $updatedFilterData;
                }
            } else {
                // Filter doesn't has data for current segment
                for ($count = 0; $count < $termInfo->docFreq; $count++) {
                    $docDelta = $frqFile->readVInt();
                    if ($docDelta % 2 == 1) {
                        $docId += ($docDelta-1)/2;
                        $freqs[$docId] = 1;
                    } else {
                        $docId += $docDelta/2;
                        $freqs[$docId] = $frqFile->readVInt();
                    }
                }

                $filterData = array();
                $result = array();
                $prxFile = $this->openCompoundFile('.prx');
                $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
                foreach ($freqs as $docId => $freq) {
                    $filterData[$docId] = 1; // 1 is just a some constant value, so we don't need additional var dereference here

                    $termPosition = 0;
                    $positions = array();

                    for ($count = 0; $count < $freq; $count++ ) {
                        $termPosition += $prxFile->readVInt();
                        $positions[] = $termPosition;
                    }

                    $result[$shift + $docId] = $positions;
                }

                $docsFilter->segmentFilters[$this->_name] = $filterData;
            }
        } else {
            for ($count = 0; $count < $termInfo->docFreq; $count++) {
                $docDelta = $frqFile->readVInt();
                if ($docDelta % 2 == 1) {
                    $docId += ($docDelta-1)/2;
                    $freqs[$docId] = 1;
                } else {
                    $docId += $docDelta/2;
                    $freqs[$docId] = $frqFile->readVInt();
                }
            }

            $result = array();
            $prxFile = $this->openCompoundFile('.prx');
            $prxFile->seek($termInfo->proxPointer, SEEK_CUR);
            foreach ($freqs as $docId => $freq) {
                $termPosition = 0;
                $positions = array();

                for ($count = 0; $count < $freq; $count++ ) {
                    $termPosition += $prxFile->readVInt();
                    $positions[] = $termPosition;
                }

                $result[$shift + $docId] = $positions;
            }
        }

        return $result;
    }

    /**
     * Load normalizatin factors from an index file
     *
     * @param integer $fieldNum
     * @throws Zend_Search_Lucene_Exception
     */
    private function _loadNorm($fieldNum)
    {
        if ($this->_hasSingleNormFile) {
            $normfFile = $this->openCompoundFile('.nrm');

            $header              = $normfFile->readBytes(3);
            $headerFormatVersion = $normfFile->readByte();

            if ($header != 'NRM'  ||  $headerFormatVersion != (int)0xFF) {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new  Zend_Search_Lucene_Exception('Wrong norms file format.');
            }

            foreach ($this->_fields as $fNum => $fieldInfo) {
                if ($fieldInfo->isIndexed) {
                    $this->_norms[$fNum] = $normfFile->readBytes($this->_docCount);
                }
            }
        } else {
            $fFile = $this->openCompoundFile('.f' . $fieldNum);
            $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount);
        }
    }

    /**
     * Returns normalization factor for specified documents
     *
     * @param integer $id
     * @param string $fieldName
     * @return float
     */
    public function norm($id, $fieldName)
    {
        $fieldNum = $this->getFieldNum($fieldName);

        if ( !($this->_fields[$fieldNum]->isIndexed) ) {
            return null;
        }

        if (!isset($this->_norms[$fieldNum])) {
            $this->_loadNorm($fieldNum);
        }

        return Zend_Search_Lucene_Search_Similarity::decodeNorm( ord($this->_norms[$fieldNum][$id]) );
    }

    /**
     * Returns norm vector, encoded in a byte string
     *
     * @param string $fieldName
     * @return string
     */
    public function normVector($fieldName)
    {
        $fieldNum = $this->getFieldNum($fieldName);

        if ($fieldNum == -1  ||  !($this->_fields[$fieldNum]->isIndexed)) {
            $similarity = Zend_Search_Lucene_Search_Similarity::getDefault();

            return str_repeat(chr($similarity->encodeNorm( $similarity->lengthNorm($fieldName, 0) )),
                              $this->_docCount);
        }

        if (!isset($this->_norms[$fieldNum])) {
            $this->_loadNorm($fieldNum);
        }

        return $this->_norms[$fieldNum];
    }


    /**
     * Returns true if any documents have been deleted from this index segment.
     *
     * @return boolean
     */
    public function hasDeletions()
    {
        return $this->_deleted !== null;
    }


    /**
     * Returns true if segment has single norms file.
     *
     * @return boolean
     */
    public function hasSingleNormFile()
    {
        return $this->_hasSingleNormFile ? true : false;
    }

    /**
     * Returns true if segment is stored using compound segment file.
     *
     * @return boolean
     */
    public function isCompound()
    {
        return $this->_isCompound;
    }

    /**
     * Deletes a document from the index segment.
     * $id is an internal document id
     *
     * @param integer
     */
    public function delete($id)
    {
        $this->_deletedDirty = true;

        if (extension_loaded('bitset')) {
            if ($this->_deleted === null) {
                $this->_deleted = bitset_empty($id);
            }
            bitset_incl($this->_deleted, $id);
        } else {
            if ($this->_deleted === null) {
                $this->_deleted = array();
            }

            $this->_deleted[$id] = 1;
        }
    }

    /**
     * Checks, that document is deleted
     *
     * @param integer
     * @return boolean
     */
    public function isDeleted($id)
    {
        if ($this->_deleted === null) {
            return false;
        }

        if (extension_loaded('bitset')) {
            return bitset_in($this->_deleted, $id);
        } else {
            return isset($this->_deleted[$id]);
        }
    }

    /**
     * Detect latest delete generation
     *
     * Is actualy used from writeChanges() method or from the constructor if it's invoked from
     * Index writer. In both cases index write lock is already obtained, so we shouldn't care
     * about it
     *
     * @return integer
     */
    private function _detectLatestDelGen()
    {
        $delFileList = array();
        foreach ($this->_directory->fileList() as $file) {
            if ($file == $this->_name . '.del') {
                // Matches <segment_name>.del file name
                $delFileList[] = 0;
            } else if (preg_match('/^' . $this->_name . '_([a-zA-Z0-9]+)\.del$/i', $file, $matches)) {
                // Matches <segment_name>_NNN.del file names
                $delFileList[] = (int)base_convert($matches[1], 36, 10);
            }
        }

        if (count($delFileList) == 0) {
            // There is no deletions file for current segment in the directory
            // Set deletions file generation number to 1
            return -1;
        } else {
            // There are some deletions files for current segment in the directory
            // Set deletions file generation number to the highest nuber
            return max($delFileList);
        }
    }

    /**
     * Write changes if it's necessary.
     *
     * This method must be invoked only from the Writer _updateSegments() method,
     * so index Write lock has to be already obtained.
     *
     * @internal
     * @throws Zend_Search_Lucene_Exceptions
     */
    public function writeChanges()
    {
        // Get new generation number
        $latestDelGen = $this->_detectLatestDelGen();

        if (!$this->_deletedDirty) {
            // There was no deletions by current process

            if ($latestDelGen == $this->_delGen) {
                // Delete file hasn't been updated by any concurrent process
                return;
            } else if ($latestDelGen > $this->_delGen) {
                // Delete file has been updated by some concurrent process
                // Reload deletions file
                $this->_delGen  = $latestDelGen;
                $this->_deleted = $this->_loadDelFile();

                return;
            } else {
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Delete file processing workflow is corrupted for the segment \'' . $this->_name . '\'.');
            }
        }

        if ($latestDelGen > $this->_delGen) {
            // Merge current deletions with latest deletions file
            $this->_delGen = $latestDelGen;

            $latestDelete = $this->_loadDelFile();

            if (extension_loaded('bitset')) {
                $this->_deleted = bitset_union($this->_deleted, $latestDelete);
            } else {
                $this->_deleted += $latestDelete;
            }
        }

        if (extension_loaded('bitset')) {
            $delBytes = $this->_deleted;
            $bitCount = count(bitset_to_array($delBytes));
        } else {
            $byteCount = floor($this->_docCount/8)+1;
            $delBytes = str_repeat(chr(0), $byteCount);
            for ($count = 0; $count < $byteCount; $count++) {
                $byte = 0;
                for ($bit = 0; $bit < 8; $bit++) {
                    if (isset($this->_deleted[$count*8 + $bit])) {
                        $byte |= (1<<$bit);
                    }
                }
                $delBytes[$count] = chr($byte);
            }
            $bitCount = count($this->_deleted);
        }

        if ($this->_delGen == -1) {
            // Set delete file generation number to 1
            $this->_delGen = 1;
        } else {
            // Increase delete file generation number by 1
            $this->_delGen++;
        }

        $delFile = $this->_directory->createFile($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del');
        $delFile->writeInt($this->_docCount);
        $delFile->writeInt($bitCount);
        $delFile->writeBytes($delBytes);

        $this->_deletedDirty = false;
    }


    /**
     * Term Dictionary File object for stream like terms reading
     *
     * @var Zend_Search_Lucene_Storage_File
     */
    private $_tisFile = null;

    /**
     * Actual offset of the .tis file data
     *
     * @var integer
     */
    private $_tisFileOffset;

    /**
     * Frequencies File object for stream like terms reading
     *
     * @var Zend_Search_Lucene_Storage_File
     */
    private $_frqFile = null;

    /**
     * Actual offset of the .frq file data
     *
     * @var integer
     */
    private $_frqFileOffset;

    /**
     * Positions File object for stream like terms reading
     *
     * @var Zend_Search_Lucene_Storage_File
     */
    private $_prxFile = null;

    /**
     * Actual offset of the .prx file in the compound file
     *
     * @var integer
     */
    private $_prxFileOffset;


    /**
     * Actual number of terms in term stream
     *
     * @var integer
     */
    private $_termCount = 0;

    /**
     * Overall number of terms in term stream
     *
     * @var integer
     */
    private $_termNum = 0;

    /**
     * Segment index interval
     *
     * @var integer
     */
    private $_indexInterval;

    /**
     * Segment skip interval
     *
     * @var integer
     */
    private $_skipInterval;

    /**
     * Last TermInfo in a terms stream
     *
     * @var Zend_Search_Lucene_Index_TermInfo
     */
    private $_lastTermInfo = null;

    /**
     * Last Term in a terms stream
     *
     * @var Zend_Search_Lucene_Index_Term
     */
    private $_lastTerm = null;

    /**
     * Map of the document IDs
     * Used to get new docID after removing deleted documents.
     * It's not very effective from memory usage point of view,
     * but much more faster, then other methods
     *
     * @var array|null
     */
    private $_docMap = null;

    /**
     * An array of all term positions in the documents.
     * Array structure: array( docId => array( pos1, pos2, ...), ...)
     *
     * Is set to null if term positions loading has to be skipped
     *
     * @var array|null
     */
    private $_lastTermPositions;


    /**
     * Terms scan mode
     *
     * Values:
     *
     * self::SM_TERMS_ONLY - terms are scanned, no additional info is retrieved
     * self::SM_FULL_INFO  - terms are scanned, frequency and position info is retrieved
     * self::SM_MERGE_INFO - terms are scanned, frequency and position info is retrieved
     *                       document numbers are compacted (shifted if segment has deleted documents)
     *
     * @var integer
     */
    private $_termsScanMode;

    /** Scan modes */
    const SM_TERMS_ONLY = 0;    // terms are scanned, no additional info is retrieved
    const SM_FULL_INFO  = 1;    // terms are scanned, frequency and position info is retrieved
    const SM_MERGE_INFO = 2;    // terms are scanned, frequency and position info is retrieved
                                // document numbers are compacted (shifted if segment contains deleted documents)

    /**
     * Reset terms stream
     *
     * $startId - id for the fist document
     * $compact - remove deleted documents
     *
     * Returns start document id for the next segment
     *
     * @param integer $startId
     * @param integer $mode
     * @throws Zend_Search_Lucene_Exception
     * @return integer
     */
    public function resetTermsStream(/** $startId = 0, $mode = self::SM_TERMS_ONLY */)
    {
        /**
         * SegmentInfo->resetTermsStream() method actually takes two optional parameters:
         *   $startId (default value is 0)
         *   $mode (default value is self::SM_TERMS_ONLY)
         */
        $argList = func_get_args();
        if (count($argList) > 2) {
            require_once 'Zend/Search/Lucene/Exception.php';
            throw new Zend_Search_Lucene_Exception('Wrong number of arguments');
        } else if (count($argList) == 2) {
            $startId = $argList[0];
            $mode    = $argList[1];
        } else if (count($argList) == 1) {
            $startId = $argList[0];
            $mode    = self::SM_TERMS_ONLY;
        } else {
            $startId = 0;
            $mode    = self::SM_TERMS_ONLY;
        }

        if ($this->_tisFile !== null) {
            $this->_tisFile = null;
        }

        $this->_tisFile = $this->openCompoundFile('.tis', false);
        $this->_tisFileOffset = $this->_tisFile->tell();

        $tiVersion = $this->_tisFile->readInt();
        if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */  &&
            $tiVersion != (int)0xFFFFFFFD /* 2.1+ format    */) {
            require_once 'Zend/Search/Lucene/Exception.php';
            throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format');
        }

        $this->_termCount     =
              $this->_termNum = $this->_tisFile->readLong(); // Read terms count
        $this->_indexInterval = $this->_tisFile->readInt();  // Read Index interval
        $this->_skipInterval  = $this->_tisFile->readInt();  // Read skip interval
        if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) {
            $maxSkipLevels = $this->_tisFile->readInt();
        }

        if ($this->_frqFile !== null) {
            $this->_frqFile = null;
        }
        if ($this->_prxFile !== null) {
            $this->_prxFile = null;
        }
        $this->_docMap = array();

        $this->_lastTerm          = new Zend_Search_Lucene_Index_Term('', -1);
        $this->_lastTermInfo      = new Zend_Search_Lucene_Index_TermInfo(0, 0, 0, 0);
        $this->_lastTermPositions = null;

        $this->_termsScanMode = $mode;

        switch ($mode) {
            case self::SM_TERMS_ONLY:
                // Do nothing
                break;

            case self::SM_FULL_INFO:
                // break intentionally omitted
            case self::SM_MERGE_INFO:
                $this->_frqFile = $this->openCompoundFile('.frq', false);
                $this->_frqFileOffset = $this->_frqFile->tell();

                $this->_prxFile = $this->openCompoundFile('.prx', false);
                $this->_prxFileOffset = $this->_prxFile->tell();

                for ($count = 0; $count < $this->_docCount; $count++) {
                    if (!$this->isDeleted($count)) {
                        $this->_docMap[$count] = $startId + (($mode == self::SM_MERGE_INFO) ? count($this->_docMap) : $count);
                    }
                }
                break;

            default:
                require_once 'Zend/Search/Lucene/Exception.php';
                throw new Zend_Search_Lucene_Exception('Wrong terms scaning mode specified.');
                break;
        }

        // Calculate next segment start id (since $this->_docMap structure may be cleaned by $this->nextTerm() call)
        $nextSegmentStartId = $startId + (($mode == self::SM_MERGE_INFO) ? count($this->_docMap) : $this->_docCount);
        $this->nextTerm();

        return $nextSegmentStartId;
    }


    /**
     * Skip terms stream up to specified term preffix.
     *
     * Prefix contains fully specified field info and portion of searched term
     *
     * @param Zend_Search_Lucene_Index_Term $prefix
     * @throws Zend_Search_Lucene_Exception
     */
    public function skipTo(Zend_Search_Lucene_Index_Term $prefix)
    {
        if ($this->_termDictionary === null) {
            $this->_loadDictionaryIndex();
        }

        $searchField = $this->getFieldNum($prefix->field);

        if ($searchField == -1) {
            /**
             * Field is not presented in this segment
             * Go to the end of dictionary
             */
            $this->_tisFile = null;
            $this->_frqFile = null;
            $this->_prxFile = null;

            $this->_lastTerm          = null;
            $this->_lastTermInfo      = null;
            $this->_lastTermPositions = null;

            return;
        }
        $searchDicField = $this->_getFieldPosition($searchField);

        // search for appropriate value in dictionary
        $lowIndex = 0;
        $highIndex = count($this->_termDictionary)-1;
        while ($highIndex >= $lowIndex) {
            // $mid = ($highIndex - $lowIndex)/2;
            $mid = ($highIndex + $lowIndex) >> 1;
            $midTerm = $this->_termDictionary[$mid];

            $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */);
            $delta = $searchDicField - $fieldNum;
            if ($delta == 0) {
                $delta = strcmp($prefix->text, $midTerm[1] /* text */);
            }

            if ($delta < 0) {
                $highIndex = $mid-1;
            } elseif ($delta > 0) {
                $lowIndex  = $mid+1;
            } else {
                // We have reached term we are looking for
                break;
            }
        }

        if ($highIndex == -1) {
            // Term is out of the dictionary range
            $this->_tisFile = null;
            $this->_frqFile = null;
            $this->_prxFile = null;

            $this->_lastTerm          = null;
            $this->_lastTermInfo      = null;
            $this->_lastTermPositions = null;

            return;
        }

        $prevPosition = $highIndex;
        $prevTerm = $this->_termDictionary[$prevPosition];
        $prevTermInfo = $this->_termDictionaryInfos[$prevPosition];

        if ($this->_tisFile === null) {
            // The end of terms stream is reached and terms dictionary file is closed
            // Perform mini-reset operation
            $this->_tisFile = $this->openCompoundFile('.tis', false);

            if ($this->_termsScanMode == self::SM_FULL_INFO  ||  $this->_termsScanMode == self::SM_MERGE_INFO) {
                $this->_frqFile = $this->openCompoundFile('.frq', false);
                $this->_prxFile = $this->openCompoundFile('.prx', false);
            }
        }
        $this->_tisFile->seek($this->_tisFileOffset + $prevTermInfo[4], SEEK_SET);

        $this->_lastTerm     = new Zend_Search_Lucene_Index_Term($prevTerm[1] /* text */,
                                                                 ($prevTerm[0] == -1) ? '' : $this->_fields[$prevTerm[0] /* field */]->name);
        $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo($prevTermInfo[0] /* docFreq */,
                                                                     $prevTermInfo[1] /* freqPointer */,
                                                                     $prevTermInfo[2] /* proxPointer */,
                                                                     $prevTermInfo[3] /* skipOffset */);
        $this->_termCount  =  $this->_termNum - $prevPosition*$this->_indexInterval;

        if ($highIndex == 0) {
            // skip start entry
            $this->nextTerm();
        } else if ($prefix->field == $this->_lastTerm->field  &&  $prefix->text  == $this->_lastTerm->text) {
            // We got exact match in the dictionary index

            if ($this->_termsScanMode == self::SM_FULL_INFO  ||  $this->_termsScanMode == self::SM_MERGE_INFO) {
                $this->_lastTermPositions = array();

                $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET);
                $freqs = array();   $docId = 0;
                for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) {
                    $docDelta = $this->_frqFile->readVInt();
                    if( $docDelta % 2 == 1 ) {
                        $docId += ($docDelta-1)/2;
                        $freqs[ $docId ] = 1;
                    } else {
                        $docId += $docDelta/2;
                        $freqs[ $docId ] = $this->_frqFile->readVInt();
                    }
                }

                $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET);
                foreach ($freqs as $docId => $freq) {
                    $termPosition = 0;  $positions = array();

                    for ($count = 0; $count < $freq; $count++ ) {
                        $termPosition += $this->_prxFile->readVInt();
                        $positions[] = $termPosition;
                    }

                    if (isset($this->_docMap[$docId])) {
                        $this->_lastTermPositions[$this->_docMap[$docId]] = $positions;
                    }
                }
            }

            return;
        }

        // Search term matching specified prefix
        while ($this->_lastTerm !== null) {
            if ( strcmp($this->_lastTerm->field, $prefix->field) > 0  ||
                 ($prefix->field == $this->_lastTerm->field  &&  strcmp($this->_lastTerm->text, $prefix->text) >= 0) ) {
                    // Current term matches or greate than the pattern
                    return;
            }

            $this->nextTerm();
        }
    }


    /**
     * Scans terms dictionary and returns next term
     *
     * @return Zend_Search_Lucene_Index_Term|null
     */
    public function nextTerm()
    {
        if ($this->_tisFile === null  ||  $this->_termCount == 0) {
            $this->_lastTerm          = null;
            $this->_lastTermInfo      = null;
            $this->_lastTermPositions = null;
            $this->_docMap            = null;

            // may be necessary for "empty" segment
            $this->_tisFile = null;
            $this->_frqFile = null;
            $this->_prxFile = null;

            return null;
        }

        $termPrefixLength = $this->_tisFile->readVInt();
        $termSuffix       = $this->_tisFile->readString();
        $termFieldNum     = $this->_tisFile->readVInt();
        $termValue        = Zend_Search_Lucene_Index_Term::getPrefix($this->_lastTerm->text, $termPrefixLength) . $termSuffix;

        $this->_lastTerm = new Zend_Search_Lucene_Index_Term($termValue, $this->_fields[$termFieldNum]->name);

        $docFreq     = $this->_tisFile->readVInt();
        $freqPointer = $this->_lastTermInfo->freqPointer + $this->_tisFile->readVInt();
        $proxPointer = $this->_lastTermInfo->proxPointer + $this->_tisFile->readVInt();
        if ($docFreq >= $this->_skipInterval) {
            $skipOffset = $this->_tisFile->readVInt();
        } else {
            $skipOffset = 0;
        }

        $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset);


        if ($this->_termsScanMode == self::SM_FULL_INFO  ||  $this->_termsScanMode == self::SM_MERGE_INFO) {
            $this->_lastTermPositions = array();

            $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET);
            $freqs = array();   $docId = 0;
            for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) {
                $docDelta = $this->_frqFile->readVInt();
                if( $docDelta % 2 == 1 ) {
                    $docId += ($docDelta-1)/2;
                    $freqs[ $docId ] = 1;
                } else {
                    $docId += $docDelta/2;
                    $freqs[ $docId ] = $this->_frqFile->readVInt();
                }
            }

            $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET);
            foreach ($freqs as $docId => $freq) {
                $termPosition = 0;  $positions = array();

                for ($count = 0; $count < $freq; $count++ ) {
                    $termPosition += $this->_prxFile->readVInt();
                    $positions[] = $termPosition;
                }

                if (isset($this->_docMap[$docId])) {
                    $this->_lastTermPositions[$this->_docMap[$docId]] = $positions;
                }
            }
        }

        $this->_termCount--;
        if ($this->_termCount == 0) {
            $this->_tisFile = null;
            $this->_frqFile = null;
            $this->_prxFile = null;
        }

        return $this->_lastTerm;
    }

    /**
     * Close terms stream
     *
     * Should be used for resources clean up if stream is not read up to the end
     */
    public function closeTermsStream()
    {
        $this->_tisFile = null;
        $this->_frqFile = null;
        $this->_prxFile = null;

        $this->_lastTerm          = null;
        $this->_lastTermInfo      = null;
        $this->_lastTermPositions = null;

        $this->_docMap            = null;
    }


    /**
     * Returns term in current position
     *
     * @return Zend_Search_Lucene_Index_Term|null
     */
    public function currentTerm()
    {
        return $this->_lastTerm;
    }


    /**
     * Returns an array of all term positions in the documents.
     * Return array structure: array( docId => array( pos1, pos2, ...), ...)
     *
     * @return array
     */
    public function currentTermPositions()
    {
        return $this->_lastTermPositions;
    }
}