From 314feddd484ae93a40145e8017ffaee5bf9ea0cc Mon Sep 17 00:00:00 2001 From: helpfulrobot Date: Sat, 21 Nov 2015 19:19:20 +1300 Subject: [PATCH] Converted to PSR-2 --- code/search/FullTextSearch.php | 210 +- code/search/SearchIndex.php | 1169 ++++++----- code/search/SearchIntrospection.php | 137 +- code/search/SearchQuery.php | 215 +- code/search/SearchUpdater.php | 385 ++-- code/search/SearchVariant.php | 359 ++-- .../SearchVariantSiteTreeSubsitesPolyhome.php | 148 +- code/search/SearchVariantSubsites.php | 151 +- code/search/SearchVariantVersioned.php | 126 +- .../SearchUpdateBatchedProcessor.php | 319 +-- .../SearchUpdateCommitJobProcessor.php | 509 ++--- .../SearchUpdateImmediateProcessor.php | 10 +- .../SearchUpdateMessageQueueProcessor.php | 28 +- .../processors/SearchUpdateProcessor.php | 257 +-- .../SearchUpdateQueuedJobProcessor.php | 182 +- code/solr/Solr.php | 676 +++--- code/solr/Solr3Service.php | 8 +- code/solr/Solr4Service.php | 95 +- code/solr/SolrConfigStore.php | 173 +- code/solr/SolrIndex.php | 1806 +++++++++-------- code/solr/SolrService.php | 121 +- .../solr/reindex/handlers/SolrReindexBase.php | 404 ++-- .../reindex/handlers/SolrReindexHandler.php | 64 +- .../handlers/SolrReindexImmediateHandler.php | 115 +- .../handlers/SolrReindexMessageHandler.php | 74 +- .../handlers/SolrReindexQueuedHandler.php | 158 +- .../jobs/SolrReindexGroupQueuedJob.php | 187 +- .../reindex/jobs/SolrReindexQueuedJob.php | 153 +- .../reindex/jobs/SolrReindexQueuedJobBase.php | 205 +- code/utils/CombinationsArrayIterator.php | 120 +- code/utils/MultipleArrayIterator.php | 78 +- code/utils/WebDAV.php | 106 +- code/utils/logging/MonologFactory.php | 166 +- code/utils/logging/QueuedJobLogHandler.php | 83 +- code/utils/logging/SearchLogFactory.php | 34 +- tests/BatchedProcessorTest.php | 439 ++-- tests/SearchUpdaterTest.php | 318 +-- ...rchVariantSiteTreeSubsitesPolyhomeTest.php | 119 +- tests/SearchVariantVersionedTest.php | 175 +- tests/Solr4ServiceTest.php | 123 +- tests/SolrIndexTest.php | 531 ++--- tests/SolrIndexVersionedTest.php | 314 +-- tests/SolrReindexQueuedTest.php | 370 ++-- tests/SolrReindexTest.php | 893 ++++---- 44 files changed, 6459 insertions(+), 5854 deletions(-) diff --git a/code/search/FullTextSearch.php b/code/search/FullTextSearch.php index d0616c5..905a03c 100644 --- a/code/search/FullTextSearch.php +++ b/code/search/FullTextSearch.php @@ -3,124 +3,134 @@ /** * Base class to manage active search indexes. */ -class FullTextSearch { +class FullTextSearch +{ + protected static $all_indexes = null; - static protected $all_indexes = null; + protected static $indexes_by_subclass = array(); - static protected $indexes_by_subclass = array(); + /** + * Optional list of index names to limit to. If left empty, all subclasses of SearchIndex + * will be used + * + * @var array + * @config + */ + private static $indexes = array(); - /** - * Optional list of index names to limit to. If left empty, all subclasses of SearchIndex - * will be used - * - * @var array - * @config - */ - private static $indexes = array(); + /** + * Get all the instantiable search indexes (so all the user created indexes, but not the connector or library level + * abstract indexes). Can optionally be filtered to only return indexes that are subclasses of some class + * + * @static + * @param String $class - Class name to filter indexes by, so that all returned indexes are subclasses of provided class + * @param bool $rebuild - If true, don't use cached values + */ + public static function get_indexes($class = null, $rebuild = false) + { + if ($rebuild) { + self::$all_indexes = null; + self::$indexes_by_subclass = array(); + } - /** - * Get all the instantiable search indexes (so all the user created indexes, but not the connector or library level - * abstract indexes). Can optionally be filtered to only return indexes that are subclasses of some class - * - * @static - * @param String $class - Class name to filter indexes by, so that all returned indexes are subclasses of provided class - * @param bool $rebuild - If true, don't use cached values - */ - static function get_indexes($class = null, $rebuild = false) { - if ($rebuild) { - self::$all_indexes = null; - self::$indexes_by_subclass = array(); - } + if (!$class) { + if (self::$all_indexes === null) { + // Get declared indexes, or otherwise default to all subclasses of SearchIndex + $classes = Config::inst()->get(__CLASS__, 'indexes') + ?: ClassInfo::subclassesFor('SearchIndex'); - if (!$class) { - if (self::$all_indexes === null) { - // Get declared indexes, or otherwise default to all subclasses of SearchIndex - $classes = Config::inst()->get(__CLASS__, 'indexes') - ?: ClassInfo::subclassesFor('SearchIndex'); + $hidden = array(); + $candidates = array(); + foreach ($classes as $class) { + // Check if this index is disabled + $hides = $class::config()->hide_ancestor; + if ($hides) { + $hidden[] = $hides; + } - $hidden = array(); - $candidates = array(); - foreach ($classes as $class) { - // Check if this index is disabled - $hides = $class::config()->hide_ancestor; - if($hides) { - $hidden[] = $hides; - } + // Check if this index is abstract + $ref = new ReflectionClass($class); + if (!$ref->isInstantiable()) { + continue; + } - // Check if this index is abstract - $ref = new ReflectionClass($class); - if (!$ref->isInstantiable()) { - continue; - } + $candidates[] = $class; + } - $candidates[] = $class; - } + if ($hidden) { + $candidates = array_diff($candidates, $hidden); + } - if($hidden) { - $candidates = array_diff($candidates, $hidden); - } + // Create all indexes + $concrete = array(); + foreach ($candidates as $class) { + $concrete[$class] = singleton($class); + } - // Create all indexes - $concrete = array(); - foreach($candidates as $class) { - $concrete[$class] = singleton($class); - } + self::$all_indexes = $concrete; + } - self::$all_indexes = $concrete; - } + return self::$all_indexes; + } else { + if (!isset(self::$indexes_by_subclass[$class])) { + $all = self::get_indexes(); - return self::$all_indexes; - } - else { - if (!isset(self::$indexes_by_subclass[$class])) { - $all = self::get_indexes(); + $valid = array(); + foreach ($all as $indexclass => $instance) { + if (is_subclass_of($indexclass, $class)) { + $valid[$indexclass] = $instance; + } + } - $valid = array(); - foreach ($all as $indexclass => $instance) { - if (is_subclass_of($indexclass, $class)) $valid[$indexclass] = $instance; - } + self::$indexes_by_subclass[$class] = $valid; + } - self::$indexes_by_subclass[$class] = $valid; - } + return self::$indexes_by_subclass[$class]; + } + } - return self::$indexes_by_subclass[$class]; - } - } + /** + * Sometimes, like when in tests, you want to restrain the actual indexes to a subset + * + * Call with one argument - an array of class names, index instances or classname => indexinstance pairs (can be mixed). + * Alternatively call with multiple arguments, each of which is a class name or index instance + * + * From then on, fulltext search system will only see those indexes passed in this most recent call. + * + * Passing in no arguments resets back to automatic index list + * + * Alternatively you can use `FullTextSearch.indexes` to configure a list of indexes via config. + */ + public static function force_index_list() + { + $indexes = func_get_args(); - /** - * Sometimes, like when in tests, you want to restrain the actual indexes to a subset - * - * Call with one argument - an array of class names, index instances or classname => indexinstance pairs (can be mixed). - * Alternatively call with multiple arguments, each of which is a class name or index instance - * - * From then on, fulltext search system will only see those indexes passed in this most recent call. - * - * Passing in no arguments resets back to automatic index list - * - * Alternatively you can use `FullTextSearch.indexes` to configure a list of indexes via config. - */ - static function force_index_list() { - $indexes = func_get_args(); + // No arguments = back to automatic + if (!$indexes) { + self::get_indexes(null, true); + return; + } - // No arguments = back to automatic - if (!$indexes) { - self::get_indexes(null, true); - return; - } + // Arguments can be a single array + if (is_array($indexes[0])) { + $indexes = $indexes[0]; + } - // Arguments can be a single array - if (is_array($indexes[0])) $indexes = $indexes[0]; + // Reset to empty first + self::$all_indexes = array(); + self::$indexes_by_subclass = array(); - // Reset to empty first - self::$all_indexes = array(); self::$indexes_by_subclass = array(); - - // And parse out alternative type combos for arguments and add to allIndexes - foreach ($indexes as $class => $index) { - if (is_string($index)) { $class = $index; $index = singleton($class); } - if (is_numeric($class)) $class = get_class($index); - - self::$all_indexes[$class] = $index; - } - } + // And parse out alternative type combos for arguments and add to allIndexes + foreach ($indexes as $class => $index) { + if (is_string($index)) { + $class = $index; + $index = singleton($class); + } + if (is_numeric($class)) { + $class = get_class($index); + } + self::$all_indexes[$class] = $index; + } + } } diff --git a/code/search/SearchIndex.php b/code/search/SearchIndex.php index 6c547b6..ec87358 100644 --- a/code/search/SearchIndex.php +++ b/code/search/SearchIndex.php @@ -27,576 +27,647 @@ * - Specifying update rules that are not extractable from metadata (because the values come from functions for instance) * */ -abstract class SearchIndex extends ViewableData { - - /** - * Allows this index to hide a parent index. Specifies the name of a parent index to disable - * - * @var string - * @config - */ - private static $hide_ancestor; - - public function __construct() { - parent::__construct(); - $this->init(); - - foreach ($this->getClasses() as $class => $options) { - SearchVariant::with($class, $options['include_children'])->call('alterDefinition', $class, $this); - } - - $this->buildDependancyList(); - } - - public function __toString() { - return 'Search Index ' . get_class($this); - } - - /** - * Examines the classes this index is built on to try and find defined fields in the class hierarchy for those classes. - * Looks for db and viewable-data fields, although can't nessecarily find type for viewable-data fields. - */ - public function fieldData($field, $forceType = null, $extraOptions = array()) { - $fullfield = str_replace(".", "_", $field); - $sources = $this->getClasses(); - - foreach ($sources as $source => $options) { - $sources[$source]['base'] = ClassInfo::baseDataClass($source); - $sources[$source]['lookup_chain'] = array(); - } - - $found = array(); - - if (strpos($field, '.') !== false) { - $lookups = explode(".", $field); - $field = array_pop($lookups); - - foreach ($lookups as $lookup) { - $next = array(); - - foreach ($sources as $source => $options) { - $class = null; - - foreach (SearchIntrospection::hierarchy($source, $options['include_children']) as $dataclass) { - $singleton = singleton($dataclass); - - if ($hasOne = $singleton->has_one($lookup)) { - $class = $hasOne; - $options['lookup_chain'][] = array( - 'call' => 'method', 'method' => $lookup, - 'through' => 'has_one', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => "{$lookup}ID" - ); - } - else if ($hasMany = $singleton->has_many($lookup)) { - $class = $hasMany; - $options['multi_valued'] = true; - $options['lookup_chain'][] = array( - 'call' => 'method', 'method' => $lookup, - 'through' => 'has_many', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => $singleton->getRemoteJoinField($lookup, 'has_many') - ); - } - else if ($manyMany = $singleton->many_many($lookup)) { - $class = $manyMany[1]; - $options['multi_valued'] = true; - $options['lookup_chain'][] = array( - 'call' => 'method', 'method' => $lookup, - 'through' => 'many_many', 'class' => $dataclass, 'otherclass' => $class, 'details' => $manyMany - ); - } - - if ($class) { - if (!isset($options['origin'])) $options['origin'] = $dataclass; - $next[$class] = $options; - continue 2; - } - } - } - - if (!$next) return $next; // Early out to avoid excessive empty looping - $sources = $next; - } - } - - foreach ($sources as $class => $options) { - $dataclasses = SearchIntrospection::hierarchy($class, $options['include_children']); - - while (count($dataclasses)) { - $dataclass = array_shift($dataclasses); - $type = null; $fieldoptions = $options; - - $fields = DataObject::database_fields($dataclass); - - if (isset($fields[$field])) { - $type = $fields[$field]; - $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field); - } - else { - $singleton = singleton($dataclass); - - if ($singleton->hasMethod("get$field") || $singleton->hasField($field)) { - $type = $singleton->castingClass($field); - if (!$type) $type = 'String'; - - if ($singleton->hasMethod("get$field")) $fieldoptions['lookup_chain'][] = array('call' => 'method', 'method' => "get$field"); - else $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field); - } - } - - if ($type) { - // Don't search through child classes of a class we matched on. TODO: Should we? - $dataclasses = array_diff($dataclasses, array_values(ClassInfo::subclassesFor($dataclass))); - // Trim arguments off the type string - if (preg_match('/^(\w+)\(/', $type, $match)) $type = $match[1]; - // Get the origin - $origin = isset($fieldoptions['origin']) ? $fieldoptions['origin'] : $dataclass; - - $found["{$origin}_{$fullfield}"] = array( - 'name' => "{$origin}_{$fullfield}", - 'field' => $field, - 'fullfield' => $fullfield, - 'base' => $fieldoptions['base'], - 'origin' => $origin, - 'class' => $dataclass, - 'lookup_chain' => $fieldoptions['lookup_chain'], - 'type' => $forceType ? $forceType : $type, - 'multi_valued' => isset($fieldoptions['multi_valued']) ? true : false, - 'extra_options' => $extraOptions - ); - } - } - } - - return $found; - } - - /** Public, but should only be altered by variants */ - - protected $classes = array(); - - protected $fulltextFields = array(); - - public $filterFields = array(); - - protected $sortFields = array(); - - protected $excludedVariantStates = array(); - - /** - * Add a DataObject subclass whose instances should be included in this index - * - * Can only be called when addFulltextField, addFilterField, addSortField and addAllFulltextFields have not - * yet been called for this index instance - * - * @throws Exception - * @param String $class - The class to include - * @param array $options - TODO: Remove - */ - public function addClass($class, $options = array()) { - if ($this->fulltextFields || $this->filterFields || $this->sortFields) { - throw new Exception('Can\'t add class to Index after fields have already been added'); - } - - if (!DataObject::has_own_table($class)) { - throw new InvalidArgumentException('Can\'t add classes which don\'t have data tables (no $db or $has_one set on the class)'); - } - - $options = array_merge(array( - 'include_children' => true - ), $options); - - $this->classes[$class] = $options; - } - - /** - * Get the classes added by addClass - */ - public function getClasses() { return $this->classes; } - - /** - * Add a field that should be fulltext searchable - * @param String $field - The field to add - * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) - * @param String $extraOptions - Dependent on search implementation - */ - public function addFulltextField($field, $forceType = null, $extraOptions = array()) { - $this->fulltextFields = array_merge($this->fulltextFields, $this->fieldData($field, $forceType, $extraOptions)); - } - - public function getFulltextFields() { return $this->fulltextFields; } - - /** - * Add a field that should be filterable - * @param String $field - The field to add - * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) - * @param String $extraOptions - Dependent on search implementation - */ - public function addFilterField($field, $forceType = null, $extraOptions = array()) { - $this->filterFields = array_merge($this->filterFields, $this->fieldData($field, $forceType, $extraOptions)); - } - - public function getFilterFields() { return $this->filterFields; } - - /** - * Add a field that should be sortable - * @param String $field - The field to add - * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) - * @param String $extraOptions - Dependent on search implementation - */ - public function addSortField($field, $forceType = null, $extraOptions = array()) { - $this->sortFields = array_merge($this->sortFields, $this->fieldData($field, $forceType, $extraOptions)); - } - - public function getSortFields() { return $this->sortFields; } - - /** - * Add all database-backed text fields as fulltext searchable fields. - * - * For every class included in the index, examines those classes and all subclasses looking for "Text" database - * fields (Varchar, Text, HTMLText, etc) and adds them all as fulltext searchable fields. - */ - public function addAllFulltextFields($includeSubclasses = true) { - foreach ($this->getClasses() as $class => $options) { - foreach (SearchIntrospection::hierarchy($class, $includeSubclasses, true) as $dataclass) { - $fields = DataObject::database_fields($dataclass); - - foreach ($fields as $field => $type) { - if (preg_match('/^(\w+)\(/', $type, $match)) $type = $match[1]; - if (is_subclass_of($type, 'StringField')) $this->addFulltextField($field); - } - } - } - } - - /** - * Returns an interator that will let you interate through all added fields, regardless of whether they - * were added as fulltext, filter or sort fields. - * - * @return MultipleArrayIterator - */ - public function getFieldsIterator() { - return new MultipleArrayIterator($this->fulltextFields, $this->filterFields, $this->sortFields); - } - - public function excludeVariantState($state) { - $this->excludedVariantStates[] = $state; - } - - /** Returns true if some variant state should be ignored */ - public function variantStateExcluded($state) { - foreach ($this->excludedVariantStates as $excludedstate) { - $matches = true; - - foreach ($excludedstate as $variant => $variantstate) { - if (!isset($state[$variant]) || $state[$variant] != $variantstate) { $matches = false; break; } - } - - if ($matches) return true; - } - } - - public $dependancyList = array(); - - public function buildDependancyList() { - $this->dependancyList = array_keys($this->getClasses()); - - foreach ($this->getFieldsIterator() as $name => $field) { - if (!isset($field['class'])) continue; - SearchIntrospection::add_unique_by_ancestor($this->dependancyList, $field['class']); - } - } - - public $derivedFields = null; - - /** - * Returns an array where each member is all the fields and the classes that are at the end of some - * specific lookup chain from one of the base classes - */ - public function getDerivedFields() { - if ($this->derivedFields === null) { - $this->derivedFields = array(); - - foreach ($this->getFieldsIterator() as $name => $field) { - if (count($field['lookup_chain']) < 2) continue; - - $key = sha1($field['base'].serialize($field['lookup_chain'])); - $fieldname = "{$field['class']}:{$field['field']}"; - - if (isset($this->derivedFields[$key])) { - $this->derivedFields[$key]['fields'][$fieldname] = $fieldname; - SearchIntrospection::add_unique_by_ancestor($this->derivedFields['classes'], $field['class']); - } - else { - $chain = array_reverse($field['lookup_chain']); - array_shift($chain); - - $this->derivedFields[$key] = array( - 'base' => $field['base'], - 'fields' => array($fieldname => $fieldname), - 'classes' => array($field['class']), - 'chain' => $chain - ); - } - } - } - - return $this->derivedFields; - } - - /** - * Get the "document ID" (a database & variant unique id) given some "Base" class, DataObject ID and state array - * - * @param String $base - The base class of the object - * @param Integer $id - The ID of the object - * @param Array $state - The variant state of the object - * @return string - The document ID as a string - */ - public function getDocumentIDForState($base, $id, $state) { - ksort($state); - $parts = array('id' => $id, 'base' => $base, 'state' => json_encode($state)); - return implode('-', array_values($parts)); - } - - /** - * Get the "document ID" (a database & variant unique id) given some "Base" class and DataObject - * - * @param DataObject $object - The object - * @param String $base - The base class of the object - * @param Boolean $includesubs - TODO: Probably going away - * @return string - The document ID as a string - */ - public function getDocumentID($object, $base, $includesubs) { - return $this->getDocumentIDForState($base, $object->ID, SearchVariant::current_state($base, $includesubs)); - } - - /** - * Given an object and a field definition (as returned by fieldData) get the current value of that field on that object - * - * @param DataObject $object - The object to get the value from - * @param Array $field - The field definition to use - * @return Mixed - The value of the field, or null if we couldn't look it up for some reason - */ - protected function _getFieldValue($object, $field) { - set_error_handler(create_function('$no, $str', 'throw new Exception("HTML Parse Error: ".$str);'), E_ALL); - - try { - foreach ($field['lookup_chain'] as $step) { - // Just fail if we've fallen off the end of the chain - if (!$object) return null; - - // If we're looking up this step on an array or SS_List, do the step on every item, merge result - if (is_array($object) || $object instanceof SS_List) { - $next = array(); - - foreach ($object as $item) { - if ($step['call'] == 'method') { - $method = $step['method']; - $item = $item->$method(); - } - else { - $property = $step['property']; - $item = $item->$property; - } - - if ($item instanceof SS_List) $next = array_merge($next, $item->toArray()); - elseif (is_array($item)) $next = array_merge($next, $item); - else $next[] = $item; - } - - $object = $next; - } - // Otherwise, just call - else { - if ($step['call'] == 'method') { - $method = $step['method']; - $object = $object->$method(); - } - elseif ($step['call'] == 'variant') { - $variants = SearchVariant::variants($field['base'], true); - $variant = $variants[$step['variant']]; $method = $step['method']; - $object = $variant->$method($object); - } - else { - $property = $step['property']; - $object = $object->$property; - } - } - } - } - catch (Exception $e) { - $object = null; - } - - restore_error_handler(); - return $object; - } - - /** - * Given a class, object id, set of stateful ids and a list of changed fields (in a special format), - * return what statefulids need updating in this index - * - * Internal function used by SearchUpdater. - * - * @param $class - * @param $id - * @param $statefulids - * @param $fields - * @return array - */ - public function getDirtyIDs($class, $id, $statefulids, $fields) { - $dirty = array(); - - // First, if this object is directly contained in the index, add it - foreach ($this->classes as $searchclass => $options) { - if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { - - $base = ClassInfo::baseDataClass($searchclass); - $dirty[$base] = array(); - foreach ($statefulids as $statefulid) { - $key = serialize($statefulid); - $dirty[$base][$key] = $statefulid; - } - } - } - - $current = SearchVariant::current_state(); - - - // Then, for every derived field - foreach ($this->getDerivedFields() as $derivation) { - // If the this object is a subclass of any of the classes we want a field from - if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) continue; - if (!array_intersect_key($fields, $derivation['fields'])) continue; - - foreach (SearchVariant::reindex_states($class, false) as $state) { - SearchVariant::activate_state($state); - - $ids = array($id); - - foreach ($derivation['chain'] as $step) { - if ($step['through'] == 'has_one') { - $sql = new SQLQuery('"ID"', '"'.$step['class'].'"', '"'.$step['foreignkey'].'" IN ('.implode(',', $ids).')'); - singleton($step['class'])->extend('augmentSQL', $sql); - - $ids = $sql->execute()->column(); - } - else if ($step['through'] == 'has_many') { - $sql = new SQLQuery('"'.$step['class'].'"."ID"', '"'.$step['class'].'"', '"'.$step['otherclass'].'"."ID" IN ('.implode(',', $ids).')'); - $sql->addInnerJoin($step['otherclass'], '"'.$step['class'].'"."ID" = "'.$step['otherclass'].'"."'.$step['foreignkey'].'"'); - singleton($step['class'])->extend('augmentSQL', $sql); - - $ids = $sql->execute()->column(); - } - } - - SearchVariant::activate_state($current); - - if ($ids) { - $base = $derivation['base']; - if (!isset($dirty[$base])) $dirty[$base] = array(); - - foreach ($ids as $id) { - $statefulid = array('id' => $id, 'state' => $state); - $key = serialize($statefulid); - $dirty[$base][$key] = $statefulid; - } - } - } - } - - return $dirty; - } - - /** !! These should be implemented by the full text search engine */ - - abstract public function add($object) ; - abstract public function delete($base, $id, $state) ; - - abstract public function commit(); - - /** !! These should be implemented by the specific index */ - - /** - * Called during construction, this is the method that builds the structure. - * Used instead of overriding __construct as we have specific execution order - code that has - * to be run before _and/or_ after this. - */ - abstract public function init(); +abstract class SearchIndex extends ViewableData +{ + /** + * Allows this index to hide a parent index. Specifies the name of a parent index to disable + * + * @var string + * @config + */ + private static $hide_ancestor; + + public function __construct() + { + parent::__construct(); + $this->init(); + + foreach ($this->getClasses() as $class => $options) { + SearchVariant::with($class, $options['include_children'])->call('alterDefinition', $class, $this); + } + + $this->buildDependancyList(); + } + + public function __toString() + { + return 'Search Index ' . get_class($this); + } + + /** + * Examines the classes this index is built on to try and find defined fields in the class hierarchy for those classes. + * Looks for db and viewable-data fields, although can't nessecarily find type for viewable-data fields. + */ + public function fieldData($field, $forceType = null, $extraOptions = array()) + { + $fullfield = str_replace(".", "_", $field); + $sources = $this->getClasses(); + + foreach ($sources as $source => $options) { + $sources[$source]['base'] = ClassInfo::baseDataClass($source); + $sources[$source]['lookup_chain'] = array(); + } + + $found = array(); + + if (strpos($field, '.') !== false) { + $lookups = explode(".", $field); + $field = array_pop($lookups); + + foreach ($lookups as $lookup) { + $next = array(); + + foreach ($sources as $source => $options) { + $class = null; + + foreach (SearchIntrospection::hierarchy($source, $options['include_children']) as $dataclass) { + $singleton = singleton($dataclass); + + if ($hasOne = $singleton->has_one($lookup)) { + $class = $hasOne; + $options['lookup_chain'][] = array( + 'call' => 'method', 'method' => $lookup, + 'through' => 'has_one', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => "{$lookup}ID" + ); + } elseif ($hasMany = $singleton->has_many($lookup)) { + $class = $hasMany; + $options['multi_valued'] = true; + $options['lookup_chain'][] = array( + 'call' => 'method', 'method' => $lookup, + 'through' => 'has_many', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => $singleton->getRemoteJoinField($lookup, 'has_many') + ); + } elseif ($manyMany = $singleton->many_many($lookup)) { + $class = $manyMany[1]; + $options['multi_valued'] = true; + $options['lookup_chain'][] = array( + 'call' => 'method', 'method' => $lookup, + 'through' => 'many_many', 'class' => $dataclass, 'otherclass' => $class, 'details' => $manyMany + ); + } + + if ($class) { + if (!isset($options['origin'])) { + $options['origin'] = $dataclass; + } + $next[$class] = $options; + continue 2; + } + } + } + + if (!$next) { + return $next; + } // Early out to avoid excessive empty looping + $sources = $next; + } + } + + foreach ($sources as $class => $options) { + $dataclasses = SearchIntrospection::hierarchy($class, $options['include_children']); + + while (count($dataclasses)) { + $dataclass = array_shift($dataclasses); + $type = null; + $fieldoptions = $options; + + $fields = DataObject::database_fields($dataclass); + + if (isset($fields[$field])) { + $type = $fields[$field]; + $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field); + } else { + $singleton = singleton($dataclass); + + if ($singleton->hasMethod("get$field") || $singleton->hasField($field)) { + $type = $singleton->castingClass($field); + if (!$type) { + $type = 'String'; + } + + if ($singleton->hasMethod("get$field")) { + $fieldoptions['lookup_chain'][] = array('call' => 'method', 'method' => "get$field"); + } else { + $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field); + } + } + } + + if ($type) { + // Don't search through child classes of a class we matched on. TODO: Should we? + $dataclasses = array_diff($dataclasses, array_values(ClassInfo::subclassesFor($dataclass))); + // Trim arguments off the type string + if (preg_match('/^(\w+)\(/', $type, $match)) { + $type = $match[1]; + } + // Get the origin + $origin = isset($fieldoptions['origin']) ? $fieldoptions['origin'] : $dataclass; + + $found["{$origin}_{$fullfield}"] = array( + 'name' => "{$origin}_{$fullfield}", + 'field' => $field, + 'fullfield' => $fullfield, + 'base' => $fieldoptions['base'], + 'origin' => $origin, + 'class' => $dataclass, + 'lookup_chain' => $fieldoptions['lookup_chain'], + 'type' => $forceType ? $forceType : $type, + 'multi_valued' => isset($fieldoptions['multi_valued']) ? true : false, + 'extra_options' => $extraOptions + ); + } + } + } + + return $found; + } + + /** Public, but should only be altered by variants */ + + protected $classes = array(); + + protected $fulltextFields = array(); + + public $filterFields = array(); + + protected $sortFields = array(); + + protected $excludedVariantStates = array(); + + /** + * Add a DataObject subclass whose instances should be included in this index + * + * Can only be called when addFulltextField, addFilterField, addSortField and addAllFulltextFields have not + * yet been called for this index instance + * + * @throws Exception + * @param String $class - The class to include + * @param array $options - TODO: Remove + */ + public function addClass($class, $options = array()) + { + if ($this->fulltextFields || $this->filterFields || $this->sortFields) { + throw new Exception('Can\'t add class to Index after fields have already been added'); + } + + if (!DataObject::has_own_table($class)) { + throw new InvalidArgumentException('Can\'t add classes which don\'t have data tables (no $db or $has_one set on the class)'); + } + + $options = array_merge(array( + 'include_children' => true + ), $options); + + $this->classes[$class] = $options; + } + + /** + * Get the classes added by addClass + */ + public function getClasses() + { + return $this->classes; + } + + /** + * Add a field that should be fulltext searchable + * @param String $field - The field to add + * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) + * @param String $extraOptions - Dependent on search implementation + */ + public function addFulltextField($field, $forceType = null, $extraOptions = array()) + { + $this->fulltextFields = array_merge($this->fulltextFields, $this->fieldData($field, $forceType, $extraOptions)); + } + + public function getFulltextFields() + { + return $this->fulltextFields; + } + + /** + * Add a field that should be filterable + * @param String $field - The field to add + * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) + * @param String $extraOptions - Dependent on search implementation + */ + public function addFilterField($field, $forceType = null, $extraOptions = array()) + { + $this->filterFields = array_merge($this->filterFields, $this->fieldData($field, $forceType, $extraOptions)); + } + + public function getFilterFields() + { + return $this->filterFields; + } + + /** + * Add a field that should be sortable + * @param String $field - The field to add + * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) + * @param String $extraOptions - Dependent on search implementation + */ + public function addSortField($field, $forceType = null, $extraOptions = array()) + { + $this->sortFields = array_merge($this->sortFields, $this->fieldData($field, $forceType, $extraOptions)); + } + + public function getSortFields() + { + return $this->sortFields; + } + + /** + * Add all database-backed text fields as fulltext searchable fields. + * + * For every class included in the index, examines those classes and all subclasses looking for "Text" database + * fields (Varchar, Text, HTMLText, etc) and adds them all as fulltext searchable fields. + */ + public function addAllFulltextFields($includeSubclasses = true) + { + foreach ($this->getClasses() as $class => $options) { + foreach (SearchIntrospection::hierarchy($class, $includeSubclasses, true) as $dataclass) { + $fields = DataObject::database_fields($dataclass); + + foreach ($fields as $field => $type) { + if (preg_match('/^(\w+)\(/', $type, $match)) { + $type = $match[1]; + } + if (is_subclass_of($type, 'StringField')) { + $this->addFulltextField($field); + } + } + } + } + } + + /** + * Returns an interator that will let you interate through all added fields, regardless of whether they + * were added as fulltext, filter or sort fields. + * + * @return MultipleArrayIterator + */ + public function getFieldsIterator() + { + return new MultipleArrayIterator($this->fulltextFields, $this->filterFields, $this->sortFields); + } + + public function excludeVariantState($state) + { + $this->excludedVariantStates[] = $state; + } + + /** Returns true if some variant state should be ignored */ + public function variantStateExcluded($state) + { + foreach ($this->excludedVariantStates as $excludedstate) { + $matches = true; + + foreach ($excludedstate as $variant => $variantstate) { + if (!isset($state[$variant]) || $state[$variant] != $variantstate) { + $matches = false; + break; + } + } + + if ($matches) { + return true; + } + } + } + + public $dependancyList = array(); + + public function buildDependancyList() + { + $this->dependancyList = array_keys($this->getClasses()); + + foreach ($this->getFieldsIterator() as $name => $field) { + if (!isset($field['class'])) { + continue; + } + SearchIntrospection::add_unique_by_ancestor($this->dependancyList, $field['class']); + } + } + + public $derivedFields = null; + + /** + * Returns an array where each member is all the fields and the classes that are at the end of some + * specific lookup chain from one of the base classes + */ + public function getDerivedFields() + { + if ($this->derivedFields === null) { + $this->derivedFields = array(); + + foreach ($this->getFieldsIterator() as $name => $field) { + if (count($field['lookup_chain']) < 2) { + continue; + } + + $key = sha1($field['base'].serialize($field['lookup_chain'])); + $fieldname = "{$field['class']}:{$field['field']}"; + + if (isset($this->derivedFields[$key])) { + $this->derivedFields[$key]['fields'][$fieldname] = $fieldname; + SearchIntrospection::add_unique_by_ancestor($this->derivedFields['classes'], $field['class']); + } else { + $chain = array_reverse($field['lookup_chain']); + array_shift($chain); + + $this->derivedFields[$key] = array( + 'base' => $field['base'], + 'fields' => array($fieldname => $fieldname), + 'classes' => array($field['class']), + 'chain' => $chain + ); + } + } + } + + return $this->derivedFields; + } + + /** + * Get the "document ID" (a database & variant unique id) given some "Base" class, DataObject ID and state array + * + * @param String $base - The base class of the object + * @param Integer $id - The ID of the object + * @param Array $state - The variant state of the object + * @return string - The document ID as a string + */ + public function getDocumentIDForState($base, $id, $state) + { + ksort($state); + $parts = array('id' => $id, 'base' => $base, 'state' => json_encode($state)); + return implode('-', array_values($parts)); + } + + /** + * Get the "document ID" (a database & variant unique id) given some "Base" class and DataObject + * + * @param DataObject $object - The object + * @param String $base - The base class of the object + * @param Boolean $includesubs - TODO: Probably going away + * @return string - The document ID as a string + */ + public function getDocumentID($object, $base, $includesubs) + { + return $this->getDocumentIDForState($base, $object->ID, SearchVariant::current_state($base, $includesubs)); + } + + /** + * Given an object and a field definition (as returned by fieldData) get the current value of that field on that object + * + * @param DataObject $object - The object to get the value from + * @param Array $field - The field definition to use + * @return Mixed - The value of the field, or null if we couldn't look it up for some reason + */ + protected function _getFieldValue($object, $field) + { + set_error_handler(create_function('$no, $str', 'throw new Exception("HTML Parse Error: ".$str);'), E_ALL); + + try { + foreach ($field['lookup_chain'] as $step) { + // Just fail if we've fallen off the end of the chain + if (!$object) { + return null; + } + + // If we're looking up this step on an array or SS_List, do the step on every item, merge result + if (is_array($object) || $object instanceof SS_List) { + $next = array(); + + foreach ($object as $item) { + if ($step['call'] == 'method') { + $method = $step['method']; + $item = $item->$method(); + } else { + $property = $step['property']; + $item = $item->$property; + } + + if ($item instanceof SS_List) { + $next = array_merge($next, $item->toArray()); + } elseif (is_array($item)) { + $next = array_merge($next, $item); + } else { + $next[] = $item; + } + } + + $object = $next; + } + // Otherwise, just call + else { + if ($step['call'] == 'method') { + $method = $step['method']; + $object = $object->$method(); + } elseif ($step['call'] == 'variant') { + $variants = SearchVariant::variants($field['base'], true); + $variant = $variants[$step['variant']]; + $method = $step['method']; + $object = $variant->$method($object); + } else { + $property = $step['property']; + $object = $object->$property; + } + } + } + } catch (Exception $e) { + $object = null; + } + + restore_error_handler(); + return $object; + } + + /** + * Given a class, object id, set of stateful ids and a list of changed fields (in a special format), + * return what statefulids need updating in this index + * + * Internal function used by SearchUpdater. + * + * @param $class + * @param $id + * @param $statefulids + * @param $fields + * @return array + */ + public function getDirtyIDs($class, $id, $statefulids, $fields) + { + $dirty = array(); + + // First, if this object is directly contained in the index, add it + foreach ($this->classes as $searchclass => $options) { + if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { + $base = ClassInfo::baseDataClass($searchclass); + $dirty[$base] = array(); + foreach ($statefulids as $statefulid) { + $key = serialize($statefulid); + $dirty[$base][$key] = $statefulid; + } + } + } + + $current = SearchVariant::current_state(); + + + // Then, for every derived field + foreach ($this->getDerivedFields() as $derivation) { + // If the this object is a subclass of any of the classes we want a field from + if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) { + continue; + } + if (!array_intersect_key($fields, $derivation['fields'])) { + continue; + } + + foreach (SearchVariant::reindex_states($class, false) as $state) { + SearchVariant::activate_state($state); + + $ids = array($id); + + foreach ($derivation['chain'] as $step) { + if ($step['through'] == 'has_one') { + $sql = new SQLQuery('"ID"', '"'.$step['class'].'"', '"'.$step['foreignkey'].'" IN ('.implode(',', $ids).')'); + singleton($step['class'])->extend('augmentSQL', $sql); + + $ids = $sql->execute()->column(); + } elseif ($step['through'] == 'has_many') { + $sql = new SQLQuery('"'.$step['class'].'"."ID"', '"'.$step['class'].'"', '"'.$step['otherclass'].'"."ID" IN ('.implode(',', $ids).')'); + $sql->addInnerJoin($step['otherclass'], '"'.$step['class'].'"."ID" = "'.$step['otherclass'].'"."'.$step['foreignkey'].'"'); + singleton($step['class'])->extend('augmentSQL', $sql); + + $ids = $sql->execute()->column(); + } + } + + SearchVariant::activate_state($current); + + if ($ids) { + $base = $derivation['base']; + if (!isset($dirty[$base])) { + $dirty[$base] = array(); + } + + foreach ($ids as $id) { + $statefulid = array('id' => $id, 'state' => $state); + $key = serialize($statefulid); + $dirty[$base][$key] = $statefulid; + } + } + } + } + + return $dirty; + } + + /** !! These should be implemented by the full text search engine */ + + abstract public function add($object) ; + abstract public function delete($base, $id, $state) ; + + abstract public function commit(); + + /** !! These should be implemented by the specific index */ + + /** + * Called during construction, this is the method that builds the structure. + * Used instead of overriding __construct as we have specific execution order - code that has + * to be run before _and/or_ after this. + */ + abstract public function init(); } /** * A search index that does nothing. Useful for testing */ -abstract class SearchIndex_Null extends SearchIndex { +abstract class SearchIndex_Null extends SearchIndex +{ + public function add($object) + { + } - public function add($object) { } - - public function delete($base, $id, $state) { } - - public function commit() { } + public function delete($base, $id, $state) + { + } + public function commit() + { + } } /** * A search index that just records actions. Useful for testing */ -abstract class SearchIndex_Recording extends SearchIndex { +abstract class SearchIndex_Recording extends SearchIndex +{ + public $added = array(); + public $deleted = array(); + public $committed = false; - public $added = array(); - public $deleted = array(); - public $committed = false; + public function reset() + { + $this->added = array(); + $this->deleted = array(); + $this->committed = false; + } - public function reset() { - $this->added = array(); - $this->deleted = array(); - $this->committed = false; - } + public function add($object) + { + $res = array(); - public function add($object) { - $res = array(); + $res['ID'] = $object->ID; + + foreach ($this->getFieldsIterator() as $name => $field) { + $val = $this->_getFieldValue($object, $field); + $res[$name] = $val; + } - $res['ID'] = $object->ID; - - foreach ($this->getFieldsIterator() as $name => $field) { - $val = $this->_getFieldValue($object, $field); - $res[$name] = $val; - } + $this->added[] = $res; + } - $this->added[] = $res; - } + public function getAdded($fields = array()) + { + $res = array(); - public function getAdded($fields = array()) { - $res = array(); + foreach ($this->added as $added) { + $filtered = array(); + foreach ($fields as $field) { + if (isset($added[$field])) { + $filtered[$field] = $added[$field]; + } + } + $res[] = $filtered; + } - foreach ($this->added as $added) { - $filtered = array(); - foreach ($fields as $field) { - if (isset($added[$field])) $filtered[$field] = $added[$field]; - } - $res[] = $filtered; - } + return $res; + } - return $res; - } + public function delete($base, $id, $state) + { + $this->deleted[] = array('base' => $base, 'id' => $id, 'state' => $state); + } - public function delete($base, $id, $state) { - $this->deleted[] = array('base' => $base, 'id' => $id, 'state' => $state); - } + public function commit() + { + $this->committed = true; + } + + public function getIndexName() + { + return get_class($this); + } - public function commit() { - $this->committed = true; - } - - public function getIndexName() { - return get_class($this); - } + public function getIsCommitted() + { + return $this->committed; + } - public function getIsCommitted() { - return $this->committed; - } - - public function getService() { - // Causes commits to the service to be redirected back to the same object - return $this; - } - + public function getService() + { + // Causes commits to the service to be redirected back to the same object + return $this; + } } diff --git a/code/search/SearchIntrospection.php b/code/search/SearchIntrospection.php index bcebe40..e88aeac 100644 --- a/code/search/SearchIntrospection.php +++ b/code/search/SearchIntrospection.php @@ -3,76 +3,91 @@ /** * Some additional introspection tools that are used often by the fulltext search code */ -class SearchIntrospection { +class SearchIntrospection +{ + protected static $ancestry = array(); - protected static $ancestry = array(); + /** + * Check if class is subclass of (a) the class in $of, or (b) any of the classes in the array $of + * @static + * @param $class + * @param $of + * @return bool + */ + public static function is_subclass_of($class, $of) + { + $ancestry = isset(self::$ancestry[$class]) ? self::$ancestry[$class] : (self::$ancestry[$class] = ClassInfo::ancestry($class)); + return is_array($of) ? (bool)array_intersect($of, $ancestry) : array_key_exists($of, $ancestry); + } - /** - * Check if class is subclass of (a) the class in $of, or (b) any of the classes in the array $of - * @static - * @param $class - * @param $of - * @return bool - */ - static function is_subclass_of ($class, $of) { - $ancestry = isset(self::$ancestry[$class]) ? self::$ancestry[$class] : (self::$ancestry[$class] = ClassInfo::ancestry($class)); - return is_array($of) ? (bool)array_intersect($of, $ancestry) : array_key_exists($of, $ancestry); - } + protected static $hierarchy = array(); - protected static $hierarchy = array(); + /** + * Get all the classes involved in a DataObject hierarchy - both super and optionally subclasses + * + * @static + * @param String $class - The class to query + * @param bool $includeSubclasses - True to return subclasses as well as super classes + * @param bool $dataOnly - True to only return classes that have tables + * @return Array - Integer keys, String values as classes sorted by depth (most super first) + */ + public static function hierarchy($class, $includeSubclasses = true, $dataOnly = false) + { + $key = "$class!" . ($includeSubclasses ? 'sc' : 'an') . '!' . ($dataOnly ? 'do' : 'al'); + + if (!isset(self::$hierarchy[$key])) { + $classes = array_values(ClassInfo::ancestry($class)); + if ($includeSubclasses) { + $classes = array_unique(array_merge($classes, array_values(ClassInfo::subclassesFor($class)))); + } - /** - * Get all the classes involved in a DataObject hierarchy - both super and optionally subclasses - * - * @static - * @param String $class - The class to query - * @param bool $includeSubclasses - True to return subclasses as well as super classes - * @param bool $dataOnly - True to only return classes that have tables - * @return Array - Integer keys, String values as classes sorted by depth (most super first) - */ - static function hierarchy ($class, $includeSubclasses = true, $dataOnly = false) { - $key = "$class!" . ($includeSubclasses ? 'sc' : 'an') . '!' . ($dataOnly ? 'do' : 'al'); - - if (!isset(self::$hierarchy[$key])) { - $classes = array_values(ClassInfo::ancestry($class)); - if ($includeSubclasses) $classes = array_unique(array_merge($classes, array_values(ClassInfo::subclassesFor($class)))); + $idx = array_search('DataObject', $classes); + if ($idx !== false) { + array_splice($classes, 0, $idx+1); + } - $idx = array_search('DataObject', $classes); - if ($idx !== false) array_splice($classes, 0, $idx+1); + if ($dataOnly) { + foreach ($classes as $i => $class) { + if (!DataObject::has_own_table($class)) { + unset($classes[$i]); + } + } + } - if ($dataOnly) foreach($classes as $i => $class) { - if (!DataObject::has_own_table($class)) unset($classes[$i]); - } + self::$hierarchy[$key] = $classes; + } - self::$hierarchy[$key] = $classes; - } + return self::$hierarchy[$key]; + } - return self::$hierarchy[$key]; - } + /** + * Add classes to list, keeping only the parent when parent & child are both in list after add + */ + public static function add_unique_by_ancestor(&$list, $class) + { + // If class already has parent in list, just ignore + if (self::is_subclass_of($class, $list)) { + return; + } - /** - * Add classes to list, keeping only the parent when parent & child are both in list after add - */ - static function add_unique_by_ancestor(&$list, $class) { - // If class already has parent in list, just ignore - if (self::is_subclass_of($class, $list)) return; + // Strip out any subclasses of $class already in the list + $children = ClassInfo::subclassesFor($class); + $list = array_diff($list, $children); - // Strip out any subclasses of $class already in the list - $children = ClassInfo::subclassesFor($class); - $list = array_diff($list, $children); - - // Then add the class in - $list[] = $class; - } - - /** - * Does this class, it's parent (or optionally one of it's children) have the passed extension attached? - */ - static function has_extension($class, $extension, $includeSubclasses = true) { - foreach (self::hierarchy($class, $includeSubclasses) as $relatedclass) { - if ($relatedclass::has_extension($extension)) return true; - } - return false; - } + // Then add the class in + $list[] = $class; + } + /** + * Does this class, it's parent (or optionally one of it's children) have the passed extension attached? + */ + public static function has_extension($class, $extension, $includeSubclasses = true) + { + foreach (self::hierarchy($class, $includeSubclasses) as $relatedclass) { + if ($relatedclass::has_extension($extension)) { + return true; + } + } + return false; + } } diff --git a/code/search/SearchQuery.php b/code/search/SearchQuery.php index 52e4bb2..8f5d08b 100644 --- a/code/search/SearchQuery.php +++ b/code/search/SearchQuery.php @@ -5,129 +5,148 @@ * * API very much still in flux. */ -class SearchQuery extends ViewableData { +class SearchQuery extends ViewableData +{ + public static $missing = null; + public static $present = null; - static $missing = null; - static $present = null; + public static $default_page_size = 10; - static $default_page_size = 10; + /** These are public, but only for index & variant access - API users should not manually access these */ - /** These are public, but only for index & variant access - API users should not manually access these */ + public $search = array(); - public $search = array(); + public $classes = array(); - public $classes = array(); + public $require = array(); + public $exclude = array(); - public $require = array(); - public $exclude = array(); + protected $start = 0; + protected $limit = -1; - protected $start = 0; - protected $limit = -1; + /** These are the API functions */ - /** These are the API functions */ + public function __construct() + { + if (self::$missing === null) { + self::$missing = new stdClass(); + } + if (self::$present === null) { + self::$present = new stdClass(); + } + } - function __construct() { - if (self::$missing === null) self::$missing = new stdClass(); - if (self::$present === null) self::$present = new stdClass(); - } + /** + * @param String $text Search terms. Exact format (grouping, boolean expressions, etc.) depends on the search implementation. + * @param array $fields Limits the search to specific fields (using composite field names) + * @param array $boost Map of composite field names to float values. The higher the value, + * the more important the field gets for relevancy. + */ + public function search($text, $fields = null, $boost = array()) + { + $this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => false); + } - /** - * @param String $text Search terms. Exact format (grouping, boolean expressions, etc.) depends on the search implementation. - * @param array $fields Limits the search to specific fields (using composite field names) - * @param array $boost Map of composite field names to float values. The higher the value, - * the more important the field gets for relevancy. - */ - function search($text, $fields = null, $boost = array()) { - $this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => false); - } + /** + * Similar to {@link search()}, but uses stemming and other similarity algorithms + * to find the searched terms. For example, a term "fishing" would also likely find results + * containing "fish" or "fisher". Depends on search implementation. + * + * @param String $text See {@link search()} + * @param array $fields See {@link search()} + * @param array $boost See {@link search()} + */ + public function fuzzysearch($text, $fields = null, $boost = array()) + { + $this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => true); + } - /** - * Similar to {@link search()}, but uses stemming and other similarity algorithms - * to find the searched terms. For example, a term "fishing" would also likely find results - * containing "fish" or "fisher". Depends on search implementation. - * - * @param String $text See {@link search()} - * @param array $fields See {@link search()} - * @param array $boost See {@link search()} - */ - function fuzzysearch($text, $fields = null, $boost = array()) { - $this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => true); - } + public function inClass($class, $includeSubclasses = true) + { + $this->classes[] = array('class' => $class, 'includeSubclasses' => $includeSubclasses); + } - function inClass($class, $includeSubclasses = true) { - $this->classes[] = array('class' => $class, 'includeSubclasses' => $includeSubclasses); - } + /** + * Similar to {@link search()}, but typically used to further narrow down + * based on other facets which don't influence the field relevancy. + * + * @param String $field Composite name of the field + * @param Mixed $values Scalar value, array of values, or an instance of SearchQuery_Range + */ + public function filter($field, $values) + { + $requires = isset($this->require[$field]) ? $this->require[$field] : array(); + $values = is_array($values) ? $values : array($values); + $this->require[$field] = array_merge($requires, $values); + } - /** - * Similar to {@link search()}, but typically used to further narrow down - * based on other facets which don't influence the field relevancy. - * - * @param String $field Composite name of the field - * @param Mixed $values Scalar value, array of values, or an instance of SearchQuery_Range - */ - function filter($field, $values) { - $requires = isset($this->require[$field]) ? $this->require[$field] : array(); - $values = is_array($values) ? $values : array($values); - $this->require[$field] = array_merge($requires, $values); - } + /** + * Excludes results which match these criteria, inverse of {@link filter()}. + * + * @param String $field + * @param mixed $values + */ + public function exclude($field, $values) + { + $excludes = isset($this->exclude[$field]) ? $this->exclude[$field] : array(); + $values = is_array($values) ? $values : array($values); + $this->exclude[$field] = array_merge($excludes, $values); + } - /** - * Excludes results which match these criteria, inverse of {@link filter()}. - * - * @param String $field - * @param mixed $values - */ - function exclude($field, $values) { - $excludes = isset($this->exclude[$field]) ? $this->exclude[$field] : array(); - $values = is_array($values) ? $values : array($values); - $this->exclude[$field] = array_merge($excludes, $values); - } + public function start($start) + { + $this->start = $start; + } - function start($start) { - $this->start = $start; - } + public function limit($limit) + { + $this->limit = $limit; + } - function limit($limit) { - $this->limit = $limit; - } + public function page($page) + { + $this->start = $page * self::$default_page_size; + $this->limit = self::$default_page_size; + } - function page($page) { - $this->start = $page * self::$default_page_size; - $this->limit = self::$default_page_size; - } + public function isfiltered() + { + return $this->search || $this->classes || $this->require || $this->exclude; + } - function isfiltered() { - return $this->search || $this->classes || $this->require || $this->exclude; - } - - function __toString() { - return "Search Query\n"; - } + public function __toString() + { + return "Search Query\n"; + } } /** * Create one of these and pass as one of the values in filter or exclude to filter or exclude by a (possibly * open ended) range */ -class SearchQuery_Range { +class SearchQuery_Range +{ + public $start = null; + public $end = null; - public $start = null; - public $end = null; + public function __construct($start = null, $end = null) + { + $this->start = $start; + $this->end = $end; + } - function __construct($start = null, $end = null) { - $this->start = $start; - $this->end = $end; - } + public function start($start) + { + $this->start = $start; + } - function start($start) { - $this->start = $start; - } + public function end($end) + { + $this->end = $end; + } - function end($end) { - $this->end = $end; - } - - function isfiltered() { - return $this->start !== null || $this->end !== null; - } -} \ No newline at end of file + public function isfiltered() + { + return $this->start !== null || $this->end !== null; + } +} diff --git a/code/search/SearchUpdater.php b/code/search/SearchUpdater.php index f0b7590..4a9dcde 100644 --- a/code/search/SearchUpdater.php +++ b/code/search/SearchUpdater.php @@ -12,22 +12,25 @@ * * TODO: The way we bind in is awful hacky. */ -class SearchUpdater extends Object { +class SearchUpdater extends Object +{ + /** + * Replace the database object with a subclass that captures all manipulations and passes them to us + */ + public static function bind_manipulation_capture() + { + global $databaseConfig; - /** - * Replace the database object with a subclass that captures all manipulations and passes them to us - */ - static function bind_manipulation_capture() { - global $databaseConfig; + $current = DB::getConn(); + if (!$current || @$current->isManipulationCapture) { + return; + } // If not yet set, or its already captured, just return - $current = DB::getConn(); - if (!$current || @$current->isManipulationCapture) return; // If not yet set, or its already captured, just return + $type = get_class($current); + $file = TEMP_FOLDER."/.cache.SMC.$type"; - $type = get_class($current); - $file = TEMP_FOLDER."/.cache.SMC.$type"; - - if (!is_file($file)) { - file_put_contents($file, "setConnector($current->getConnector()); - $captured->setQueryBuilder($current->getQueryBuilder()); - $captured->setSchemaManager($current->getSchemaManager()); - } + // Framework 3.2+ ORM needs some dependencies set + if (method_exists($captured, "setConnector")) { + $captured->setConnector($current->getConnector()); + $captured->setQueryBuilder($current->getQueryBuilder()); + $captured->setSchemaManager($current->getSchemaManager()); + } - // The connection might have had it's name changed (like if we're currently in a test) - $captured->selectDatabase($current->currentDatabase()); - DB::setConn($captured); - } + // The connection might have had it's name changed (like if we're currently in a test) + $captured->selectDatabase($current->currentDatabase()); + DB::setConn($captured); + } - static $registered = false; - /** @var SearchUpdateProcessor */ - static $processor = null; + public static $registered = false; + /** @var SearchUpdateProcessor */ + public static $processor = null; - /** - * Called by the SearchManiplateCapture database adapter with every manipulation made against the database. - * - * Check every index to see what objects need re-inserting into what indexes to keep the index fresh, - * but doesn't actually do it yet. - * - * TODO: This is pretty sensitive to the format of manipulation that DataObject::write produces. Specifically, - * it expects the actual class of the object to be present as a table, regardless of if any fields changed in that table - * (so a class => array( 'fields' => array() ) item), in order to find the actual class for a set of table manipulations - */ - static function handle_manipulation($manipulation) { - // First, extract any state that is in the manipulation itself - foreach ($manipulation as $table => $details) { - $manipulation[$table]['class'] = $table; - $manipulation[$table]['state'] = array(); - } + /** + * Called by the SearchManiplateCapture database adapter with every manipulation made against the database. + * + * Check every index to see what objects need re-inserting into what indexes to keep the index fresh, + * but doesn't actually do it yet. + * + * TODO: This is pretty sensitive to the format of manipulation that DataObject::write produces. Specifically, + * it expects the actual class of the object to be present as a table, regardless of if any fields changed in that table + * (so a class => array( 'fields' => array() ) item), in order to find the actual class for a set of table manipulations + */ + public static function handle_manipulation($manipulation) + { + // First, extract any state that is in the manipulation itself + foreach ($manipulation as $table => $details) { + $manipulation[$table]['class'] = $table; + $manipulation[$table]['state'] = array(); + } - SearchVariant::call('extractManipulationState', $manipulation); + SearchVariant::call('extractManipulationState', $manipulation); - // Then combine the manipulation back into object field sets + // Then combine the manipulation back into object field sets - $writes = array(); + $writes = array(); - foreach ($manipulation as $table => $details) { - if (!isset($details['id']) || !isset($details['fields'])) continue; + foreach ($manipulation as $table => $details) { + if (!isset($details['id']) || !isset($details['fields'])) { + continue; + } - $id = $details['id']; - $state = $details['state']; - $class = $details['class']; - $fields = $details['fields']; + $id = $details['id']; + $state = $details['state']; + $class = $details['class']; + $fields = $details['fields']; - $base = ClassInfo::baseDataClass($class); - $key = "$id:$base:".serialize($state); + $base = ClassInfo::baseDataClass($class); + $key = "$id:$base:".serialize($state); - $statefulids = array(array('id' => $id, 'state' => $state)); + $statefulids = array(array('id' => $id, 'state' => $state)); - // Is this the first table for this particular object? Then add an item to $writes - if (!isset($writes[$key])) { - $writes[$key] = array( - 'base' => $base, - 'class' => $class, - 'id' => $id, - 'statefulids' => $statefulids, - 'fields' => array() - ); - } - // Otherwise update the class label if it's more specific than the currently recorded one - else if (is_subclass_of($class, $writes[$key]['class'])) { - $writes[$key]['class'] = $class; - } + // Is this the first table for this particular object? Then add an item to $writes + if (!isset($writes[$key])) { + $writes[$key] = array( + 'base' => $base, + 'class' => $class, + 'id' => $id, + 'statefulids' => $statefulids, + 'fields' => array() + ); + } + // Otherwise update the class label if it's more specific than the currently recorded one + elseif (is_subclass_of($class, $writes[$key]['class'])) { + $writes[$key]['class'] = $class; + } - // Update the fields - foreach ($fields as $field => $value) { - $writes[$key]['fields']["$class:$field"] = $value; - } - } + // Update the fields + foreach ($fields as $field => $value) { + $writes[$key]['fields']["$class:$field"] = $value; + } + } - // Then extract any state that is needed for the writes + // Then extract any state that is needed for the writes - SearchVariant::call('extractManipulationWriteState', $writes); + SearchVariant::call('extractManipulationWriteState', $writes); - // Submit all of these writes to the search processor + // Submit all of these writes to the search processor - static::process_writes($writes); - } + static::process_writes($writes); + } - /** - * Send updates to the current search processor for execution - * - * @param array $writes - */ - public static function process_writes($writes) { - foreach ($writes as $write) { - // For every index - foreach (FullTextSearch::get_indexes() as $index => $instance) { - // If that index as a field from this class - if (SearchIntrospection::is_subclass_of($write['class'], $instance->dependancyList)) { - // Get the dirty IDs - $dirtyids = $instance->getDirtyIDs($write['class'], $write['id'], $write['statefulids'], $write['fields']); + /** + * Send updates to the current search processor for execution + * + * @param array $writes + */ + public static function process_writes($writes) + { + foreach ($writes as $write) { + // For every index + foreach (FullTextSearch::get_indexes() as $index => $instance) { + // If that index as a field from this class + if (SearchIntrospection::is_subclass_of($write['class'], $instance->dependancyList)) { + // Get the dirty IDs + $dirtyids = $instance->getDirtyIDs($write['class'], $write['id'], $write['statefulids'], $write['fields']); - // Then add then then to the global list to deal with later - foreach ($dirtyids as $dirtyclass => $ids) { - if ($ids) { - if (!self::$processor) { - self::$processor = Injector::inst()->create('SearchUpdateProcessor'); - } - self::$processor->addDirtyIDs($dirtyclass, $ids, $index); - } - } - } - } - } + // Then add then then to the global list to deal with later + foreach ($dirtyids as $dirtyclass => $ids) { + if ($ids) { + if (!self::$processor) { + self::$processor = Injector::inst()->create('SearchUpdateProcessor'); + } + self::$processor->addDirtyIDs($dirtyclass, $ids, $index); + } + } + } + } + } - // If we do have some work to do register the shutdown function to actually do the work + // If we do have some work to do register the shutdown function to actually do the work - // Don't do it if we're testing - there's no database connection outside the test methods, so we'd - // just get errors - $runningTests = class_exists('SapphireTest', false) && SapphireTest::is_running_test(); + // Don't do it if we're testing - there's no database connection outside the test methods, so we'd + // just get errors + $runningTests = class_exists('SapphireTest', false) && SapphireTest::is_running_test(); - if (self::$processor && !self::$registered && !$runningTests) { - register_shutdown_function(array("SearchUpdater", "flush_dirty_indexes")); - self::$registered = true; - } - } + if (self::$processor && !self::$registered && !$runningTests) { + register_shutdown_function(array("SearchUpdater", "flush_dirty_indexes")); + self::$registered = true; + } + } - /** - * Throw away the recorded dirty IDs without doing anything with them. - */ - static function clear_dirty_indexes() { - self::$processor = null; - } + /** + * Throw away the recorded dirty IDs without doing anything with them. + */ + public static function clear_dirty_indexes() + { + self::$processor = null; + } - /** - * Do something with the recorded dirty IDs, where that "something" depends on the value of self::$update_method, - * either immediately update the indexes, queue a messsage to update the indexes at some point in the future, or - * just throw the dirty IDs away. - */ - static function flush_dirty_indexes() { - if (!self::$processor) return; - self::$processor->triggerProcessing(); - self::$processor = null; - } + /** + * Do something with the recorded dirty IDs, where that "something" depends on the value of self::$update_method, + * either immediately update the indexes, queue a messsage to update the indexes at some point in the future, or + * just throw the dirty IDs away. + */ + public static function flush_dirty_indexes() + { + if (!self::$processor) { + return; + } + self::$processor->triggerProcessing(); + self::$processor = null; + } } -class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter { - public function preRequest(SS_HTTPRequest $request, Session $session, DataModel $model) { - SearchUpdater::bind_manipulation_capture(); - } +class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter +{ + public function preRequest(SS_HTTPRequest $request, Session $session, DataModel $model) + { + SearchUpdater::bind_manipulation_capture(); + } - public function postRequest(SS_HTTPRequest $request, SS_HTTPResponse $response, DataModel $model) { - /* NOP */ - } + public function postRequest(SS_HTTPRequest $request, SS_HTTPResponse $response, DataModel $model) + { + /* NOP */ + } } /** @@ -203,54 +217,57 @@ class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter { * indexed. This causes the object to be marked for deletion from the index. */ -class SearchUpdater_ObjectHandler extends DataExtension { +class SearchUpdater_ObjectHandler extends DataExtension +{ + public function onAfterDelete() + { + // Calling delete() on empty objects does nothing + if (!$this->owner->ID) { + return; + } - public function onAfterDelete() { - // Calling delete() on empty objects does nothing - if (!$this->owner->ID) return; + // Force SearchUpdater to mark this record as dirty + $manipulation = array( + $this->owner->ClassName => array( + 'fields' => array(), + 'id' => $this->owner->ID, + 'command' => 'update' + ) + ); + $this->owner->extend('augmentWrite', $manipulation); + SearchUpdater::handle_manipulation($manipulation); + } - // Force SearchUpdater to mark this record as dirty - $manipulation = array( - $this->owner->ClassName => array( - 'fields' => array(), - 'id' => $this->owner->ID, - 'command' => 'update' - ) - ); - $this->owner->extend('augmentWrite', $manipulation); - SearchUpdater::handle_manipulation($manipulation); - } + /** + * Forces this object to trigger a re-index in the current state + */ + public function triggerReindex() + { + if (!$this->owner->ID) { + return; + } - /** - * Forces this object to trigger a re-index in the current state - */ - public function triggerReindex() { - if (!$this->owner->ID) { - return; - } + $id = $this->owner->ID; + $class = $this->owner->ClassName; + $state = SearchVariant::current_state($class); + $base = ClassInfo::baseDataClass($class); + $key = "$id:$base:".serialize($state); - $id = $this->owner->ID; - $class = $this->owner->ClassName; - $state = SearchVariant::current_state($class); - $base = ClassInfo::baseDataClass($class); - $key = "$id:$base:".serialize($state); + $statefulids = array(array( + 'id' => $id, + 'state' => $state + )); - $statefulids = array(array( - 'id' => $id, - 'state' => $state - )); - - $writes = array( - $key => array( - 'base' => $base, - 'class' => $class, - 'id' => $id, - 'statefulids' => $statefulids, - 'fields' => array() - ) - ); - - SearchUpdater::process_writes($writes); - } + $writes = array( + $key => array( + 'base' => $base, + 'class' => $class, + 'id' => $id, + 'statefulids' => $statefulids, + 'fields' => array() + ) + ); + SearchUpdater::process_writes($writes); + } } diff --git a/code/search/SearchVariant.php b/code/search/SearchVariant.php index e7e7436..1c6f325 100644 --- a/code/search/SearchVariant.php +++ b/code/search/SearchVariant.php @@ -4,205 +4,226 @@ * A Search Variant handles decorators and other situations where the items to reindex or search through are modified * from the default state - for instance, dealing with Versioned or Subsite */ -abstract class SearchVariant { +abstract class SearchVariant +{ + public function __construct() + { + } - function __construct() {} + /*** OVERRIDES start here */ - /*** OVERRIDES start here */ + /** + * Variants can provide any functions they want, but they _must_ override these functions + * with specific ones + */ - /** - * Variants can provide any functions they want, but they _must_ override these functions - * with specific ones - */ + /** + * Return false if there is something missing from the environment (probably a + * not installed module) that means this variant can't apply to any class + */ + abstract public function appliesToEnvironment(); - /** - * Return false if there is something missing from the environment (probably a - * not installed module) that means this variant can't apply to any class - */ - abstract function appliesToEnvironment(); + /** + * Return true if this variant applies to the passed class & subclass + */ + abstract public function appliesTo($class, $includeSubclasses); - /** - * Return true if this variant applies to the passed class & subclass - */ - abstract function appliesTo($class, $includeSubclasses); + /** + * Return the current state + */ + abstract public function currentState(); + /** + * Return all states to step through to reindex all items + */ + abstract public function reindexStates(); + /** + * Activate the passed state + */ + abstract public function activateState($state); - /** - * Return the current state - */ - abstract function currentState(); - /** - * Return all states to step through to reindex all items - */ - abstract function reindexStates(); - /** - * Activate the passed state - */ - abstract function activateState($state); + /** + * Apply this variant to a search query + * + * @param SearchQuery $query + * @param SearchIndex $index + */ + abstract public function alterQuery($query, $index); - /** - * Apply this variant to a search query - * - * @param SearchQuery $query - * @param SearchIndex $index - */ - abstract public function alterQuery($query, $index); + /*** OVERRIDES end here*/ - /*** OVERRIDES end here*/ + /** Holds a cache of all variants */ + protected static $variants = null; + /** Holds a cache of the variants keyed by "class!" "1"? (1 = include subclasses) */ + protected static $class_variants = array(); - /** Holds a cache of all variants */ - protected static $variants = null; - /** Holds a cache of the variants keyed by "class!" "1"? (1 = include subclasses) */ - protected static $class_variants = array(); + /** + * Returns an array of variants. + * + * With no arguments, returns all variants + * + * With a classname as the first argument, returns the variants that apply to that class + * (optionally including subclasses) + * + * @static + * @param string $class - The class name to get variants for + * @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class + * @return array - An array of (string)$variantClassName => (Object)$variantInstance pairs + */ + public static function variants($class = null, $includeSubclasses = true) + { + if (!$class) { + if (self::$variants === null) { + $classes = ClassInfo::subclassesFor('SearchVariant'); - /** - * Returns an array of variants. - * - * With no arguments, returns all variants - * - * With a classname as the first argument, returns the variants that apply to that class - * (optionally including subclasses) - * - * @static - * @param string $class - The class name to get variants for - * @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class - * @return array - An array of (string)$variantClassName => (Object)$variantInstance pairs - */ - public static function variants($class = null, $includeSubclasses = true) { - if (!$class) { - if (self::$variants === null) { - $classes = ClassInfo::subclassesFor('SearchVariant'); + $concrete = array(); + foreach ($classes as $variantclass) { + $ref = new ReflectionClass($variantclass); + if ($ref->isInstantiable()) { + $variant = singleton($variantclass); + if ($variant->appliesToEnvironment()) { + $concrete[$variantclass] = $variant; + } + } + } - $concrete = array(); - foreach ($classes as $variantclass) { - $ref = new ReflectionClass($variantclass); - if ($ref->isInstantiable()) { - $variant = singleton($variantclass); - if ($variant->appliesToEnvironment()) $concrete[$variantclass] = $variant; - } - } + self::$variants = $concrete; + } - self::$variants = $concrete; - } + return self::$variants; + } else { + $key = $class . '!' . $includeSubclasses; - return self::$variants; - } - else { - $key = $class . '!' . $includeSubclasses; + if (!isset(self::$class_variants[$key])) { + self::$class_variants[$key] = array(); - if (!isset(self::$class_variants[$key])) { - self::$class_variants[$key] = array(); + foreach (self::variants() as $variantclass => $instance) { + if ($instance->appliesTo($class, $includeSubclasses)) { + self::$class_variants[$key][$variantclass] = $instance; + } + } + } - foreach (self::variants() as $variantclass => $instance) { - if ($instance->appliesTo($class, $includeSubclasses)) self::$class_variants[$key][$variantclass] = $instance; - } - } + return self::$class_variants[$key]; + } + } - return self::$class_variants[$key]; - } - } + /** Holds a cache of SearchVariant_Caller instances, one for each class/includeSubclasses setting */ + protected static $call_instances = array(); - /** Holds a cache of SearchVariant_Caller instances, one for each class/includeSubclasses setting */ - protected static $call_instances = array(); + /** + * Lets you call any function on all variants that support it, in the same manner as "Object#extend" calls + * a method from extensions. + * + * Usage: SearchVariant::with(...)->call($method, $arg1, ...); + * + * @static + * + * @param string $class - (Optional) a classname. If passed, only variants that apply to that class will be checked / called + * + * @param bool $includeSubclasses - (Optional) If false, only variants that apply strictly to the passed class or its super-classes + * will be checked. If true (the default), variants that apply to any sub-class of the passed class with also be checked + * + * @return An object with one method, call() + */ + public static function with($class = null, $includeSubclasses = true) + { + // Make the cache key + $key = $class ? $class . '!' . $includeSubclasses : '!'; + // If no SearchVariant_Caller instance yet, create it + if (!isset(self::$call_instances[$key])) { + self::$call_instances[$key] = new SearchVariant_Caller(self::variants($class, $includeSubclasses)); + } + // Then return it + return self::$call_instances[$key]; + } - /** - * Lets you call any function on all variants that support it, in the same manner as "Object#extend" calls - * a method from extensions. - * - * Usage: SearchVariant::with(...)->call($method, $arg1, ...); - * - * @static - * - * @param string $class - (Optional) a classname. If passed, only variants that apply to that class will be checked / called - * - * @param bool $includeSubclasses - (Optional) If false, only variants that apply strictly to the passed class or its super-classes - * will be checked. If true (the default), variants that apply to any sub-class of the passed class with also be checked - * - * @return An object with one method, call() - */ - static function with($class = null, $includeSubclasses = true) { - // Make the cache key - $key = $class ? $class . '!' . $includeSubclasses : '!'; - // If no SearchVariant_Caller instance yet, create it - if (!isset(self::$call_instances[$key])) self::$call_instances[$key] = new SearchVariant_Caller(self::variants($class, $includeSubclasses)); - // Then return it - return self::$call_instances[$key]; - } + /** + * A shortcut to with when calling without passing in a class, + * + * SearchVariant::call(...) ==== SearchVariant::with()->call(...); + */ + public static function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null) + { + return self::with()->call($method, $a1, $a2, $a3, $a4, $a5, $a6, $a7); + } - /** - * A shortcut to with when calling without passing in a class, - * - * SearchVariant::call(...) ==== SearchVariant::with()->call(...); - */ - static function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null) { - return self::with()->call($method, $a1, $a2, $a3, $a4, $a5, $a6, $a7); - } + /** + * Get the current state of every variant + * @static + * @return array + */ + public static function current_state($class = null, $includeSubclasses = true) + { + $state = array(); + foreach (self::variants($class, $includeSubclasses) as $variant => $instance) { + $state[$variant] = $instance->currentState(); + } + return $state; + } - /** - * Get the current state of every variant - * @static - * @return array - */ - static function current_state($class = null, $includeSubclasses = true) { - $state = array(); - foreach (self::variants($class, $includeSubclasses) as $variant => $instance) { - $state[$variant] = $instance->currentState(); - } - return $state; - } + /** + * Activate all the states in the passed argument + * @static + * @param (array) $state. A set of (string)$variantClass => (any)$state pairs , e.g. as returned by + * SearchVariant::current_state() + * @return void + */ + public static function activate_state($state) + { + foreach (self::variants() as $variant => $instance) { + if (isset($state[$variant])) { + $instance->activateState($state[$variant]); + } + } + } - /** - * Activate all the states in the passed argument - * @static - * @param (array) $state. A set of (string)$variantClass => (any)$state pairs , e.g. as returned by - * SearchVariant::current_state() - * @return void - */ - static function activate_state($state) { - foreach (self::variants() as $variant => $instance) { - if (isset($state[$variant])) $instance->activateState($state[$variant]); - } - } + /** + * Return an iterator that, when used in a for loop, activates one combination of reindex states per loop, and restores + * back to the original state at the end + * @static + * @param string $class - The class name to get variants for + * @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class + * @return SearchVariant_ReindexStateIteratorRet - The iterator to foreach loop over + */ + public static function reindex_states($class = null, $includeSubclasses = true) + { + $allstates = array(); - /** - * Return an iterator that, when used in a for loop, activates one combination of reindex states per loop, and restores - * back to the original state at the end - * @static - * @param string $class - The class name to get variants for - * @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class - * @return SearchVariant_ReindexStateIteratorRet - The iterator to foreach loop over - */ - static function reindex_states($class = null, $includeSubclasses = true) { - $allstates = array(); + foreach (self::variants($class, $includeSubclasses) as $variant => $instance) { + if ($states = $instance->reindexStates()) { + $allstates[$variant] = $states; + } + } - foreach (self::variants($class, $includeSubclasses) as $variant => $instance) { - if ($states = $instance->reindexStates()) $allstates[$variant] = $states; - } - - return $allstates ? new CombinationsArrayIterator($allstates) : array(array()); - } + return $allstates ? new CombinationsArrayIterator($allstates) : array(array()); + } } /** * Internal utility class used to hold the state of the SearchVariant::with call */ -class SearchVariant_Caller { - protected $variants = null; +class SearchVariant_Caller +{ + protected $variants = null; - function __construct($variants) { - $this->variants = $variants; - } + public function __construct($variants) + { + $this->variants = $variants; + } - function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null) { - $values = array(); + public function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null) + { + $values = array(); - foreach ($this->variants as $variant) { - if (method_exists($variant, $method)) { - $value = $variant->$method($a1, $a2, $a3, $a4, $a5, $a6, $a7); - if ($value !== null) $values[] = $value; - } - } + foreach ($this->variants as $variant) { + if (method_exists($variant, $method)) { + $value = $variant->$method($a1, $a2, $a3, $a4, $a5, $a6, $a7); + if ($value !== null) { + $values[] = $value; + } + } + } - return $values; - } + return $values; + } } - diff --git a/code/search/SearchVariantSiteTreeSubsitesPolyhome.php b/code/search/SearchVariantSiteTreeSubsitesPolyhome.php index 480bfa2..4403d5e 100644 --- a/code/search/SearchVariantSiteTreeSubsitesPolyhome.php +++ b/code/search/SearchVariantSiteTreeSubsitesPolyhome.php @@ -1,85 +1,95 @@ ID; + } + } - if ($ids === null) { - $ids = array(0); - foreach (DataObject::get('Subsite') as $subsite) $ids[] = $subsite->ID; - } + return $ids; + } + public function activateState($state) + { + if (Controller::has_curr()) { + Subsite::changeSubsite($state); + } else { + // TODO: This is a nasty hack - calling Subsite::changeSubsite after request ends + // throws error because no current controller to access session on + $_REQUEST['SubsiteID'] = $state; + } + } - return $ids; - } - function activateState($state) { - if (Controller::has_curr()) { - Subsite::changeSubsite($state); - } - else { - // TODO: This is a nasty hack - calling Subsite::changeSubsite after request ends - // throws error because no current controller to access session on - $_REQUEST['SubsiteID'] = $state; - } - } + public function alterDefinition($base, $index) + { + $self = get_class($this); + + $index->filterFields['_subsite'] = array( + 'name' => '_subsite', + 'field' => '_subsite', + 'fullfield' => '_subsite', + 'base' => $base, + 'origin' => $base, + 'type' => 'Int', + 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) + ); + } - function alterDefinition($base, $index) { - $self = get_class($this); - - $index->filterFields['_subsite'] = array( - 'name' => '_subsite', - 'field' => '_subsite', - 'fullfield' => '_subsite', - 'base' => $base, - 'origin' => $base, - 'type' => 'Int', - 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) - ); - } + public function alterQuery($query, $index) + { + $subsite = Subsite::currentSubsiteID(); + $query->filter('_subsite', array($subsite, SearchQuery::$missing)); + } - public function alterQuery($query, $index) { - $subsite = Subsite::currentSubsiteID(); - $query->filter('_subsite', array($subsite, SearchQuery::$missing)); - } + public static $subsites = null; - static $subsites = null; + /** + * We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit + * deletes, just new versions with different members) so just always use all of them + */ + public function extractManipulationWriteState(&$writes) + { + $self = get_class($this); - /** - * We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit - * deletes, just new versions with different members) so just always use all of them - */ - function extractManipulationWriteState(&$writes) { - $self = get_class($this); + foreach ($writes as $key => $write) { + if (!$this->appliesTo($write['class'], true)) { + continue; + } - foreach ($writes as $key => $write) { - if (!$this->appliesTo($write['class'], true)) continue; + if (self::$subsites === null) { + $query = new SQLQuery('ID', 'Subsite'); + self::$subsites = array_merge(array('0'), $query->execute()->column()); + } - if (self::$subsites === null) { - $query = new SQLQuery('ID', 'Subsite'); - self::$subsites = array_merge(array('0'), $query->execute()->column()); - } + $next = array(); - $next = array(); - - foreach ($write['statefulids'] as $i => $statefulid) { - foreach (self::$subsites as $subsiteID) { - $next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => $subsiteID))); - } - } - - $writes[$key]['statefulids'] = $next; - } - } + foreach ($write['statefulids'] as $i => $statefulid) { + foreach (self::$subsites as $subsiteID) { + $next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => $subsiteID))); + } + } + $writes[$key]['statefulids'] = $next; + } + } } diff --git a/code/search/SearchVariantSubsites.php b/code/search/SearchVariantSubsites.php index 51886b3..ace782d 100644 --- a/code/search/SearchVariantSubsites.php +++ b/code/search/SearchVariantSubsites.php @@ -1,88 +1,99 @@ ID; + } + } - if ($ids === null) { - $ids = array('0'); - foreach (DataObject::get('Subsite') as $subsite) $ids[] = (string)$subsite->ID; - } + return $ids; + } - return $ids; - } + public function activateState($state) + { + // We always just set the $_GET variable rather than store in Session - this always works, has highest priority + // in Subsite::currentSubsiteID() and doesn't persist unlike Subsite::changeSubsite + $_GET['SubsiteID'] = $state; + Permission::flush_permission_cache(); + } - function activateState($state) { - // We always just set the $_GET variable rather than store in Session - this always works, has highest priority - // in Subsite::currentSubsiteID() and doesn't persist unlike Subsite::changeSubsite - $_GET['SubsiteID'] = $state; - Permission::flush_permission_cache(); - } + public function alterDefinition($base, $index) + { + $self = get_class($this); + + $index->filterFields['_subsite'] = array( + 'name' => '_subsite', + 'field' => '_subsite', + 'fullfield' => '_subsite', + 'base' => $base, + 'origin' => $base, + 'type' => 'Int', + 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) + ); + } - function alterDefinition($base, $index) { - $self = get_class($this); - - $index->filterFields['_subsite'] = array( - 'name' => '_subsite', - 'field' => '_subsite', - 'fullfield' => '_subsite', - 'base' => $base, - 'origin' => $base, - 'type' => 'Int', - 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) - ); - } + public function alterQuery($query, $index) + { + $subsite = Subsite::currentSubsiteID(); + $query->filter('_subsite', array($subsite, SearchQuery::$missing)); + } - function alterQuery($query, $index) { - $subsite = Subsite::currentSubsiteID(); - $query->filter('_subsite', array($subsite, SearchQuery::$missing)); - } + public static $subsites = null; - static $subsites = null; + /** + * We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit + * deletes, just new versions with different members) so just always use all of them + */ + public function extractManipulationWriteState(&$writes) + { + $self = get_class($this); - /** - * We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit - * deletes, just new versions with different members) so just always use all of them - */ - function extractManipulationWriteState(&$writes) { - $self = get_class($this); + foreach ($writes as $key => $write) { + if (!$this->appliesTo($write['class'], true)) { + continue; + } - foreach ($writes as $key => $write) { - if (!$this->appliesTo($write['class'], true)) continue; + if (self::$subsites === null) { + $query = new SQLQuery('"ID"', '"Subsite"'); + self::$subsites = array_merge(array('0'), $query->execute()->column()); + } - if (self::$subsites === null) { - $query = new SQLQuery('"ID"', '"Subsite"'); - self::$subsites = array_merge(array('0'), $query->execute()->column()); - } + $next = array(); - $next = array(); - - foreach ($write['statefulids'] as $i => $statefulid) { - foreach (self::$subsites as $subsiteID) { - $next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => (string)$subsiteID))); - } - } - - $writes[$key]['statefulids'] = $next; - } - } + foreach ($write['statefulids'] as $i => $statefulid) { + foreach (self::$subsites as $subsiteID) { + $next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => (string)$subsiteID))); + } + } + $writes[$key]['statefulids'] = $next; + } + } } diff --git a/code/search/SearchVariantVersioned.php b/code/search/SearchVariantVersioned.php index e160f55..4e11af0 100644 --- a/code/search/SearchVariantVersioned.php +++ b/code/search/SearchVariantVersioned.php @@ -1,70 +1,84 @@ filterFields['_versionedstage'] = array( + 'name' => '_versionedstage', + 'field' => '_versionedstage', + 'fullfield' => '_versionedstage', + 'base' => $base, + 'origin' => $base, + 'type' => 'String', + 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) + ); + } - $index->filterFields['_versionedstage'] = array( - 'name' => '_versionedstage', - 'field' => '_versionedstage', - 'fullfield' => '_versionedstage', - 'base' => $base, - 'origin' => $base, - 'type' => 'String', - 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) - ); - } + public function alterQuery($query, $index) + { + $stage = Versioned::current_stage(); + $query->filter('_versionedstage', array($stage, SearchQuery::$missing)); + } + + public function extractManipulationState(&$manipulation) + { + $self = get_class($this); + + foreach ($manipulation as $table => $details) { + $class = $details['class']; + $stage = 'Stage'; - public function alterQuery($query, $index) { - $stage = Versioned::current_stage(); - $query->filter('_versionedstage', array($stage, SearchQuery::$missing)); - } - - function extractManipulationState(&$manipulation) { - $self = get_class($this); - - foreach ($manipulation as $table => $details) { - $class = $details['class']; - $stage = 'Stage'; + if (preg_match('/^(.*)_Live$/', $table, $matches)) { + $class = $matches[1]; + $stage = 'Live'; + } - if (preg_match('/^(.*)_Live$/', $table, $matches)) { - $class = $matches[1]; - $stage = 'Live'; - } + if (ClassInfo::exists($class) && $this->appliesTo($class, false)) { + $manipulation[$table]['class'] = $class; + $manipulation[$table]['state'][$self] = $stage; + } + } + } - if (ClassInfo::exists($class) && $this->appliesTo($class, false)) { - $manipulation[$table]['class'] = $class; - $manipulation[$table]['state'][$self] = $stage; - } - } - } - - function extractStates(&$table, &$ids, &$fields) { - $class = $table; - $suffix = null; + public function extractStates(&$table, &$ids, &$fields) + { + $class = $table; + $suffix = null; - if (ClassInfo::exists($class) && $this->appliesTo($class, false)) { - $table = $class; - $self = get_class($this); - - foreach ($ids as $i => $statefulid) { - $ids[$i]['state'][$self] = $suffix ? $suffix : 'Stage'; - } - } - } + if (ClassInfo::exists($class) && $this->appliesTo($class, false)) { + $table = $class; + $self = get_class($this); + foreach ($ids as $i => $statefulid) { + $ids[$i]['state'][$self] = $suffix ? $suffix : 'Stage'; + } + } + } } diff --git a/code/search/processors/SearchUpdateBatchedProcessor.php b/code/search/processors/SearchUpdateBatchedProcessor.php index 17845e3..b684939 100644 --- a/code/search/processors/SearchUpdateBatchedProcessor.php +++ b/code/search/processors/SearchUpdateBatchedProcessor.php @@ -3,155 +3,176 @@ /** * Provides batching of search updates */ -abstract class SearchUpdateBatchedProcessor extends SearchUpdateProcessor { - - /** - * List of batches to be processed - * - * @var array - */ - protected $batches; - - /** - * Pointer to index of $batches assigned to $current. - * Set to 0 (first index) if not started, or count + 1 if completed. - * - * @var int - */ - protected $currentBatch; - - /** - * List of indexes successfully comitted in the current batch - * - * @var array - */ - protected $completedIndexes; - - /** - * Maximum number of record-states to process in one batch. - * Set to zero to process all records in a single batch - * - * @config - * @var int - */ - private static $batch_size = 100; - - /** - * Up to this number of additional ids can be added to any batch in order to reduce the number - * of batches - * - * @config - * @var int - */ - private static $batch_soft_cap = 10; - - public function __construct() { - parent::__construct(); - - $this->batches = array(); - $this->setBatch(0); - } - - /** - * Set the current batch index - * - * @param int $batch Index of the batch - */ - protected function setBatch($batch) { - $this->currentBatch = $batch; - } - - protected function getSource() { - if(isset($this->batches[$this->currentBatch])) { - return $this->batches[$this->currentBatch]; - } - } - - /** - * Process the current queue - * - * @return boolean - */ - public function process() { - // Skip blank queues - if(empty($this->batches)) return true; - - // Don't re-process completed queue - if($this->currentBatch >= count($this->batches)) return true; - - // Send current patch to indexes - $this->prepareIndexes(); - - // Advance to next batch if successful - $this->setBatch($this->currentBatch + 1); - return true; - } - - /** - * Segments batches acording to the specified rules - * - * @param array $source Source input - * @return array Batches - */ - protected function segmentBatches($source) { - // Measure batch_size - $batchSize = Config::inst()->get(get_class(), 'batch_size'); - if($batchSize === 0) return array($source); - $softCap = Config::inst()->get(get_class(), 'batch_soft_cap'); - - // Clear batches - $batches = array(); - $current = array(); - $currentSize = 0; +abstract class SearchUpdateBatchedProcessor extends SearchUpdateProcessor +{ + /** + * List of batches to be processed + * + * @var array + */ + protected $batches; + + /** + * Pointer to index of $batches assigned to $current. + * Set to 0 (first index) if not started, or count + 1 if completed. + * + * @var int + */ + protected $currentBatch; + + /** + * List of indexes successfully comitted in the current batch + * + * @var array + */ + protected $completedIndexes; + + /** + * Maximum number of record-states to process in one batch. + * Set to zero to process all records in a single batch + * + * @config + * @var int + */ + private static $batch_size = 100; + + /** + * Up to this number of additional ids can be added to any batch in order to reduce the number + * of batches + * + * @config + * @var int + */ + private static $batch_soft_cap = 10; + + public function __construct() + { + parent::__construct(); + + $this->batches = array(); + $this->setBatch(0); + } + + /** + * Set the current batch index + * + * @param int $batch Index of the batch + */ + protected function setBatch($batch) + { + $this->currentBatch = $batch; + } + + protected function getSource() + { + if (isset($this->batches[$this->currentBatch])) { + return $this->batches[$this->currentBatch]; + } + } + + /** + * Process the current queue + * + * @return boolean + */ + public function process() + { + // Skip blank queues + if (empty($this->batches)) { + return true; + } + + // Don't re-process completed queue + if ($this->currentBatch >= count($this->batches)) { + return true; + } + + // Send current patch to indexes + $this->prepareIndexes(); + + // Advance to next batch if successful + $this->setBatch($this->currentBatch + 1); + return true; + } + + /** + * Segments batches acording to the specified rules + * + * @param array $source Source input + * @return array Batches + */ + protected function segmentBatches($source) + { + // Measure batch_size + $batchSize = Config::inst()->get(get_class(), 'batch_size'); + if ($batchSize === 0) { + return array($source); + } + $softCap = Config::inst()->get(get_class(), 'batch_soft_cap'); + + // Clear batches + $batches = array(); + $current = array(); + $currentSize = 0; - // Build batches from data - foreach ($source as $base => $statefulids) { - if (!$statefulids) continue; + // Build batches from data + foreach ($source as $base => $statefulids) { + if (!$statefulids) { + continue; + } - foreach ($statefulids as $stateKey => $statefulid) { - $state = $statefulid['state']; - $ids = $statefulid['ids']; - if(!$ids) continue; - - // Extract items from $ids until empty - while($ids) { - // Estimate maximum number of items to take for this iteration, allowing for the soft cap - $take = $batchSize - $currentSize; - if(count($ids) <= $take + $softCap) $take += $softCap; - $items = array_slice($ids, 0, $take, true); - $ids = array_slice($ids, count($items), null, true); - - // Update batch - $currentSize += count($items); - $merge = array( - $base => array( - $stateKey => array( - 'state' => $state, - 'ids' => $items - ) - ) - ); - $current = $current ? array_merge_recursive($current, $merge) : $merge; - if($currentSize >= $batchSize) { - $batches[] = $current; - $current = array(); - $currentSize = 0; - } - } - } - } - // Add incomplete batch - if($currentSize) $batches[] = $current; - - return $batches; - } - - public function batchData() { - $this->batches = $this->segmentBatches($this->dirty); - $this->setBatch(0); - } - - public function triggerProcessing() { - $this->batchData(); - } + foreach ($statefulids as $stateKey => $statefulid) { + $state = $statefulid['state']; + $ids = $statefulid['ids']; + if (!$ids) { + continue; + } + + // Extract items from $ids until empty + while ($ids) { + // Estimate maximum number of items to take for this iteration, allowing for the soft cap + $take = $batchSize - $currentSize; + if (count($ids) <= $take + $softCap) { + $take += $softCap; + } + $items = array_slice($ids, 0, $take, true); + $ids = array_slice($ids, count($items), null, true); + + // Update batch + $currentSize += count($items); + $merge = array( + $base => array( + $stateKey => array( + 'state' => $state, + 'ids' => $items + ) + ) + ); + $current = $current ? array_merge_recursive($current, $merge) : $merge; + if ($currentSize >= $batchSize) { + $batches[] = $current; + $current = array(); + $currentSize = 0; + } + } + } + } + // Add incomplete batch + if ($currentSize) { + $batches[] = $current; + } + + return $batches; + } + + public function batchData() + { + $this->batches = $this->segmentBatches($this->dirty); + $this->setBatch(0); + } + + public function triggerProcessing() + { + $this->batchData(); + } } diff --git a/code/search/processors/SearchUpdateCommitJobProcessor.php b/code/search/processors/SearchUpdateCommitJobProcessor.php index f14743c..35144a1 100644 --- a/code/search/processors/SearchUpdateCommitJobProcessor.php +++ b/code/search/processors/SearchUpdateCommitJobProcessor.php @@ -1,249 +1,266 @@ create(__CLASS__); - $id = singleton('QueuedJobService')->queueJob($commit, $startAfter); - - if($dirty) { - $indexes = FullTextSearch::get_indexes(); - static::$dirty_indexes = array_keys($indexes); - } - return $id; - } - - public function getJobType() { - return Config::inst()->get(__CLASS__, 'commit_queue'); - } - - public function getSignature() { - // There is only ever one commit job on the queue so the signature is consistent - // See QueuedJobService::queueJob() for the code that prevents duplication - return __CLASS__; - } - - public function getTitle() { - return "FullTextSearch Commit Job"; - } - - /** - * Get the list of index names we should process - * - * @return array - */ - public function getAllIndexes() { - if(empty($this->indexes)) { - $indexes = FullTextSearch::get_indexes(); - $this->indexes = array_keys($indexes); - } - return $this->indexes; - } - - public function jobFinished() { - // If we've indexed exactly as many as we would like, we are done - return $this->skipped - || (count($this->getAllIndexes()) <= count($this->completed)); - } - - public function prepareForRestart() { - // NOOP - } - - public function afterComplete() { - // NOOP - } - - /** - * Abort this job, potentially rescheduling a replacement if there is still work to do - */ - protected function discardJob() { - $this->skipped = true; - - // If we do not have dirty records, then assume that these dirty records were committed - // already this request (but probably another job), so we don't need to commit anything else. - // This could occur if we completed multiple searchupdate jobs in a prior request, and - // we only need one commit job to commit all of them in the current request. - if(empty(static::$dirty_indexes)) { - $this->addMessage("Indexing already completed this request: Discarding this job"); - return; - } - - - // If any commit has run, but some (or all) indexes are un-comitted, we must re-schedule this task. - // This could occur if we completed a searchupdate job in a prior request, as well as in - // the current request - $cooldown = Config::inst()->get(__CLASS__, 'cooldown'); - $now = new DateTime(SS_Datetime::now()->getValue()); - $now->add(new DateInterval('PT'.$cooldown.'S')); - $runat = $now->Format('Y-m-d H:i:s'); - - $this->addMessage("Indexing already run this request, but incomplete. Re-scheduling for {$runat}"); - - // Queue after the given cooldown - static::queue(false, $runat); - } - - public function process() { - // If we have already run an instance of SearchUpdateCommitJobProcessor this request, immediately - // quit this job to prevent hitting warming search limits in Solr - if(static::$has_run) { - $this->discardJob(); - return true; - } - - // To prevent other commit jobs from running this request - static::$has_run = true; - - // Run all incompleted indexes - $indexNames = $this->getAllIndexes(); - foreach ($indexNames as $name) { - $index = singleton($name); - $this->commitIndex($index); - } - - $this->addMessage("All indexes committed"); - - return true; - } - - /** - * Commits a specific index - * - * @param SolrIndex $index - * @throws Exception - */ - protected function commitIndex($index) { - // Skip index if this is already complete - $name = get_class($index); - if(in_array($name, $this->completed)) { - $this->addMessage("Skipping already comitted index {$name}"); - return; - } - - // Bypass SolrIndex::commit exception handling so that queuedjobs can handle the error - $this->addMessage("Committing index {$name}"); - $index->getService()->commit(false, false, false); - $this->addMessage("Committing index {$name} was successful"); - - // If this index is currently marked as dirty, it's now clean - if(in_array($name, static::$dirty_indexes)) { - static::$dirty_indexes = array_diff(static::$dirty_indexes, array($name)); - } - - // Mark complete - $this->completed[] = $name; - } - - public function setup() { - // NOOP - } - - public function getJobData() { - $data = new stdClass(); - $data->totalSteps = count($this->getAllIndexes()); - $data->currentStep = count($this->completed); - $data->isComplete = $this->jobFinished(); - $data->messages = $this->messages; - - $data->jobData = new stdClass(); - $data->jobData->skipped = $this->skipped; - $data->jobData->completed = $this->completed; - $data->jobData->indexes = $this->getAllIndexes(); - - return $data; - } - - public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { - $this->isComplete = $isComplete; - $this->messages = $messages; - - $this->skipped = $jobData->skipped; - $this->completed = $jobData->completed; - $this->indexes = $jobData->indexes; - } - - public function addMessage($message, $severity='INFO') { - $severity = strtoupper($severity); - $this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message"; - } - - public function getMessages() { - return $this->messages; - } - +if (!interface_exists('QueuedJob')) { + return; +} + +class SearchUpdateCommitJobProcessor implements QueuedJob +{ + /** + * The QueuedJob queue to use when processing commits + * + * @config + * @var int + */ + private static $commit_queue = 2; // QueuedJob::QUEUED; + + /** + * List of indexes to commit + * + * @var array + */ + protected $indexes = array(); + + /** + * True if this job is skipped to be be re-scheduled in the future + * + * @var boolean + */ + protected $skipped = false; + + /** + * List of completed indexes + * + * @var array + */ + protected $completed = array(); + + /** + * List of messages + * + * @var array + */ + protected $messages = array(); + + /** + * List of dirty indexes to be committed + * + * @var array + */ + public static $dirty_indexes = true; + + /** + * If solrindex::commit has already been performed, but additional commits are necessary, + * how long do we wait before attempting to touch the index again? + * + * {@see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers} + * + * @var int + * @config + */ + private static $cooldown = 300; + + /** + * True if any commits have been executed this request. If so, any attempts to run subsequent commits + * should be delayed until next queuedjob to prevent solr reaching maxWarmingSearchers + * + * {@see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers} + * + * @var boolean + */ + public static $has_run = false; + + /** + * This method is invoked once indexes with dirty ids have been updapted and a commit is necessary + * + * @param boolean $dirty Marks all indexes as dirty by default. Set to false if there are known comitted and + * clean indexes + * @param string $startAfter Start date + * @return int The ID of the next queuedjob to run. This could be a new one or an existing one. + */ + public static function queue($dirty = true, $startAfter = null) + { + $commit = Injector::inst()->create(__CLASS__); + $id = singleton('QueuedJobService')->queueJob($commit, $startAfter); + + if ($dirty) { + $indexes = FullTextSearch::get_indexes(); + static::$dirty_indexes = array_keys($indexes); + } + return $id; + } + + public function getJobType() + { + return Config::inst()->get(__CLASS__, 'commit_queue'); + } + + public function getSignature() + { + // There is only ever one commit job on the queue so the signature is consistent + // See QueuedJobService::queueJob() for the code that prevents duplication + return __CLASS__; + } + + public function getTitle() + { + return "FullTextSearch Commit Job"; + } + + /** + * Get the list of index names we should process + * + * @return array + */ + public function getAllIndexes() + { + if (empty($this->indexes)) { + $indexes = FullTextSearch::get_indexes(); + $this->indexes = array_keys($indexes); + } + return $this->indexes; + } + + public function jobFinished() + { + // If we've indexed exactly as many as we would like, we are done + return $this->skipped + || (count($this->getAllIndexes()) <= count($this->completed)); + } + + public function prepareForRestart() + { + // NOOP + } + + public function afterComplete() + { + // NOOP + } + + /** + * Abort this job, potentially rescheduling a replacement if there is still work to do + */ + protected function discardJob() + { + $this->skipped = true; + + // If we do not have dirty records, then assume that these dirty records were committed + // already this request (but probably another job), so we don't need to commit anything else. + // This could occur if we completed multiple searchupdate jobs in a prior request, and + // we only need one commit job to commit all of them in the current request. + if (empty(static::$dirty_indexes)) { + $this->addMessage("Indexing already completed this request: Discarding this job"); + return; + } + + + // If any commit has run, but some (or all) indexes are un-comitted, we must re-schedule this task. + // This could occur if we completed a searchupdate job in a prior request, as well as in + // the current request + $cooldown = Config::inst()->get(__CLASS__, 'cooldown'); + $now = new DateTime(SS_Datetime::now()->getValue()); + $now->add(new DateInterval('PT'.$cooldown.'S')); + $runat = $now->Format('Y-m-d H:i:s'); + + $this->addMessage("Indexing already run this request, but incomplete. Re-scheduling for {$runat}"); + + // Queue after the given cooldown + static::queue(false, $runat); + } + + public function process() + { + // If we have already run an instance of SearchUpdateCommitJobProcessor this request, immediately + // quit this job to prevent hitting warming search limits in Solr + if (static::$has_run) { + $this->discardJob(); + return true; + } + + // To prevent other commit jobs from running this request + static::$has_run = true; + + // Run all incompleted indexes + $indexNames = $this->getAllIndexes(); + foreach ($indexNames as $name) { + $index = singleton($name); + $this->commitIndex($index); + } + + $this->addMessage("All indexes committed"); + + return true; + } + + /** + * Commits a specific index + * + * @param SolrIndex $index + * @throws Exception + */ + protected function commitIndex($index) + { + // Skip index if this is already complete + $name = get_class($index); + if (in_array($name, $this->completed)) { + $this->addMessage("Skipping already comitted index {$name}"); + return; + } + + // Bypass SolrIndex::commit exception handling so that queuedjobs can handle the error + $this->addMessage("Committing index {$name}"); + $index->getService()->commit(false, false, false); + $this->addMessage("Committing index {$name} was successful"); + + // If this index is currently marked as dirty, it's now clean + if (in_array($name, static::$dirty_indexes)) { + static::$dirty_indexes = array_diff(static::$dirty_indexes, array($name)); + } + + // Mark complete + $this->completed[] = $name; + } + + public function setup() + { + // NOOP + } + + public function getJobData() + { + $data = new stdClass(); + $data->totalSteps = count($this->getAllIndexes()); + $data->currentStep = count($this->completed); + $data->isComplete = $this->jobFinished(); + $data->messages = $this->messages; + + $data->jobData = new stdClass(); + $data->jobData->skipped = $this->skipped; + $data->jobData->completed = $this->completed; + $data->jobData->indexes = $this->getAllIndexes(); + + return $data; + } + + public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) + { + $this->isComplete = $isComplete; + $this->messages = $messages; + + $this->skipped = $jobData->skipped; + $this->completed = $jobData->completed; + $this->indexes = $jobData->indexes; + } + + public function addMessage($message, $severity='INFO') + { + $severity = strtoupper($severity); + $this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message"; + } + + public function getMessages() + { + return $this->messages; + } } diff --git a/code/search/processors/SearchUpdateImmediateProcessor.php b/code/search/processors/SearchUpdateImmediateProcessor.php index 2f80666..4ae9dae 100644 --- a/code/search/processors/SearchUpdateImmediateProcessor.php +++ b/code/search/processors/SearchUpdateImmediateProcessor.php @@ -1,7 +1,9 @@ process(); - } +class SearchUpdateImmediateProcessor extends SearchUpdateProcessor +{ + public function triggerProcessing() + { + $this->process(); + } } diff --git a/code/search/processors/SearchUpdateMessageQueueProcessor.php b/code/search/processors/SearchUpdateMessageQueueProcessor.php index 0538826..cbb5028 100644 --- a/code/search/processors/SearchUpdateMessageQueueProcessor.php +++ b/code/search/processors/SearchUpdateMessageQueueProcessor.php @@ -1,17 +1,19 @@ get('SearchMessageQueueUpdater', 'reindex_queue'), - new MethodInvocationMessage($this, "process") - ); - } + public function triggerProcessing() + { + MessageQueue::send( + Config::inst()->get('SearchMessageQueueUpdater', 'reindex_queue'), + new MethodInvocationMessage($this, "process") + ); + } } diff --git a/code/search/processors/SearchUpdateProcessor.php b/code/search/processors/SearchUpdateProcessor.php index 71c40a3..579a60c 100644 --- a/code/search/processors/SearchUpdateProcessor.php +++ b/code/search/processors/SearchUpdateProcessor.php @@ -1,137 +1,146 @@ array( - * '$State Key' => array( - * 'state' => array( - * 'key1' => 'value', - * 'key2' => 'value' - * ), - * 'ids' => array( - * '*id*' => array( - * '*Index Name 1*', - * '*Index Name 2*' - * ) - * ) - * ) - * ) - * ) - * - * @var array - */ - protected $dirty; - - public function __construct() { - $this->dirty = array(); - } +abstract class SearchUpdateProcessor +{ + /** + * List of dirty records to process in format + * + * array( + * '$BaseClass' => array( + * '$State Key' => array( + * 'state' => array( + * 'key1' => 'value', + * 'key2' => 'value' + * ), + * 'ids' => array( + * '*id*' => array( + * '*Index Name 1*', + * '*Index Name 2*' + * ) + * ) + * ) + * ) + * ) + * + * @var array + */ + protected $dirty; + + public function __construct() + { + $this->dirty = array(); + } - public function addDirtyIDs($class, $statefulids, $index) { - $base = ClassInfo::baseDataClass($class); - $forclass = isset($this->dirty[$base]) ? $this->dirty[$base] : array(); + public function addDirtyIDs($class, $statefulids, $index) + { + $base = ClassInfo::baseDataClass($class); + $forclass = isset($this->dirty[$base]) ? $this->dirty[$base] : array(); - foreach ($statefulids as $statefulid) { - $id = $statefulid['id']; - $state = $statefulid['state']; $statekey = serialize($state); + foreach ($statefulids as $statefulid) { + $id = $statefulid['id']; + $state = $statefulid['state']; + $statekey = serialize($state); - if (!isset($forclass[$statekey])) { - $forclass[$statekey] = array('state' => $state, 'ids' => array($id => array($index))); - } - else if (!isset($forclass[$statekey]['ids'][$id])) { - $forclass[$statekey]['ids'][$id] = array($index); - } - else if (array_search($index, $forclass[$statekey]['ids'][$id]) === false) { - $forclass[$statekey]['ids'][$id][] = $index; - // dirty count stays the same - } - } + if (!isset($forclass[$statekey])) { + $forclass[$statekey] = array('state' => $state, 'ids' => array($id => array($index))); + } elseif (!isset($forclass[$statekey]['ids'][$id])) { + $forclass[$statekey]['ids'][$id] = array($index); + } elseif (array_search($index, $forclass[$statekey]['ids'][$id]) === false) { + $forclass[$statekey]['ids'][$id][] = $index; + // dirty count stays the same + } + } - $this->dirty[$base] = $forclass; - } - - /** - * Generates the list of indexes to process for the dirty items - * - * @return array - */ - protected function prepareIndexes() { - $originalState = SearchVariant::current_state(); - $dirtyIndexes = array(); - $dirty = $this->getSource(); - $indexes = FullTextSearch::get_indexes(); - foreach ($dirty as $base => $statefulids) { - if (!$statefulids) continue; + $this->dirty[$base] = $forclass; + } + + /** + * Generates the list of indexes to process for the dirty items + * + * @return array + */ + protected function prepareIndexes() + { + $originalState = SearchVariant::current_state(); + $dirtyIndexes = array(); + $dirty = $this->getSource(); + $indexes = FullTextSearch::get_indexes(); + foreach ($dirty as $base => $statefulids) { + if (!$statefulids) { + continue; + } - foreach ($statefulids as $statefulid) { - $state = $statefulid['state']; - $ids = $statefulid['ids']; + foreach ($statefulids as $statefulid) { + $state = $statefulid['state']; + $ids = $statefulid['ids']; - SearchVariant::activate_state($state); + SearchVariant::activate_state($state); - // Ensure that indexes for all new / updated objects are included - $objs = DataObject::get($base)->byIDs(array_keys($ids)); - foreach ($objs as $obj) { - foreach ($ids[$obj->ID] as $index) { - if (!$indexes[$index]->variantStateExcluded($state)) { - $indexes[$index]->add($obj); - $dirtyIndexes[$index] = $indexes[$index]; - } - } - unset($ids[$obj->ID]); - } + // Ensure that indexes for all new / updated objects are included + $objs = DataObject::get($base)->byIDs(array_keys($ids)); + foreach ($objs as $obj) { + foreach ($ids[$obj->ID] as $index) { + if (!$indexes[$index]->variantStateExcluded($state)) { + $indexes[$index]->add($obj); + $dirtyIndexes[$index] = $indexes[$index]; + } + } + unset($ids[$obj->ID]); + } - // Generate list of records that do not exist and should be removed - foreach ($ids as $id => $fromindexes) { - foreach ($fromindexes as $index) { - if (!$indexes[$index]->variantStateExcluded($state)) { - $indexes[$index]->delete($base, $id, $state); - $dirtyIndexes[$index] = $indexes[$index]; - } - } - } - } - } - - SearchVariant::activate_state($originalState); - return $dirtyIndexes; - } - - /** - * Commits the specified index to the Solr service - * - * @param SolrIndex $index Index object - * @return bool Flag indicating success - */ - protected function commitIndex($index) { - return $index->commit() !== false; - } - - /** - * Gets the record data source to process - * - * @return array - */ - protected function getSource() { - return $this->dirty; - } + // Generate list of records that do not exist and should be removed + foreach ($ids as $id => $fromindexes) { + foreach ($fromindexes as $index) { + if (!$indexes[$index]->variantStateExcluded($state)) { + $indexes[$index]->delete($base, $id, $state); + $dirtyIndexes[$index] = $indexes[$index]; + } + } + } + } + } + + SearchVariant::activate_state($originalState); + return $dirtyIndexes; + } + + /** + * Commits the specified index to the Solr service + * + * @param SolrIndex $index Index object + * @return bool Flag indicating success + */ + protected function commitIndex($index) + { + return $index->commit() !== false; + } + + /** + * Gets the record data source to process + * + * @return array + */ + protected function getSource() + { + return $this->dirty; + } - /** - * Process all indexes, returning true if successful - * - * @return bool Flag indicating success - */ - public function process() { - // Generate and commit all instances - $indexes = $this->prepareIndexes(); - foreach ($indexes as $index) { - if(!$this->commitIndex($index)) return false; - } - return true; - } + /** + * Process all indexes, returning true if successful + * + * @return bool Flag indicating success + */ + public function process() + { + // Generate and commit all instances + $indexes = $this->prepareIndexes(); + foreach ($indexes as $index) { + if (!$this->commitIndex($index)) { + return false; + } + } + return true; + } - abstract public function triggerProcessing(); + abstract public function triggerProcessing(); } diff --git a/code/search/processors/SearchUpdateQueuedJobProcessor.php b/code/search/processors/SearchUpdateQueuedJobProcessor.php index 4214061..4ce211d 100644 --- a/code/search/processors/SearchUpdateQueuedJobProcessor.php +++ b/code/search/processors/SearchUpdateQueuedJobProcessor.php @@ -1,87 +1,101 @@ queueJob($this); - } - - public function getTitle() { - return "FullTextSearch Update Job"; - } - - public function getSignature() { - return md5(get_class($this) . time() . mt_rand(0, 100000)); - } - - public function getJobType() { - return Config::inst()->get('SearchUpdateQueuedJobProcessor', 'reindex_queue'); - } - - public function jobFinished() { - return $this->currentBatch >= count($this->batches); - } - - public function setup() { - // NOP - } - - public function prepareForRestart() { - // NOP - } - - public function afterComplete() { - // Once indexing is complete, commit later in order to avoid solr limits - // see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers - SearchUpdateCommitJobProcessor::queue(); - } - - public function getJobData() { - $data = new stdClass(); - $data->totalSteps = count($this->batches); - $data->currentStep = $this->currentBatch; - $data->isComplete = $this->jobFinished(); - $data->messages = $this->messages; - - $data->jobData = new stdClass(); - $data->jobData->batches = $this->batches; - $data->jobData->currentBatch = $this->currentBatch; - - return $data; - } - - public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { - $this->isComplete = $isComplete; - $this->messages = $messages; - - $this->batches = $jobData->batches; - $this->currentBatch = $jobData->currentBatch; - } - - public function addMessage($message, $severity='INFO') { - $severity = strtoupper($severity); - $this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message"; - } - - public function process() { - $result = parent::process(); - - if($this->jobFinished()) { - $this->addMessage("All batched updates complete. Queuing commit job"); - } - - return $result; - } +if (!interface_exists('QueuedJob')) { + return; +} + +class SearchUpdateQueuedJobProcessor extends SearchUpdateBatchedProcessor implements QueuedJob +{ + /** + * The QueuedJob queue to use when processing updates + * @config + * @var int + */ + private static $reindex_queue = 2; // QueuedJob::QUEUED; + + protected $messages = array(); + + public function triggerProcessing() + { + parent::triggerProcessing(); + singleton('QueuedJobService')->queueJob($this); + } + + public function getTitle() + { + return "FullTextSearch Update Job"; + } + + public function getSignature() + { + return md5(get_class($this) . time() . mt_rand(0, 100000)); + } + + public function getJobType() + { + return Config::inst()->get('SearchUpdateQueuedJobProcessor', 'reindex_queue'); + } + + public function jobFinished() + { + return $this->currentBatch >= count($this->batches); + } + + public function setup() + { + // NOP + } + + public function prepareForRestart() + { + // NOP + } + + public function afterComplete() + { + // Once indexing is complete, commit later in order to avoid solr limits + // see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers + SearchUpdateCommitJobProcessor::queue(); + } + + public function getJobData() + { + $data = new stdClass(); + $data->totalSteps = count($this->batches); + $data->currentStep = $this->currentBatch; + $data->isComplete = $this->jobFinished(); + $data->messages = $this->messages; + + $data->jobData = new stdClass(); + $data->jobData->batches = $this->batches; + $data->jobData->currentBatch = $this->currentBatch; + + return $data; + } + + public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) + { + $this->isComplete = $isComplete; + $this->messages = $messages; + + $this->batches = $jobData->batches; + $this->currentBatch = $jobData->currentBatch; + } + + public function addMessage($message, $severity='INFO') + { + $severity = strtoupper($severity); + $this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message"; + } + + public function process() + { + $result = parent::process(); + + if ($this->jobFinished()) { + $this->addMessage("All batched updates complete. Queuing commit job"); + } + + return $result; + } } diff --git a/code/solr/Solr.php b/code/solr/Solr.php index 8363e42..249a513 100644 --- a/code/solr/Solr.php +++ b/code/solr/Solr.php @@ -5,284 +5,296 @@ use Monolog\Handler\StreamHandler; use Monolog\Logger; use Psr\Log\LoggerInterface; -class Solr { +class Solr +{ + /** + * Configuration on where to find the solr server and how to get new index configurations into it. + * + * Required fields: + * host (default: localhost) - The host or IP Solr is listening on + * port (default: 8983) - The port Solr is listening on + * path (default: /solr) - The suburl the solr service is available on + * + * Optional fields: + * version (default: 4) - The Solr server version. Currently supports 3 and 4 (you can add a sub-version like 4.5 if + * you like, but currently it has no effect) + * service (default: depends on version, Solr3Service for 3, Solr4Service for 4) + * the class that provides actual communcation to the Solr server + * extraspath (default: /fulltextsearch/conf/solr/{version}/extras/) - Absolute path to + * the folder containing templates which are used for generating the schema and field definitions. + * templates (default: /fulltextsearch/conf/solr/{version}/templates/) - Absolute path to + * the configuration default files, e.g. solrconfig.xml. + * + * indexstore => an array with + * + * mode - a classname which implements SolrConfigStore, or 'file' or 'webdav' + * + * When mode == SolrConfigStore_File or file (indexes should be written on a local filesystem) + * path - The (locally accessible) path to write the index configurations to. + * remotepath (default: the same as indexpath) - The path that the Solr server will read the index configurations from + * + * When mode == SolrConfigStore_WebDAV or webdav (indexes should stored on a remote Solr server via webdav) + * auth (default: none) - A username:password pair string to use to auth against the webdav server + * path (default: /solrindex) - The suburl on the solr host that is set up to accept index configurations via webdav + * remotepath - The path that the Solr server will read the index configurations from + */ + protected static $solr_options = array(); - /** - * Configuration on where to find the solr server and how to get new index configurations into it. - * - * Required fields: - * host (default: localhost) - The host or IP Solr is listening on - * port (default: 8983) - The port Solr is listening on - * path (default: /solr) - The suburl the solr service is available on - * - * Optional fields: - * version (default: 4) - The Solr server version. Currently supports 3 and 4 (you can add a sub-version like 4.5 if - * you like, but currently it has no effect) - * service (default: depends on version, Solr3Service for 3, Solr4Service for 4) - * the class that provides actual communcation to the Solr server - * extraspath (default: /fulltextsearch/conf/solr/{version}/extras/) - Absolute path to - * the folder containing templates which are used for generating the schema and field definitions. - * templates (default: /fulltextsearch/conf/solr/{version}/templates/) - Absolute path to - * the configuration default files, e.g. solrconfig.xml. - * - * indexstore => an array with - * - * mode - a classname which implements SolrConfigStore, or 'file' or 'webdav' - * - * When mode == SolrConfigStore_File or file (indexes should be written on a local filesystem) - * path - The (locally accessible) path to write the index configurations to. - * remotepath (default: the same as indexpath) - The path that the Solr server will read the index configurations from - * - * When mode == SolrConfigStore_WebDAV or webdav (indexes should stored on a remote Solr server via webdav) - * auth (default: none) - A username:password pair string to use to auth against the webdav server - * path (default: /solrindex) - The suburl on the solr host that is set up to accept index configurations via webdav - * remotepath - The path that the Solr server will read the index configurations from - */ - protected static $solr_options = array(); + /** A cache of solr_options with the defaults all merged in */ + protected static $merged_solr_options = null; - /** A cache of solr_options with the defaults all merged in */ - protected static $merged_solr_options = null; + /** + * Update the configuration for Solr. See $solr_options for a discussion of the accepted array keys + * @param array $options - The options to update + */ + public static function configure_server($options = array()) + { + self::$solr_options = array_merge(self::$solr_options, $options); + self::$merged_solr_options = null; - /** - * Update the configuration for Solr. See $solr_options for a discussion of the accepted array keys - * @param array $options - The options to update - */ - static function configure_server($options = array()) { - self::$solr_options = array_merge(self::$solr_options, $options); - self::$merged_solr_options = null; + self::$service_singleton = null; + self::$service_core_singletons = array(); + } - self::$service_singleton = null; - self::$service_core_singletons = array(); - } + /** + * Get the configured Solr options with the defaults all merged in + * @return array - The merged options + */ + public static function solr_options() + { + if (self::$merged_solr_options) { + return self::$merged_solr_options; + } - /** - * Get the configured Solr options with the defaults all merged in - * @return array - The merged options - */ - static function solr_options() { - if (self::$merged_solr_options) return self::$merged_solr_options; + $defaults = array( + 'host' => 'localhost', + 'port' => 8983, + 'path' => '/solr', + 'version' => '4' + ); - $defaults = array( - 'host' => 'localhost', - 'port' => 8983, - 'path' => '/solr', - 'version' => '4' - ); + // Build some by-version defaults + $version = isset(self::$solr_options['version']) ? self::$solr_options['version'] : $defaults['version']; - // Build some by-version defaults - $version = isset(self::$solr_options['version']) ? self::$solr_options['version'] : $defaults['version']; + if (version_compare($version, '4', '>=')) { + $versionDefaults = array( + 'service' => 'Solr4Service', + 'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/extras/', + 'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/templates/', + ); + } else { + $versionDefaults = array( + 'service' => 'Solr3Service', + 'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/extras/', + 'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/templates/', + ); + } - if (version_compare($version, '4', '>=')){ - $versionDefaults = array( - 'service' => 'Solr4Service', - 'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/extras/', - 'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/templates/', - ); - } - else { - $versionDefaults = array( - 'service' => 'Solr3Service', - 'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/extras/', - 'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/templates/', - ); - } - - return (self::$merged_solr_options = array_merge($defaults, $versionDefaults, self::$solr_options)); - } + return (self::$merged_solr_options = array_merge($defaults, $versionDefaults, self::$solr_options)); + } - static function set_service_class($class) { - user_error('set_service_class is deprecated - pass as part of $options to configure_server', E_USER_WARNING); - self::configure_server(array('service' => $class)); - } + public static function set_service_class($class) + { + user_error('set_service_class is deprecated - pass as part of $options to configure_server', E_USER_WARNING); + self::configure_server(array('service' => $class)); + } - /** @var SolrService | null - The instance of SolrService for core management */ - static protected $service_singleton = null; - /** @var [SolrService_Core] - The instances of SolrService_Core for each core */ - static protected $service_core_singletons = array(); + /** @var SolrService | null - The instance of SolrService for core management */ + protected static $service_singleton = null; + /** @var [SolrService_Core] - The instances of SolrService_Core for each core */ + protected static $service_core_singletons = array(); - /** - * Get a SolrService - * - * @param string $core Optional core name - * @return SolrService_Core - */ - static function service($core = null) { - $options = self::solr_options(); + /** + * Get a SolrService + * + * @param string $core Optional core name + * @return SolrService_Core + */ + public static function service($core = null) + { + $options = self::solr_options(); - if (!self::$service_singleton) { - self::$service_singleton = Object::create( - $options['service'], $options['host'], $options['port'], $options['path'] - ); - } + if (!self::$service_singleton) { + self::$service_singleton = Object::create( + $options['service'], $options['host'], $options['port'], $options['path'] + ); + } - if ($core) { - if (!isset(self::$service_core_singletons[$core])) { - self::$service_core_singletons[$core] = self::$service_singleton->serviceForCore( - singleton($core)->getIndexName() - ); - } + if ($core) { + if (!isset(self::$service_core_singletons[$core])) { + self::$service_core_singletons[$core] = self::$service_singleton->serviceForCore( + singleton($core)->getIndexName() + ); + } - return self::$service_core_singletons[$core]; - } else { - return self::$service_singleton; - } - } + return self::$service_core_singletons[$core]; + } else { + return self::$service_singleton; + } + } - static function get_indexes() { - return FullTextSearch::get_indexes('SolrIndex'); - } + public static function get_indexes() + { + return FullTextSearch::get_indexes('SolrIndex'); + } - /** - * Include the thirdparty Solr client api library. Done this way to avoid issues where code is called in - * mysite/_config before fulltextsearch/_config has a change to update the include path. - */ - static function include_client_api() { - static $included = false; + /** + * Include the thirdparty Solr client api library. Done this way to avoid issues where code is called in + * mysite/_config before fulltextsearch/_config has a change to update the include path. + */ + public static function include_client_api() + { + static $included = false; - if (!$included) { - set_include_path(get_include_path() . PATH_SEPARATOR . Director::baseFolder() . '/fulltextsearch/thirdparty/solr-php-client'); - require_once('Apache/Solr/Service.php'); - require_once('Apache/Solr/Document.php'); + if (!$included) { + set_include_path(get_include_path() . PATH_SEPARATOR . Director::baseFolder() . '/fulltextsearch/thirdparty/solr-php-client'); + require_once('Apache/Solr/Service.php'); + require_once('Apache/Solr/Document.php'); - $included = true; - } - } + $included = true; + } + } } /** * Abstract class for build tasks */ -class Solr_BuildTask extends BuildTask { +class Solr_BuildTask extends BuildTask +{ + protected $enabled = false; - protected $enabled = false; + /** + * Logger + * + * @var LoggerInterface + */ + protected $logger = null; - /** - * Logger - * - * @var LoggerInterface - */ - protected $logger = null; + /** + * Get the current logger + * + * @return LoggerInterface + */ + public function getLogger() + { + return $this->logger; + } - /** - * Get the current logger - * - * @return LoggerInterface - */ - public function getLogger() { - return $this->logger; - } + /** + * Assign a new logger + * + * @param LoggerInterface $logger + */ + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } - /** - * Assign a new logger - * - * @param LoggerInterface $logger - */ - public function setLogger(LoggerInterface $logger) { - $this->logger = $logger; - } + /** + * @return SearchLogFactory + */ + protected function getLoggerFactory() + { + return Injector::inst()->get('SearchLogFactory'); + } - /** - * @return SearchLogFactory - */ - protected function getLoggerFactory() { - return Injector::inst()->get('SearchLogFactory'); - } + /** + * Setup task + * + * @param SS_HTTPReqest $request + */ + public function run($request) + { + $name = get_class($this); + $verbose = $request->getVar('verbose'); - /** - * Setup task - * - * @param SS_HTTPReqest $request - */ - public function run($request) { - $name = get_class($this); - $verbose = $request->getVar('verbose'); - - // Set new logger - $logger = $this - ->getLoggerFactory() - ->getOutputLogger($name, $verbose); - $this->setLogger($logger); - } + // Set new logger + $logger = $this + ->getLoggerFactory() + ->getOutputLogger($name, $verbose); + $this->setLogger($logger); + } } -class Solr_Configure extends Solr_BuildTask { +class Solr_Configure extends Solr_BuildTask +{ + protected $enabled = true; - protected $enabled = true; + public function run($request) + { + parent::run($request); + + // Find the IndexStore handler, which will handle uploading config files to Solr + $store = $this->getSolrConfigStore(); + $indexes = Solr::get_indexes(); + foreach ($indexes as $instance) { + try { + $this->updateIndex($instance, $store); + } catch (Exception $e) { + // We got an exception. Warn, but continue to next index. + $this + ->getLogger() + ->error("Failure: " . $e->getMessage()); + } + } + } + + /** + * Update the index on the given store + * + * @param SolrIndex $instance Instance + * @param SolrConfigStore $store + */ + protected function updateIndex($instance, $store) + { + $index = $instance->getIndexName(); + $this->getLogger()->info("Configuring $index."); + + // Upload the config files for this index + $this->getLogger()->info("Uploading configuration ..."); + $instance->uploadConfig($store); + + // Then tell Solr to use those config files + $service = Solr::service(); + if ($service->coreIsActive($index)) { + $this->getLogger()->info("Reloading core ..."); + $service->coreReload($index); + } else { + $this->getLogger()->info("Creating core ..."); + $service->coreCreate($index, $store->instanceDir($index)); + } + + $this->getLogger()->info("Done"); + } - public function run($request) { - parent::run($request); - - // Find the IndexStore handler, which will handle uploading config files to Solr - $store = $this->getSolrConfigStore(); - $indexes = Solr::get_indexes(); - foreach ($indexes as $instance) { + /** + * Get config store + * + * @return SolrConfigStore + */ + protected function getSolrConfigStore() + { + $options = Solr::solr_options(); + + if (!isset($options['indexstore']) || !($indexstore = $options['indexstore'])) { + user_error('No index configuration for Solr provided', E_USER_ERROR); + } + + // Find the IndexStore handler, which will handle uploading config files to Solr + $mode = $indexstore['mode']; - try { - $this->updateIndex($instance, $store); - } catch(Exception $e) { - // We got an exception. Warn, but continue to next index. - $this - ->getLogger() - ->error("Failure: " . $e->getMessage()); - } - } - } - - /** - * Update the index on the given store - * - * @param SolrIndex $instance Instance - * @param SolrConfigStore $store - */ - protected function updateIndex($instance, $store) { - $index = $instance->getIndexName(); - $this->getLogger()->info("Configuring $index."); - - // Upload the config files for this index - $this->getLogger()->info("Uploading configuration ..."); - $instance->uploadConfig($store); - - // Then tell Solr to use those config files - $service = Solr::service(); - if ($service->coreIsActive($index)) { - $this->getLogger()->info("Reloading core ..."); - $service->coreReload($index); - } else { - $this->getLogger()->info("Creating core ..."); - $service->coreCreate($index, $store->instanceDir($index)); - } - - $this->getLogger()->info("Done"); - } - - /** - * Get config store - * - * @return SolrConfigStore - */ - protected function getSolrConfigStore() { - $options = Solr::solr_options(); - - if (!isset($options['indexstore']) || !($indexstore = $options['indexstore'])) { - user_error('No index configuration for Solr provided', E_USER_ERROR); - } - - // Find the IndexStore handler, which will handle uploading config files to Solr - $mode = $indexstore['mode']; - - if ($mode == 'file') { - return new SolrConfigStore_File($indexstore); - } elseif ($mode == 'webdav') { - return new SolrConfigStore_WebDAV($indexstore); - } elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, 'SolrConfigStore')) { - return new $mode($indexstore); - } else { - user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR); - } - - } + if ($mode == 'file') { + return new SolrConfigStore_File($indexstore); + } elseif ($mode == 'webdav') { + return new SolrConfigStore_WebDAV($indexstore); + } elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, 'SolrConfigStore')) { + return new $mode($indexstore); + } else { + user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR); + } + } } /** @@ -300,106 +312,110 @@ class Solr_Configure extends Solr_BuildTask { * - variantstate * - verbose (optional) */ -class Solr_Reindex extends Solr_BuildTask { +class Solr_Reindex extends Solr_BuildTask +{ + protected $enabled = true; - protected $enabled = true; + /** + * Number of records to load and index per request + * + * @var int + * @config + */ + private static $recordsPerRequest = 200; - /** - * Number of records to load and index per request - * - * @var int - * @config - */ - private static $recordsPerRequest = 200; + /** + * Get the reindex handler + * + * @return SolrReindexHandler + */ + protected function getHandler() + { + return Injector::inst()->get('SolrReindexHandler'); + } - /** - * Get the reindex handler - * - * @return SolrReindexHandler - */ - protected function getHandler() { - return Injector::inst()->get('SolrReindexHandler'); - } + /** + * @param SS_HTTPRequest $request + */ + public function run($request) + { + parent::run($request); + + // Reset state + $originalState = SearchVariant::current_state(); + $this->doReindex($request); + SearchVariant::activate_state($originalState); + } - /** - * @param SS_HTTPRequest $request - */ - public function run($request) { - parent::run($request); - - // Reset state - $originalState = SearchVariant::current_state(); - $this->doReindex($request); - SearchVariant::activate_state($originalState); - } + /** + * @param SS_HTTPRequest $request + */ + protected function doReindex($request) + { + $class = $request->getVar('class'); - /** - * @param SS_HTTPRequest $request - */ - protected function doReindex($request) { - $class = $request->getVar('class'); + // Deprecated reindex mechanism + $start = $request->getVar('start'); + if ($start !== null) { + // Run single batch directly + $indexInstance = singleton($request->getVar('index')); + $state = json_decode($request->getVar('variantstate'), true); + $this->runFrom($indexInstance, $class, $start, $state); + return; + } - // Deprecated reindex mechanism - $start = $request->getVar('start'); - if ($start !== null) { - // Run single batch directly - $indexInstance = singleton($request->getVar('index')); - $state = json_decode($request->getVar('variantstate'), true); - $this->runFrom($indexInstance, $class, $start, $state); - return; - } + // Check if we are re-indexing a single group + // If not using queuedjobs, we need to invoke Solr_Reindex as a separate process + // Otherwise each group is processed via a SolrReindexGroupJob + $groups = $request->getVar('groups'); + $handler = $this->getHandler(); + if ($groups) { + // Run grouped batches (id % groups = group) + $group = $request->getVar('group'); + $indexInstance = singleton($request->getVar('index')); + $state = json_decode($request->getVar('variantstate'), true); - // Check if we are re-indexing a single group - // If not using queuedjobs, we need to invoke Solr_Reindex as a separate process - // Otherwise each group is processed via a SolrReindexGroupJob - $groups = $request->getVar('groups'); - $handler = $this->getHandler(); - if($groups) { - // Run grouped batches (id % groups = group) - $group = $request->getVar('group'); - $indexInstance = singleton($request->getVar('index')); - $state = json_decode($request->getVar('variantstate'), true); + $handler->runGroup($this->getLogger(), $indexInstance, $state, $class, $groups, $group); + return; + } - $handler->runGroup($this->getLogger(), $indexInstance, $state, $class, $groups, $group); - return; - } + // If run at the top level, delegate to appropriate handler + $self = get_class($this); + $handler->triggerReindex($this->getLogger(), $this->config()->recordsPerRequest, $self, $class); + } - // If run at the top level, delegate to appropriate handler - $self = get_class($this); - $handler->triggerReindex($this->getLogger(), $this->config()->recordsPerRequest, $self, $class); - } + /** + * @deprecated since version 2.0.0 + */ + protected function runFrom($index, $class, $start, $variantstate) + { + DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism'); + + // Set time limit and state + increase_time_limit_to(); + SearchVariant::activate_state($variantstate); - /** - * @deprecated since version 2.0.0 - */ - protected function runFrom($index, $class, $start, $variantstate) { - DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism'); - - // Set time limit and state - increase_time_limit_to(); - SearchVariant::activate_state($variantstate); + // Generate filtered list + $items = DataList::create($class) + ->limit($this->config()->recordsPerRequest, $start); - // Generate filtered list - $items = DataList::create($class) - ->limit($this->config()->recordsPerRequest, $start); + // Add child filter + $classes = $index->getClasses(); + $options = $classes[$class]; + if (!$options['include_children']) { + $items = $items->filter('ClassName', $class); + } - // Add child filter - $classes = $index->getClasses(); - $options = $classes[$class]; - if(!$options['include_children']) { - $items = $items->filter('ClassName', $class); - } + // Process selected records in this class + $this->getLogger()->info("Adding $class"); + foreach ($items->sort("ID") as $item) { + $this->getLogger()->debug($item->ID); - // Process selected records in this class - $this->getLogger()->info("Adding $class"); - foreach ($items->sort("ID") as $item) { - $this->getLogger()->debug($item->ID); + // See SearchUpdater_ObjectHandler::triggerReindex + $item->triggerReindex(); + $item->destroy(); + } - // See SearchUpdater_ObjectHandler::triggerReindex - $item->triggerReindex(); - $item->destroy(); - } - - $this->getLogger()->info("Done"); - } + $this->getLogger()->info("Done"); + } } diff --git a/code/solr/Solr3Service.php b/code/solr/Solr3Service.php index 523a622..6c3e535 100644 --- a/code/solr/Solr3Service.php +++ b/code/solr/Solr3Service.php @@ -1,8 +1,10 @@ '; + return $this->_sendRawPost($this->_updateUrl, $rawPost, $timeout); + } + + /** + * @inheritdoc + * @see Solr4Service_Core::addDocuments + */ + public function addDocument(Apache_Solr_Document $document, $allowDups = false, + $overwritePending = true, $overwriteCommitted = true, $commitWithin = 0 + ) { + return $this->addDocuments(array($document), $allowDups, $overwritePending, $overwriteCommitted, $commitWithin); + } - $rawPost = ''; - return $this->_sendRawPost($this->_updateUrl, $rawPost, $timeout); - } - - /** - * @inheritdoc - * @see Solr4Service_Core::addDocuments - */ - public function addDocument(Apache_Solr_Document $document, $allowDups = false, - $overwritePending = true, $overwriteCommitted = true, $commitWithin = 0 - ) { - return $this->addDocuments(array($document), $allowDups, $overwritePending, $overwriteCommitted, $commitWithin); - } + /** + * Solr 4.0 compat http://wiki.apache.org/solr/UpdateXmlMessages#Optional_attributes_for_.22add.22 + * Remove allowDups, overwritePending and overwriteComitted + */ + public function addDocuments($documents, $allowDups = false, $overwritePending = true, + $overwriteCommitted = true, $commitWithin = 0 + ) { + $overwriteVal = $allowDups ? 'false' : 'true'; + $commitWithin = (int) $commitWithin; + $commitWithinString = $commitWithin > 0 ? " commitWithin=\"{$commitWithin}\"" : ''; - /** - * Solr 4.0 compat http://wiki.apache.org/solr/UpdateXmlMessages#Optional_attributes_for_.22add.22 - * Remove allowDups, overwritePending and overwriteComitted - */ - public function addDocuments($documents, $allowDups = false, $overwritePending = true, - $overwriteCommitted = true, $commitWithin = 0 - ) { - $overwriteVal = $allowDups ? 'false' : 'true'; - $commitWithin = (int) $commitWithin; - $commitWithinString = $commitWithin > 0 ? " commitWithin=\"{$commitWithin}\"" : ''; + $rawPost = ""; + foreach ($documents as $document) { + if ($document instanceof Apache_Solr_Document) { + $rawPost .= $this->_documentToXmlFragment($document); + } + } + $rawPost .= ''; - $rawPost = ""; - foreach ($documents as $document) { - if ($document instanceof Apache_Solr_Document) { - $rawPost .= $this->_documentToXmlFragment($document); - } - } - $rawPost .= ''; - - return $this->add($rawPost); - } + return $this->add($rawPost); + } } -class Solr4Service extends SolrService { - private static $core_class = 'Solr4Service_Core'; +class Solr4Service extends SolrService +{ + private static $core_class = 'Solr4Service_Core'; } - diff --git a/code/solr/SolrConfigStore.php b/code/solr/SolrConfigStore.php index bea24be..86cc0a3 100644 --- a/code/solr/SolrConfigStore.php +++ b/code/solr/SolrConfigStore.php @@ -5,29 +5,30 @@ * * The interface Solr_Configure uses to upload configuration files to Solr */ -interface SolrConfigStore { - /** - * Upload a file to Solr for index $index - * @param $index string - The name of an index (which is also used as the name of the Solr core for the index) - * @param $file string - A path to a file to upload. The base name of the file will be used on the remote side - * @return null - */ - function uploadFile($index, $file); +interface SolrConfigStore +{ + /** + * Upload a file to Solr for index $index + * @param $index string - The name of an index (which is also used as the name of the Solr core for the index) + * @param $file string - A path to a file to upload. The base name of the file will be used on the remote side + * @return null + */ + public function uploadFile($index, $file); - /** - * Upload a file to Solr from a string for index $index - * @param $index string - The name of an index (which is also used as the name of the Solr core for the index) - * @param $filename string - The base name of the file to use on the remote side - * @param $strong string - The contents of the file - * @return null - */ - function uploadString($index, $filename, $string); + /** + * Upload a file to Solr from a string for index $index + * @param $index string - The name of an index (which is also used as the name of the Solr core for the index) + * @param $filename string - The base name of the file to use on the remote side + * @param $strong string - The contents of the file + * @return null + */ + public function uploadString($index, $filename, $string); - /** - * Get the instanceDir to tell Solr to use for index $index - * @param $index string - The name of an index (which is also used as the name of the Solr core for the index) - */ - function instanceDir($index); + /** + * Get the instanceDir to tell Solr to use for index $index + * @param $index string - The name of an index (which is also used as the name of the Solr core for the index) + */ + public function instanceDir($index); } /** @@ -36,41 +37,47 @@ interface SolrConfigStore { * A ConfigStore that uploads files to a Solr instance on a locally accessible filesystem * by just using file copies */ -class SolrConfigStore_File implements SolrConfigStore { - function __construct($config) { - $this->local = $config['path']; - $this->remote = isset($config['remotepath']) ? $config['remotepath'] : $config['path']; - } +class SolrConfigStore_File implements SolrConfigStore +{ + public function __construct($config) + { + $this->local = $config['path']; + $this->remote = isset($config['remotepath']) ? $config['remotepath'] : $config['path']; + } - function getTargetDir($index) { - $targetDir = "{$this->local}/{$index}/conf"; + public function getTargetDir($index) + { + $targetDir = "{$this->local}/{$index}/conf"; - if (!is_dir($targetDir)) { - $worked = @mkdir($targetDir, 0770, true); + if (!is_dir($targetDir)) { + $worked = @mkdir($targetDir, 0770, true); - if(!$worked) { - throw new RuntimeException( - sprintf('Failed creating target directory %s, please check permissions', $targetDir) - ); - } - } + if (!$worked) { + throw new RuntimeException( + sprintf('Failed creating target directory %s, please check permissions', $targetDir) + ); + } + } - return $targetDir; - } + return $targetDir; + } - function uploadFile($index, $file) { - $targetDir = $this->getTargetDir($index); - copy($file, $targetDir.'/'.basename($file)); - } + public function uploadFile($index, $file) + { + $targetDir = $this->getTargetDir($index); + copy($file, $targetDir.'/'.basename($file)); + } - function uploadString($index, $filename, $string) { - $targetDir = $this->getTargetDir($index); - file_put_contents("$targetDir/$filename", $string); - } + public function uploadString($index, $filename, $string) + { + $targetDir = $this->getTargetDir($index); + file_put_contents("$targetDir/$filename", $string); + } - function instanceDir($index) { - return $this->remote.'/'.$index; - } + public function instanceDir($index) + { + return $this->remote.'/'.$index; + } } /** @@ -78,40 +85,50 @@ class SolrConfigStore_File implements SolrConfigStore { * * A ConfigStore that uploads files to a Solr instance via a WebDAV server */ -class SolrConfigStore_WebDAV implements SolrConfigStore { - function __construct($config) { - $options = Solr::solr_options(); +class SolrConfigStore_WebDAV implements SolrConfigStore +{ + public function __construct($config) + { + $options = Solr::solr_options(); - $this->url = implode('', array( - 'http://', - isset($config['auth']) ? $config['auth'].'@' : '', - $options['host'].':'.$options['port'], - $config['path'] - )); - $this->remote = $config['remotepath']; - } + $this->url = implode('', array( + 'http://', + isset($config['auth']) ? $config['auth'].'@' : '', + $options['host'].':'.$options['port'], + $config['path'] + )); + $this->remote = $config['remotepath']; + } - function getTargetDir($index) { - $indexdir = "{$this->url}/$index"; - if (!WebDAV::exists($indexdir)) WebDAV::mkdir($indexdir); + public function getTargetDir($index) + { + $indexdir = "{$this->url}/$index"; + if (!WebDAV::exists($indexdir)) { + WebDAV::mkdir($indexdir); + } - $targetDir = "{$this->url}/$index/conf"; - if (!WebDAV::exists($targetDir)) WebDAV::mkdir($targetDir); + $targetDir = "{$this->url}/$index/conf"; + if (!WebDAV::exists($targetDir)) { + WebDAV::mkdir($targetDir); + } - return $targetDir; - } + return $targetDir; + } - function uploadFile($index, $file) { - $targetDir = $this->getTargetDir($index); - WebDAV::upload_from_file($file, $targetDir.'/'.basename($file)); - } + public function uploadFile($index, $file) + { + $targetDir = $this->getTargetDir($index); + WebDAV::upload_from_file($file, $targetDir.'/'.basename($file)); + } - function uploadString($index, $filename, $string) { - $targetDir = $this->getTargetDir($index); - WebDAV::upload_from_string($string, "$targetDir/$filename"); - } + public function uploadString($index, $filename, $string) + { + $targetDir = $this->getTargetDir($index); + WebDAV::upload_from_string($string, "$targetDir/$filename"); + } - function instanceDir($index) { - return $this->remote ? "{$this->remote}/$index" : $index; - } + public function instanceDir($index) + { + return $this->remote ? "{$this->remote}/$index" : $index; + } } diff --git a/code/solr/SolrIndex.php b/code/solr/SolrIndex.php index 24728a6..01d98c7 100644 --- a/code/solr/SolrIndex.php +++ b/code/solr/SolrIndex.php @@ -2,868 +2,946 @@ Solr::include_client_api(); -abstract class SolrIndex extends SearchIndex { - - static $fulltextTypeMap = array( - '*' => 'text', - 'HTMLVarchar' => 'htmltext', - 'HTMLText' => 'htmltext' - ); - - static $filterTypeMap = array( - '*' => 'string', - 'Boolean' => 'boolean', - 'Date' => 'tdate', - 'SSDatetime' => 'tdate', - 'SS_Datetime' => 'tdate', - 'ForeignKey' => 'tint', - 'Int' => 'tint', - 'Float' => 'tfloat', - 'Double' => 'tdouble' - ); - - static $sortTypeMap = array(); - - protected $analyzerFields = array(); - - protected $copyFields = array(); - - protected $extrasPath = null; - - protected $templatesPath = null; - - /** - * List of boosted fields - * - * @var array - */ - protected $boostedFields = array(); - - /** - * Name of default field - * - * @var string - * @config - */ - private static $default_field = '_text'; - - /** - * List of copy fields all fulltext fields should be copied into. - * This will fallback to default_field if not specified - * - * @var array - */ - private static $copy_fields = array(); - - /** - * @return String Absolute path to the folder containing - * templates which are used for generating the schema and field definitions. - */ - function getTemplatesPath() { - $globalOptions = Solr::solr_options(); - return $this->templatesPath ? $this->templatesPath : $globalOptions['templatespath']; - } - - /** - * @return String Absolute path to the configuration default files, - * e.g. solrconfig.xml. - */ - function getExtrasPath() { - $globalOptions = Solr::solr_options(); - return $this->extrasPath ? $this->extrasPath : $globalOptions['extraspath']; - } - - function generateSchema() { - return $this->renderWith($this->getTemplatesPath() . '/schema.ss'); - } - - function getIndexName() { - return get_class($this); - } - - function getTypes() { - return $this->renderWith($this->getTemplatesPath() . '/types.ss'); - } - - /** - * Index-time analyzer which is applied to a specific field. - * Can be used to remove HTML tags, apply stemming, etc. - * - * @see http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.WhitespaceTokenizerFactory - * - * @param String $field - * @param String $type - * @param Array $params Parameters for the analyzer, usually at least a "class" - */ - function addAnalyzer($field, $type, $params) { - $fullFields = $this->fieldData($field); - if($fullFields) foreach($fullFields as $fullField => $spec) { - if(!isset($this->analyzerFields[$fullField])) $this->analyzerFields[$fullField] = array(); - $this->analyzerFields[$fullField][$type] = $params; - } - } - - /** - * Get the default text field, normally '_text' - * - * @return string - */ - public function getDefaultField() { - return $this->config()->default_field; - } - - /** - * Get list of fields each text field should be copied into. - * This will fallback to the default field if omitted. - * - * @return array - */ - protected function getCopyDestinations() { - $copyFields = $this->config()->copy_fields; - if($copyFields) { - return $copyFields; - } - // Fallback to default field - $df = $this->getDefaultField(); - return array($df); - } - - public function getFieldDefinitions() { - $xml = array(); - $stored = $this->getStoredDefault(); - - $xml[] = ""; - - // Add the hardcoded field definitions - - $xml[] = ""; - - $xml[] = ""; - $xml[] = ""; - $xml[] = ""; - - // Add the fulltext collation field - - $df = $this->getDefaultField(); - $xml[] = "" ; - - // Add the user-specified fields - - foreach ($this->fulltextFields as $name => $field) { - $xml[] = $this->getFieldDefinition($name, $field, self::$fulltextTypeMap); - } - - foreach ($this->filterFields as $name => $field) { - if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') continue; - $xml[] = $this->getFieldDefinition($name, $field); - } - - foreach ($this->sortFields as $name => $field) { - if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') continue; - $xml[] = $this->getFieldDefinition($name, $field); - } - - return implode("\n\t\t", $xml); - } - - /** - * Extract first suggestion text from collated values - * - * @param mixed $collation - * @return string - */ - protected function getCollatedSuggestion($collation = '') { - if(is_string($collation)) { - return $collation; - } - if(is_object($collation)) { - if(isset($collation->misspellingsAndCorrections)) { - foreach($collation->misspellingsAndCorrections as $key => $value) { - return $value; - } - } - } - return ''; - } - - /** - * Extract a human friendly spelling suggestion from a Solr spellcheck collation string. - * @param String $collation - * @return String - */ - protected function getNiceSuggestion($collation = '') { - $collationParts = explode(' ', $collation); - - // Remove advanced query params from the beginning of each collation part. - foreach ($collationParts as $key => &$part) { - $part = ltrim($part, '+'); - } - - return implode(' ', $collationParts); - } - - /** - * Extract a query string from a Solr spellcheck collation string. - * Useful for constructing 'Did you mean?' links, for example: - * $SuggestionNice - * @param String $collation - * @return String - */ - protected function getSuggestionQueryString($collation = '') { - return str_replace(' ', '+', $this->getNiceSuggestion($collation)); - } - - /** - * Add a field that should be stored - * - * @param string $field The field to add - * @param string $forceType The type to force this field as (required in some cases, when not - * detectable from metadata) - * @param array $extraOptions Dependent on search implementation - */ - public function addStoredField($field, $forceType = null, $extraOptions = array()) { - $options = array_merge($extraOptions, array('stored' => 'true')); - $this->addFulltextField($field, $forceType, $options); - } - - /** - * Add a fulltext field with a boosted value - * - * @param string $field The field to add - * @param string $forceType The type to force this field as (required in some cases, when not - * detectable from metadata) - * @param array $extraOptions Dependent on search implementation - * @param float $boost Numeric boosting value (defaults to 2) - */ - public function addBoostedField($field, $forceType = null, $extraOptions = array(), $boost = 2) { - $options = array_merge($extraOptions, array('boost' => $boost)); - $this->addFulltextField($field, $forceType, $options); - } - - - public function fieldData($field, $forceType = null, $extraOptions = array()) { - // Ensure that 'boost' is recorded here without being captured by solr - $boost = null; - if(array_key_exists('boost', $extraOptions)) { - $boost = $extraOptions['boost']; - unset($extraOptions['boost']); - } - $data = parent::fieldData($field, $forceType, $extraOptions); - - // Boost all fields with this name - if(isset($boost)) { - foreach($data as $fieldName => $fieldInfo) { - $this->boostedFields[$fieldName] = $boost; - } - } - return $data; - } - - /** - * Set the default boosting level for a specific field. - * Will control the default value for qf param (Query Fields), but will not - * override a query-specific value. - * - * Fields must be added before having a field boosting specified - * - * @param string $field Full field key (Model_Field) - * @param float|null $level Numeric boosting value. Set to null to clear boost - */ - public function setFieldBoosting($field, $level) { - if(!isset($this->fulltextFields[$field])) { - throw new InvalidArgumentException("No fulltext field $field exists on ".$this->getIndexName()); - } - if($level === null) { - unset($this->boostedFields[$field]); - } else { - $this->boostedFields[$field] = $level; - } - } - - /** - * Get all boosted fields - * - * @return array - */ - public function getBoostedFields() { - return $this->boostedFields; - } - - /** - * Determine the best default value for the 'qf' parameter - * - * @return array|null List of query fields, or null if not specified - */ - public function getQueryFields() { - // Not necessary to specify this unless boosting - if(empty($this->boostedFields)) { - return null; - } - $queryFields = array(); - foreach ($this->boostedFields as $fieldName => $boost) { - $queryFields[] = $fieldName . '^' . $boost; - } - - // If any fields are queried, we must always include the default field, otherwise it will be excluded - $df = $this->getDefaultField(); - if($queryFields && !isset($this->boostedFields[$df])) { - $queryFields[] = $df; - } - - return $queryFields; - } - - /** - * Gets the default 'stored' value for fields in this index - * - * @return string A default value for the 'stored' field option, either 'true' or 'false' - */ - protected function getStoredDefault() { - return Director::isDev() ? 'true' : 'false'; - } - - /** - * @param String $name - * @param Array $spec - * @param Array $typeMap - * @return String XML - */ - protected function getFieldDefinition($name, $spec, $typeMap = null) { - if(!$typeMap) $typeMap = self::$filterTypeMap; - $multiValued = (isset($spec['multi_valued']) && $spec['multi_valued']) ? "true" : ''; - $type = isset($typeMap[$spec['type']]) ? $typeMap[$spec['type']] : $typeMap['*']; - - $analyzerXml = ''; - if(isset($this->analyzerFields[$name])) { - foreach($this->analyzerFields[$name] as $analyzerType => $analyzerParams) { - $analyzerXml .= $this->toXmlTag($analyzerType, $analyzerParams); - } - } - - $fieldParams = array_merge( - array( - 'name' => $name, - 'type' => $type, - 'indexed' => 'true', - 'stored' => $this->getStoredDefault(), - 'multiValued' => $multiValued - ), - isset($spec['extra_options']) ? $spec['extra_options'] : array() - ); - - return $this->toXmlTag( - "field", - $fieldParams, - $analyzerXml ? "$analyzerXml" : null - ); - } - - /** - * Convert definition to XML tag - * - * @param String $tag - * @param String $attrs Map of attributes - * @param String $content Inner content - * @return String XML tag - */ - protected function toXmlTag($tag, $attrs, $content = null) { - $xml = "<$tag "; - if($attrs) { - $attrStrs = array(); - foreach($attrs as $attrName => $attrVal) $attrStrs[] = "$attrName='$attrVal'"; - $xml .= $attrStrs ? implode(' ', $attrStrs) : ''; - } - $xml .= $content ? ">$content" : '/>'; - return $xml; - } - - /** - * @param String $source Composite field name (_) - * @param String $dest - */ - function addCopyField($source, $dest, $extraOptions = array()) { - if(!isset($this->copyFields[$source])) $this->copyFields[$source] = array(); - $this->copyFields[$source][] = array_merge( - array('source' => $source, 'dest' => $dest), - $extraOptions - ); - } - - /** - * Generate XML for copy field definitions - * - * @return string - */ - public function getCopyFieldDefinitions() { - $xml = array(); - - // Default copy fields - foreach($this->getCopyDestinations() as $copyTo) { - foreach ($this->fulltextFields as $name => $field) { - $xml[] = ""; - } - } - - // Explicit copy fields - foreach ($this->copyFields as $source => $fields) { - foreach($fields as $fieldAttrs) { - $xml[] = $this->toXmlTag('copyField', $fieldAttrs); - } - } - - return implode("\n\t", $xml); - } - - protected function _addField($doc, $object, $field) { - $class = get_class($object); - if ($class != $field['origin'] && !is_subclass_of($class, $field['origin'])) return; - - $value = $this->_getFieldValue($object, $field); - - $type = isset(self::$filterTypeMap[$field['type']]) ? self::$filterTypeMap[$field['type']] : self::$filterTypeMap['*']; - - if (is_array($value)) foreach($value as $sub) { - /* Solr requires dates in the form 1995-12-31T23:59:59Z */ - if ($type == 'tdate') { - if(!$sub) continue; - $sub = gmdate('Y-m-d\TH:i:s\Z', strtotime($sub)); - } - - /* Solr requires numbers to be valid if presented, not just empty */ - if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($sub)) continue; - - $doc->addField($field['name'], $sub); - } - - else { - /* Solr requires dates in the form 1995-12-31T23:59:59Z */ - if ($type == 'tdate') { - if(!$value) return; - $value = gmdate('Y-m-d\TH:i:s\Z', strtotime($value)); - } - - /* Solr requires numbers to be valid if presented, not just empty */ - if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($value)) return; - - $doc->setField($field['name'], $value); - } - } - - protected function _addAs($object, $base, $options) { - $includeSubs = $options['include_children']; - - $doc = new Apache_Solr_Document(); - - // Always present fields - - $doc->setField('_documentid', $this->getDocumentID($object, $base, $includeSubs)); - $doc->setField('ID', $object->ID); - $doc->setField('ClassName', $object->ClassName); - - foreach (SearchIntrospection::hierarchy(get_class($object), false) as $class) $doc->addField('ClassHierarchy', $class); - - // Add the user-specified fields - - foreach ($this->getFieldsIterator() as $name => $field) { - if ($field['base'] == $base) $this->_addField($doc, $object, $field); - } - - try { - $this->getService()->addDocument($doc); - } catch (Exception $e) { - SS_Log::log($e, SS_Log::WARN); - return false; - } - - return $doc; - } - - function add($object) { - $class = get_class($object); - $docs = array(); - - foreach ($this->getClasses() as $searchclass => $options) { - if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { - $base = ClassInfo::baseDataClass($searchclass); - $docs[] = $this->_addAs($object, $base, $options); - } - } - - return $docs; - } - - function canAdd($class) { - foreach ($this->classes as $searchclass => $options) { - if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) return true; - } - - return false; - } - - function delete($base, $id, $state) { - $documentID = $this->getDocumentIDForState($base, $id, $state); - - try { - $this->getService()->deleteById($documentID); - } catch (Exception $e) { - SS_Log::log($e, SS_Log::WARN); - return false; - } - } - - /** - * Clear all records which do not match the given classname whitelist. - * - * Can also be used to trim an index when reducing to a narrower set of classes. - * - * Ignores current state / variant. - * - * @param array $classes List of non-obsolete classes in the same format as SolrIndex::getClasses() - * @return bool Flag if successful - */ - public function clearObsoleteClasses($classes) { - if(empty($classes)) { - return false; - } - - // Delete all records which do not match the necessary classname rules - $conditions = array(); - foreach ($classes as $class => $options) { - if ($options['include_children']) { - $conditions[] = "ClassHierarchy:{$class}"; - } else { - $conditions[] = "ClassName:{$class}"; - } - } - - // Delete records which don't match any of these conditions in this index - $deleteQuery = "-(" . implode(' ', $conditions) . ")"; - $this - ->getService() - ->deleteByQuery($deleteQuery); - return true; - } - - function commit() { - try { - $this->getService()->commit(false, false, false); - } catch (Exception $e) { - SS_Log::log($e, SS_Log::WARN); - return false; - } - } - - /** - * @param SearchQuery $query - * @param integer $offset - * @param integer $limit - * @param array $params Extra request parameters passed through to Solr - * @return ArrayData Map with the following keys: - * - 'Matches': ArrayList of the matched object instances - */ - public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = array()) { - $service = $this->getService(); - - $searchClass = count($query->classes) == 1 - ? $query->classes[0]['class'] - : null; - SearchVariant::with($searchClass) - ->call('alterQuery', $query, $this); - - $q = array(); // Query - $fq = array(); // Filter query - $qf = array(); // Query fields - $hlq = array(); // Highlight query - - // Build the search itself - $q = $this->getQueryComponent($query, $hlq); - - // If using boosting, set the clean term separately for highlighting. - // See https://issues.apache.org/jira/browse/SOLR-2632 - if(array_key_exists('hl', $params) && !array_key_exists('hl.q', $params)) { - $params['hl.q'] = implode(' ', $hlq); - } - - // Filter by class if requested - $classq = array(); - foreach ($query->classes as $class) { - if (!empty($class['includeSubclasses'])) { - $classq[] = 'ClassHierarchy:'.$class['class']; - } - else $classq[] = 'ClassName:'.$class['class']; - } - if ($classq) $fq[] = '+('.implode(' ', $classq).')'; - - // Filter by filters - $fq = array_merge($fq, $this->getFiltersComponent($query)); - - // Prepare query fields unless specified explicitly - if(isset($params['qf'])) { - $qf = $params['qf']; - } else { - $qf = $this->getQueryFields(); - } - if(is_array($qf)) { - $qf = implode(' ', $qf); - } - if($qf) { - $params['qf'] = $qf; - } - - if(!headers_sent() && !Director::isLive()) { - if ($q) header('X-Query: '.implode(' ', $q)); - if ($fq) header('X-Filters: "'.implode('", "', $fq).'"'); - if ($qf) header('X-QueryFields: '.$qf); - } - - if ($offset == -1) $offset = $query->start; - if ($limit == -1) $limit = $query->limit; - if ($limit == -1) $limit = SearchQuery::$default_page_size; - - $params = array_merge($params, array('fq' => implode(' ', $fq))); - - $res = $service->search( - $q ? implode(' ', $q) : '*:*', - $offset, - $limit, - $params, - Apache_Solr_Service::METHOD_POST - ); - - $results = new ArrayList(); - if($res->getHttpStatus() >= 200 && $res->getHttpStatus() < 300) { - foreach ($res->response->docs as $doc) { - $result = DataObject::get_by_id($doc->ClassName, $doc->ID); - if($result) { - $results->push($result); - - // Add highlighting (optional) - $docId = $doc->_documentid; - if($res->highlighting && $res->highlighting->$docId) { - // TODO Create decorator class for search results rather than adding arbitrary object properties - // TODO Allow specifying highlighted field, and lazy loading - // in case the search API needs another query (similar to SphinxSearchable->buildExcerpt()). - $combinedHighlights = array(); - foreach($res->highlighting->$docId as $field => $highlights) { - $combinedHighlights = array_merge($combinedHighlights, $highlights); - } - - // Remove entity-encoded U+FFFD replacement character. It signifies non-displayable characters, - // and shows up as an encoding error in browsers. - $result->Excerpt = DBField::create_field( - 'HTMLText', - str_replace( - '�', - '', - implode(' ... ', $combinedHighlights) - ) - ); - } - } - } - $numFound = $res->response->numFound; - } else { - $numFound = 0; - } - - $ret = array(); - $ret['Matches'] = new PaginatedList($results); - $ret['Matches']->setLimitItems(false); - // Tell PaginatedList how many results there are - $ret['Matches']->setTotalItems($numFound); - // Results for current page start at $offset - $ret['Matches']->setPageStart($offset); - // Results per page - $ret['Matches']->setPageLength($limit); - - // Include spellcheck and suggestion data. Requires spellcheck=true in $params - if(isset($res->spellcheck)) { - // Expose all spellcheck data, for custom handling. - $ret['Spellcheck'] = $res->spellcheck; - - // Suggestions. Requires spellcheck.collate=true in $params - if(isset($res->spellcheck->suggestions->collation)) { - // Extract string suggestion - $suggestion = $this->getCollatedSuggestion($res->spellcheck->suggestions->collation); - - // The collation, including advanced query params (e.g. +), suitable for making another query programmatically. - $ret['Suggestion'] = $suggestion; - - // A human friendly version of the suggestion, suitable for 'Did you mean $SuggestionNice?' display. - $ret['SuggestionNice'] = $this->getNiceSuggestion($suggestion); - - // A string suitable for appending to an href as a query string. - // For example $SuggestionNice - $ret['SuggestionQueryString'] = $this->getSuggestionQueryString($suggestion); - } - } - - return new ArrayData($ret); - } - - - /** - * Get the query (q) component for this search - * - * @param SearchQuery $searchQuery - * @param array &$hlq Highlight query returned by reference - * @return array - */ - protected function getQueryComponent(SearchQuery $searchQuery, &$hlq = array()) { - $q = array(); - foreach ($searchQuery->search as $search) { - $text = $search['text']; - preg_match_all('/"[^"]*"|\S+/', $text, $parts); - - $fuzzy = $search['fuzzy'] ? '~' : ''; - - foreach ($parts[0] as $part) { - $fields = (isset($search['fields'])) ? $search['fields'] : array(); - if(isset($search['boost'])) { - $fields = array_merge($fields, array_keys($search['boost'])); - } - if ($fields) { - $searchq = array(); - foreach ($fields as $field) { - $boost = (isset($search['boost'][$field])) ? '^' . $search['boost'][$field] : ''; - $searchq[] = "{$field}:".$part.$fuzzy.$boost; - } - $q[] = '+('.implode(' OR ', $searchq).')'; - } - else { - $q[] = '+'.$part.$fuzzy; - } - $hlq[] = $part; - } - } - return $q; - } - - /** - * Parse all require constraints for inclusion in a filter query - * - * @param SearchQuery $searchQuery - * @return array List of parsed string values for each require - */ - protected function getRequireFiltersComponent(SearchQuery $searchQuery) { - $fq = array(); - foreach ($searchQuery->require as $field => $values) { - $requireq = array(); - - foreach ($values as $value) { - if ($value === SearchQuery::$missing) { - $requireq[] = "(*:* -{$field}:[* TO *])"; - } - else if ($value === SearchQuery::$present) { - $requireq[] = "{$field}:[* TO *]"; - } - else if ($value instanceof SearchQuery_Range) { - $start = $value->start; - if ($start === null) { - $start = '*'; - } - $end = $value->end; - if ($end === null) { - $end = '*'; - } - $requireq[] = "$field:[$start TO $end]"; - } - else { - $requireq[] = $field.':"'.$value.'"'; - } - } - - $fq[] = '+('.implode(' ', $requireq).')'; - } - return $fq; - } - - /** - * Parse all exclude constraints for inclusion in a filter query - * - * @param SearchQuery $searchQuery - * @return array List of parsed string values for each exclusion - */ - protected function getExcludeFiltersComponent(SearchQuery $searchQuery) { - $fq = array(); - foreach ($searchQuery->exclude as $field => $values) { - $excludeq = array(); - $missing = false; - - foreach ($values as $value) { - if ($value === SearchQuery::$missing) { - $missing = true; - } - else if ($value === SearchQuery::$present) { - $excludeq[] = "{$field}:[* TO *]"; - } - else if ($value instanceof SearchQuery_Range) { - $start = $value->start; - if ($start === null) { - $start = '*'; - } - $end = $value->end; - if ($end === null) { - $end = '*'; - } - $excludeq[] = "$field:[$start TO $end]"; - } - else { - $excludeq[] = $field.':"'.$value.'"'; - } - } - - $fq[] = ($missing ? "+{$field}:[* TO *] " : '') . '-('.implode(' ', $excludeq).')'; - } - return $fq; - } - - /** - * Get all filter conditions for this search - * - * @param SearchQuery $searchQuery - * @return array - */ - public function getFiltersComponent(SearchQuery $searchQuery) { - return array_merge( - $this->getRequireFiltersComponent($searchQuery), - $this->getExcludeFiltersComponent($searchQuery) - ); - } - - protected $service; - - /** - * @return SolrService - */ - public function getService() { - if(!$this->service) $this->service = Solr::service(get_class($this)); - return $this->service; - } - - public function setService(SolrService $service) { - $this->service = $service; - return $this; - } - - /** - * Upload config for this index to the given store - * - * @param SolrConfigStore $store - */ - public function uploadConfig($store) { - // Upload the config files for this index - $store->uploadString( - $this->getIndexName(), - 'schema.xml', - (string)$this->generateSchema() - ); - - // Upload additional files - foreach (glob($this->getExtrasPath().'/*') as $file) { - if (is_file($file)) { - $store->uploadFile($this->getIndexName(), $file); - } - } - } +abstract class SolrIndex extends SearchIndex +{ + public static $fulltextTypeMap = array( + '*' => 'text', + 'HTMLVarchar' => 'htmltext', + 'HTMLText' => 'htmltext' + ); + + public static $filterTypeMap = array( + '*' => 'string', + 'Boolean' => 'boolean', + 'Date' => 'tdate', + 'SSDatetime' => 'tdate', + 'SS_Datetime' => 'tdate', + 'ForeignKey' => 'tint', + 'Int' => 'tint', + 'Float' => 'tfloat', + 'Double' => 'tdouble' + ); + + public static $sortTypeMap = array(); + + protected $analyzerFields = array(); + + protected $copyFields = array(); + + protected $extrasPath = null; + + protected $templatesPath = null; + + /** + * List of boosted fields + * + * @var array + */ + protected $boostedFields = array(); + + /** + * Name of default field + * + * @var string + * @config + */ + private static $default_field = '_text'; + + /** + * List of copy fields all fulltext fields should be copied into. + * This will fallback to default_field if not specified + * + * @var array + */ + private static $copy_fields = array(); + + /** + * @return String Absolute path to the folder containing + * templates which are used for generating the schema and field definitions. + */ + public function getTemplatesPath() + { + $globalOptions = Solr::solr_options(); + return $this->templatesPath ? $this->templatesPath : $globalOptions['templatespath']; + } + + /** + * @return String Absolute path to the configuration default files, + * e.g. solrconfig.xml. + */ + public function getExtrasPath() + { + $globalOptions = Solr::solr_options(); + return $this->extrasPath ? $this->extrasPath : $globalOptions['extraspath']; + } + + public function generateSchema() + { + return $this->renderWith($this->getTemplatesPath() . '/schema.ss'); + } + + public function getIndexName() + { + return get_class($this); + } + + public function getTypes() + { + return $this->renderWith($this->getTemplatesPath() . '/types.ss'); + } + + /** + * Index-time analyzer which is applied to a specific field. + * Can be used to remove HTML tags, apply stemming, etc. + * + * @see http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.WhitespaceTokenizerFactory + * + * @param String $field + * @param String $type + * @param Array $params Parameters for the analyzer, usually at least a "class" + */ + public function addAnalyzer($field, $type, $params) + { + $fullFields = $this->fieldData($field); + if ($fullFields) { + foreach ($fullFields as $fullField => $spec) { + if (!isset($this->analyzerFields[$fullField])) { + $this->analyzerFields[$fullField] = array(); + } + $this->analyzerFields[$fullField][$type] = $params; + } + } + } + + /** + * Get the default text field, normally '_text' + * + * @return string + */ + public function getDefaultField() + { + return $this->config()->default_field; + } + + /** + * Get list of fields each text field should be copied into. + * This will fallback to the default field if omitted. + * + * @return array + */ + protected function getCopyDestinations() + { + $copyFields = $this->config()->copy_fields; + if ($copyFields) { + return $copyFields; + } + // Fallback to default field + $df = $this->getDefaultField(); + return array($df); + } + + public function getFieldDefinitions() + { + $xml = array(); + $stored = $this->getStoredDefault(); + + $xml[] = ""; + + // Add the hardcoded field definitions + + $xml[] = ""; + + $xml[] = ""; + $xml[] = ""; + $xml[] = ""; + + // Add the fulltext collation field + + $df = $this->getDefaultField(); + $xml[] = "" ; + + // Add the user-specified fields + + foreach ($this->fulltextFields as $name => $field) { + $xml[] = $this->getFieldDefinition($name, $field, self::$fulltextTypeMap); + } + + foreach ($this->filterFields as $name => $field) { + if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') { + continue; + } + $xml[] = $this->getFieldDefinition($name, $field); + } + + foreach ($this->sortFields as $name => $field) { + if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') { + continue; + } + $xml[] = $this->getFieldDefinition($name, $field); + } + + return implode("\n\t\t", $xml); + } + + /** + * Extract first suggestion text from collated values + * + * @param mixed $collation + * @return string + */ + protected function getCollatedSuggestion($collation = '') + { + if (is_string($collation)) { + return $collation; + } + if (is_object($collation)) { + if (isset($collation->misspellingsAndCorrections)) { + foreach ($collation->misspellingsAndCorrections as $key => $value) { + return $value; + } + } + } + return ''; + } + + /** + * Extract a human friendly spelling suggestion from a Solr spellcheck collation string. + * @param String $collation + * @return String + */ + protected function getNiceSuggestion($collation = '') + { + $collationParts = explode(' ', $collation); + + // Remove advanced query params from the beginning of each collation part. + foreach ($collationParts as $key => &$part) { + $part = ltrim($part, '+'); + } + + return implode(' ', $collationParts); + } + + /** + * Extract a query string from a Solr spellcheck collation string. + * Useful for constructing 'Did you mean?' links, for example: + * $SuggestionNice + * @param String $collation + * @return String + */ + protected function getSuggestionQueryString($collation = '') + { + return str_replace(' ', '+', $this->getNiceSuggestion($collation)); + } + + /** + * Add a field that should be stored + * + * @param string $field The field to add + * @param string $forceType The type to force this field as (required in some cases, when not + * detectable from metadata) + * @param array $extraOptions Dependent on search implementation + */ + public function addStoredField($field, $forceType = null, $extraOptions = array()) + { + $options = array_merge($extraOptions, array('stored' => 'true')); + $this->addFulltextField($field, $forceType, $options); + } + + /** + * Add a fulltext field with a boosted value + * + * @param string $field The field to add + * @param string $forceType The type to force this field as (required in some cases, when not + * detectable from metadata) + * @param array $extraOptions Dependent on search implementation + * @param float $boost Numeric boosting value (defaults to 2) + */ + public function addBoostedField($field, $forceType = null, $extraOptions = array(), $boost = 2) + { + $options = array_merge($extraOptions, array('boost' => $boost)); + $this->addFulltextField($field, $forceType, $options); + } + + + public function fieldData($field, $forceType = null, $extraOptions = array()) + { + // Ensure that 'boost' is recorded here without being captured by solr + $boost = null; + if (array_key_exists('boost', $extraOptions)) { + $boost = $extraOptions['boost']; + unset($extraOptions['boost']); + } + $data = parent::fieldData($field, $forceType, $extraOptions); + + // Boost all fields with this name + if (isset($boost)) { + foreach ($data as $fieldName => $fieldInfo) { + $this->boostedFields[$fieldName] = $boost; + } + } + return $data; + } + + /** + * Set the default boosting level for a specific field. + * Will control the default value for qf param (Query Fields), but will not + * override a query-specific value. + * + * Fields must be added before having a field boosting specified + * + * @param string $field Full field key (Model_Field) + * @param float|null $level Numeric boosting value. Set to null to clear boost + */ + public function setFieldBoosting($field, $level) + { + if (!isset($this->fulltextFields[$field])) { + throw new InvalidArgumentException("No fulltext field $field exists on ".$this->getIndexName()); + } + if ($level === null) { + unset($this->boostedFields[$field]); + } else { + $this->boostedFields[$field] = $level; + } + } + + /** + * Get all boosted fields + * + * @return array + */ + public function getBoostedFields() + { + return $this->boostedFields; + } + + /** + * Determine the best default value for the 'qf' parameter + * + * @return array|null List of query fields, or null if not specified + */ + public function getQueryFields() + { + // Not necessary to specify this unless boosting + if (empty($this->boostedFields)) { + return null; + } + $queryFields = array(); + foreach ($this->boostedFields as $fieldName => $boost) { + $queryFields[] = $fieldName . '^' . $boost; + } + + // If any fields are queried, we must always include the default field, otherwise it will be excluded + $df = $this->getDefaultField(); + if ($queryFields && !isset($this->boostedFields[$df])) { + $queryFields[] = $df; + } + + return $queryFields; + } + + /** + * Gets the default 'stored' value for fields in this index + * + * @return string A default value for the 'stored' field option, either 'true' or 'false' + */ + protected function getStoredDefault() + { + return Director::isDev() ? 'true' : 'false'; + } + + /** + * @param String $name + * @param Array $spec + * @param Array $typeMap + * @return String XML + */ + protected function getFieldDefinition($name, $spec, $typeMap = null) + { + if (!$typeMap) { + $typeMap = self::$filterTypeMap; + } + $multiValued = (isset($spec['multi_valued']) && $spec['multi_valued']) ? "true" : ''; + $type = isset($typeMap[$spec['type']]) ? $typeMap[$spec['type']] : $typeMap['*']; + + $analyzerXml = ''; + if (isset($this->analyzerFields[$name])) { + foreach ($this->analyzerFields[$name] as $analyzerType => $analyzerParams) { + $analyzerXml .= $this->toXmlTag($analyzerType, $analyzerParams); + } + } + + $fieldParams = array_merge( + array( + 'name' => $name, + 'type' => $type, + 'indexed' => 'true', + 'stored' => $this->getStoredDefault(), + 'multiValued' => $multiValued + ), + isset($spec['extra_options']) ? $spec['extra_options'] : array() + ); + + return $this->toXmlTag( + "field", + $fieldParams, + $analyzerXml ? "$analyzerXml" : null + ); + } + + /** + * Convert definition to XML tag + * + * @param String $tag + * @param String $attrs Map of attributes + * @param String $content Inner content + * @return String XML tag + */ + protected function toXmlTag($tag, $attrs, $content = null) + { + $xml = "<$tag "; + if ($attrs) { + $attrStrs = array(); + foreach ($attrs as $attrName => $attrVal) { + $attrStrs[] = "$attrName='$attrVal'"; + } + $xml .= $attrStrs ? implode(' ', $attrStrs) : ''; + } + $xml .= $content ? ">$content" : '/>'; + return $xml; + } + + /** + * @param String $source Composite field name (_) + * @param String $dest + */ + public function addCopyField($source, $dest, $extraOptions = array()) + { + if (!isset($this->copyFields[$source])) { + $this->copyFields[$source] = array(); + } + $this->copyFields[$source][] = array_merge( + array('source' => $source, 'dest' => $dest), + $extraOptions + ); + } + + /** + * Generate XML for copy field definitions + * + * @return string + */ + public function getCopyFieldDefinitions() + { + $xml = array(); + + // Default copy fields + foreach ($this->getCopyDestinations() as $copyTo) { + foreach ($this->fulltextFields as $name => $field) { + $xml[] = ""; + } + } + + // Explicit copy fields + foreach ($this->copyFields as $source => $fields) { + foreach ($fields as $fieldAttrs) { + $xml[] = $this->toXmlTag('copyField', $fieldAttrs); + } + } + + return implode("\n\t", $xml); + } + + protected function _addField($doc, $object, $field) + { + $class = get_class($object); + if ($class != $field['origin'] && !is_subclass_of($class, $field['origin'])) { + return; + } + + $value = $this->_getFieldValue($object, $field); + + $type = isset(self::$filterTypeMap[$field['type']]) ? self::$filterTypeMap[$field['type']] : self::$filterTypeMap['*']; + + if (is_array($value)) { + foreach ($value as $sub) { + /* Solr requires dates in the form 1995-12-31T23:59:59Z */ + if ($type == 'tdate') { + if (!$sub) { + continue; + } + $sub = gmdate('Y-m-d\TH:i:s\Z', strtotime($sub)); + } + + /* Solr requires numbers to be valid if presented, not just empty */ + if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($sub)) { + continue; + } + + $doc->addField($field['name'], $sub); + } + } else { + /* Solr requires dates in the form 1995-12-31T23:59:59Z */ + if ($type == 'tdate') { + if (!$value) { + return; + } + $value = gmdate('Y-m-d\TH:i:s\Z', strtotime($value)); + } + + /* Solr requires numbers to be valid if presented, not just empty */ + if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($value)) { + return; + } + + $doc->setField($field['name'], $value); + } + } + + protected function _addAs($object, $base, $options) + { + $includeSubs = $options['include_children']; + + $doc = new Apache_Solr_Document(); + + // Always present fields + + $doc->setField('_documentid', $this->getDocumentID($object, $base, $includeSubs)); + $doc->setField('ID', $object->ID); + $doc->setField('ClassName', $object->ClassName); + + foreach (SearchIntrospection::hierarchy(get_class($object), false) as $class) { + $doc->addField('ClassHierarchy', $class); + } + + // Add the user-specified fields + + foreach ($this->getFieldsIterator() as $name => $field) { + if ($field['base'] == $base) { + $this->_addField($doc, $object, $field); + } + } + + try { + $this->getService()->addDocument($doc); + } catch (Exception $e) { + SS_Log::log($e, SS_Log::WARN); + return false; + } + + return $doc; + } + + public function add($object) + { + $class = get_class($object); + $docs = array(); + + foreach ($this->getClasses() as $searchclass => $options) { + if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { + $base = ClassInfo::baseDataClass($searchclass); + $docs[] = $this->_addAs($object, $base, $options); + } + } + + return $docs; + } + + public function canAdd($class) + { + foreach ($this->classes as $searchclass => $options) { + if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { + return true; + } + } + + return false; + } + + public function delete($base, $id, $state) + { + $documentID = $this->getDocumentIDForState($base, $id, $state); + + try { + $this->getService()->deleteById($documentID); + } catch (Exception $e) { + SS_Log::log($e, SS_Log::WARN); + return false; + } + } + + /** + * Clear all records which do not match the given classname whitelist. + * + * Can also be used to trim an index when reducing to a narrower set of classes. + * + * Ignores current state / variant. + * + * @param array $classes List of non-obsolete classes in the same format as SolrIndex::getClasses() + * @return bool Flag if successful + */ + public function clearObsoleteClasses($classes) + { + if (empty($classes)) { + return false; + } + + // Delete all records which do not match the necessary classname rules + $conditions = array(); + foreach ($classes as $class => $options) { + if ($options['include_children']) { + $conditions[] = "ClassHierarchy:{$class}"; + } else { + $conditions[] = "ClassName:{$class}"; + } + } + + // Delete records which don't match any of these conditions in this index + $deleteQuery = "-(" . implode(' ', $conditions) . ")"; + $this + ->getService() + ->deleteByQuery($deleteQuery); + return true; + } + + public function commit() + { + try { + $this->getService()->commit(false, false, false); + } catch (Exception $e) { + SS_Log::log($e, SS_Log::WARN); + return false; + } + } + + /** + * @param SearchQuery $query + * @param integer $offset + * @param integer $limit + * @param array $params Extra request parameters passed through to Solr + * @return ArrayData Map with the following keys: + * - 'Matches': ArrayList of the matched object instances + */ + public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = array()) + { + $service = $this->getService(); + + $searchClass = count($query->classes) == 1 + ? $query->classes[0]['class'] + : null; + SearchVariant::with($searchClass) + ->call('alterQuery', $query, $this); + + $q = array(); // Query + $fq = array(); // Filter query + $qf = array(); // Query fields + $hlq = array(); // Highlight query + + // Build the search itself + $q = $this->getQueryComponent($query, $hlq); + + // If using boosting, set the clean term separately for highlighting. + // See https://issues.apache.org/jira/browse/SOLR-2632 + if (array_key_exists('hl', $params) && !array_key_exists('hl.q', $params)) { + $params['hl.q'] = implode(' ', $hlq); + } + + // Filter by class if requested + $classq = array(); + foreach ($query->classes as $class) { + if (!empty($class['includeSubclasses'])) { + $classq[] = 'ClassHierarchy:'.$class['class']; + } else { + $classq[] = 'ClassName:'.$class['class']; + } + } + if ($classq) { + $fq[] = '+('.implode(' ', $classq).')'; + } + + // Filter by filters + $fq = array_merge($fq, $this->getFiltersComponent($query)); + + // Prepare query fields unless specified explicitly + if (isset($params['qf'])) { + $qf = $params['qf']; + } else { + $qf = $this->getQueryFields(); + } + if (is_array($qf)) { + $qf = implode(' ', $qf); + } + if ($qf) { + $params['qf'] = $qf; + } + + if (!headers_sent() && !Director::isLive()) { + if ($q) { + header('X-Query: '.implode(' ', $q)); + } + if ($fq) { + header('X-Filters: "'.implode('", "', $fq).'"'); + } + if ($qf) { + header('X-QueryFields: '.$qf); + } + } + + if ($offset == -1) { + $offset = $query->start; + } + if ($limit == -1) { + $limit = $query->limit; + } + if ($limit == -1) { + $limit = SearchQuery::$default_page_size; + } + + $params = array_merge($params, array('fq' => implode(' ', $fq))); + + $res = $service->search( + $q ? implode(' ', $q) : '*:*', + $offset, + $limit, + $params, + Apache_Solr_Service::METHOD_POST + ); + + $results = new ArrayList(); + if ($res->getHttpStatus() >= 200 && $res->getHttpStatus() < 300) { + foreach ($res->response->docs as $doc) { + $result = DataObject::get_by_id($doc->ClassName, $doc->ID); + if ($result) { + $results->push($result); + + // Add highlighting (optional) + $docId = $doc->_documentid; + if ($res->highlighting && $res->highlighting->$docId) { + // TODO Create decorator class for search results rather than adding arbitrary object properties + // TODO Allow specifying highlighted field, and lazy loading + // in case the search API needs another query (similar to SphinxSearchable->buildExcerpt()). + $combinedHighlights = array(); + foreach ($res->highlighting->$docId as $field => $highlights) { + $combinedHighlights = array_merge($combinedHighlights, $highlights); + } + + // Remove entity-encoded U+FFFD replacement character. It signifies non-displayable characters, + // and shows up as an encoding error in browsers. + $result->Excerpt = DBField::create_field( + 'HTMLText', + str_replace( + '�', + '', + implode(' ... ', $combinedHighlights) + ) + ); + } + } + } + $numFound = $res->response->numFound; + } else { + $numFound = 0; + } + + $ret = array(); + $ret['Matches'] = new PaginatedList($results); + $ret['Matches']->setLimitItems(false); + // Tell PaginatedList how many results there are + $ret['Matches']->setTotalItems($numFound); + // Results for current page start at $offset + $ret['Matches']->setPageStart($offset); + // Results per page + $ret['Matches']->setPageLength($limit); + + // Include spellcheck and suggestion data. Requires spellcheck=true in $params + if (isset($res->spellcheck)) { + // Expose all spellcheck data, for custom handling. + $ret['Spellcheck'] = $res->spellcheck; + + // Suggestions. Requires spellcheck.collate=true in $params + if (isset($res->spellcheck->suggestions->collation)) { + // Extract string suggestion + $suggestion = $this->getCollatedSuggestion($res->spellcheck->suggestions->collation); + + // The collation, including advanced query params (e.g. +), suitable for making another query programmatically. + $ret['Suggestion'] = $suggestion; + + // A human friendly version of the suggestion, suitable for 'Did you mean $SuggestionNice?' display. + $ret['SuggestionNice'] = $this->getNiceSuggestion($suggestion); + + // A string suitable for appending to an href as a query string. + // For example $SuggestionNice + $ret['SuggestionQueryString'] = $this->getSuggestionQueryString($suggestion); + } + } + + return new ArrayData($ret); + } + + + /** + * Get the query (q) component for this search + * + * @param SearchQuery $searchQuery + * @param array &$hlq Highlight query returned by reference + * @return array + */ + protected function getQueryComponent(SearchQuery $searchQuery, &$hlq = array()) + { + $q = array(); + foreach ($searchQuery->search as $search) { + $text = $search['text']; + preg_match_all('/"[^"]*"|\S+/', $text, $parts); + + $fuzzy = $search['fuzzy'] ? '~' : ''; + + foreach ($parts[0] as $part) { + $fields = (isset($search['fields'])) ? $search['fields'] : array(); + if (isset($search['boost'])) { + $fields = array_merge($fields, array_keys($search['boost'])); + } + if ($fields) { + $searchq = array(); + foreach ($fields as $field) { + $boost = (isset($search['boost'][$field])) ? '^' . $search['boost'][$field] : ''; + $searchq[] = "{$field}:".$part.$fuzzy.$boost; + } + $q[] = '+('.implode(' OR ', $searchq).')'; + } else { + $q[] = '+'.$part.$fuzzy; + } + $hlq[] = $part; + } + } + return $q; + } + + /** + * Parse all require constraints for inclusion in a filter query + * + * @param SearchQuery $searchQuery + * @return array List of parsed string values for each require + */ + protected function getRequireFiltersComponent(SearchQuery $searchQuery) + { + $fq = array(); + foreach ($searchQuery->require as $field => $values) { + $requireq = array(); + + foreach ($values as $value) { + if ($value === SearchQuery::$missing) { + $requireq[] = "(*:* -{$field}:[* TO *])"; + } elseif ($value === SearchQuery::$present) { + $requireq[] = "{$field}:[* TO *]"; + } elseif ($value instanceof SearchQuery_Range) { + $start = $value->start; + if ($start === null) { + $start = '*'; + } + $end = $value->end; + if ($end === null) { + $end = '*'; + } + $requireq[] = "$field:[$start TO $end]"; + } else { + $requireq[] = $field.':"'.$value.'"'; + } + } + + $fq[] = '+('.implode(' ', $requireq).')'; + } + return $fq; + } + + /** + * Parse all exclude constraints for inclusion in a filter query + * + * @param SearchQuery $searchQuery + * @return array List of parsed string values for each exclusion + */ + protected function getExcludeFiltersComponent(SearchQuery $searchQuery) + { + $fq = array(); + foreach ($searchQuery->exclude as $field => $values) { + $excludeq = array(); + $missing = false; + + foreach ($values as $value) { + if ($value === SearchQuery::$missing) { + $missing = true; + } elseif ($value === SearchQuery::$present) { + $excludeq[] = "{$field}:[* TO *]"; + } elseif ($value instanceof SearchQuery_Range) { + $start = $value->start; + if ($start === null) { + $start = '*'; + } + $end = $value->end; + if ($end === null) { + $end = '*'; + } + $excludeq[] = "$field:[$start TO $end]"; + } else { + $excludeq[] = $field.':"'.$value.'"'; + } + } + + $fq[] = ($missing ? "+{$field}:[* TO *] " : '') . '-('.implode(' ', $excludeq).')'; + } + return $fq; + } + + /** + * Get all filter conditions for this search + * + * @param SearchQuery $searchQuery + * @return array + */ + public function getFiltersComponent(SearchQuery $searchQuery) + { + return array_merge( + $this->getRequireFiltersComponent($searchQuery), + $this->getExcludeFiltersComponent($searchQuery) + ); + } + + protected $service; + + /** + * @return SolrService + */ + public function getService() + { + if (!$this->service) { + $this->service = Solr::service(get_class($this)); + } + return $this->service; + } + + public function setService(SolrService $service) + { + $this->service = $service; + return $this; + } + + /** + * Upload config for this index to the given store + * + * @param SolrConfigStore $store + */ + public function uploadConfig($store) + { + // Upload the config files for this index + $store->uploadString( + $this->getIndexName(), + 'schema.xml', + (string)$this->generateSchema() + ); + + // Upload additional files + foreach (glob($this->getExtrasPath().'/*') as $file) { + if (is_file($file)) { + $store->uploadFile($this->getIndexName(), $file); + } + } + } } diff --git a/code/solr/SolrService.php b/code/solr/SolrService.php index 1e099e6..7b8c448 100644 --- a/code/solr/SolrService.php +++ b/code/solr/SolrService.php @@ -5,7 +5,8 @@ Solr::include_client_api(); /** * The API for accessing a specific core of a Solr server. Exactly the same as Apache_Solr_Service for now. */ -class SolrService_Core extends Apache_Solr_Service { +class SolrService_Core extends Apache_Solr_Service +{ } /** @@ -13,65 +14,77 @@ class SolrService_Core extends Apache_Solr_Service { * plus extra methods for interrogating, creating, reloading and getting SolrService_Core instances * for Solr cores. */ -class SolrService extends SolrService_Core { - private static $core_class = 'SolrService_Core'; +class SolrService extends SolrService_Core +{ + private static $core_class = 'SolrService_Core'; - /** - * Handle encoding the GET parameters and making the HTTP call to execute a core command - */ - protected function coreCommand($command, $core, $params=array()) { - $command = strtoupper($command); + /** + * Handle encoding the GET parameters and making the HTTP call to execute a core command + */ + protected function coreCommand($command, $core, $params=array()) + { + $command = strtoupper($command); - $params = array_merge($params, array('action' => $command, 'wt' => 'json')); - $params[$command == 'CREATE' ? 'name' : 'core'] = $core; + $params = array_merge($params, array('action' => $command, 'wt' => 'json')); + $params[$command == 'CREATE' ? 'name' : 'core'] = $core; - return $this->_sendRawGet($this->_constructUrl('admin/cores', $params)); - } + return $this->_sendRawGet($this->_constructUrl('admin/cores', $params)); + } - /** - * Is the passed core active? - * @param $core string - The name of the core - * @return boolean - True if that core exists & is active - */ - public function coreIsActive($core) { - $result = $this->coreCommand('STATUS', $core); - return isset($result->status->$core->uptime); - } + /** + * Is the passed core active? + * @param $core string - The name of the core + * @return boolean - True if that core exists & is active + */ + public function coreIsActive($core) + { + $result = $this->coreCommand('STATUS', $core); + return isset($result->status->$core->uptime); + } - /** - * Create a new core - * @param $core string - The name of the core - * @param $instancedir string - The base path of the core on the server - * @param $config string - The filename of solrconfig.xml on the server. Default is $instancedir/solrconfig.xml - * @param $schema string - The filename of schema.xml on the server. Default is $instancedir/schema.xml - * @param $datadir string - The path to store data for this core on the server. Default depends on solrconfig.xml - * @return Apache_Solr_Response - */ - public function coreCreate($core, $instancedir, $config=null, $schema=null, $datadir=null) { - $args = array('instanceDir' => $instancedir); - if ($config) $args['config'] = $config; - if ($schema) $args['schema'] = $schema; - if ($datadir) $args['dataDir'] = $datadir; + /** + * Create a new core + * @param $core string - The name of the core + * @param $instancedir string - The base path of the core on the server + * @param $config string - The filename of solrconfig.xml on the server. Default is $instancedir/solrconfig.xml + * @param $schema string - The filename of schema.xml on the server. Default is $instancedir/schema.xml + * @param $datadir string - The path to store data for this core on the server. Default depends on solrconfig.xml + * @return Apache_Solr_Response + */ + public function coreCreate($core, $instancedir, $config=null, $schema=null, $datadir=null) + { + $args = array('instanceDir' => $instancedir); + if ($config) { + $args['config'] = $config; + } + if ($schema) { + $args['schema'] = $schema; + } + if ($datadir) { + $args['dataDir'] = $datadir; + } - return $this->coreCommand('CREATE', $core, $args); - } + return $this->coreCommand('CREATE', $core, $args); + } - /** - * Reload a core - * @param $core string - The name of the core - * @return Apache_Solr_Response - */ - public function coreReload($core) { - return $this->coreCommand('RELOAD', $core); - } + /** + * Reload a core + * @param $core string - The name of the core + * @return Apache_Solr_Response + */ + public function coreReload($core) + { + return $this->coreCommand('RELOAD', $core); + } - /** - * Create a new Solr3Service_Core instance for the passed core - * @param $core string - The name of the core - * @return Solr3Service_Core - */ - public function serviceForCore($core) { - $klass = Config::inst()->get(get_called_class(), 'core_class'); - return new $klass($this->_host, $this->_port, $this->_path.$core, $this->_httpTransport); - } + /** + * Create a new Solr3Service_Core instance for the passed core + * @param $core string - The name of the core + * @return Solr3Service_Core + */ + public function serviceForCore($core) + { + $klass = Config::inst()->get(get_called_class(), 'core_class'); + return new $klass($this->_host, $this->_port, $this->_path.$core, $this->_httpTransport); + } } diff --git a/code/solr/reindex/handlers/SolrReindexBase.php b/code/solr/reindex/handlers/SolrReindexBase.php index 44f6b81..86f1843 100644 --- a/code/solr/reindex/handlers/SolrReindexBase.php +++ b/code/solr/reindex/handlers/SolrReindexBase.php @@ -5,226 +5,230 @@ use Psr\Log\LoggerInterface; /** * Base class for re-indexing of solr content */ -abstract class SolrReindexBase implements SolrReindexHandler { +abstract class SolrReindexBase implements SolrReindexHandler +{ + public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) + { + foreach (Solr::get_indexes() as $indexInstance) { + $this->processIndex($logger, $indexInstance, $batchSize, $taskName, $classes); + } + } + + /** + * Process index for a single SolrIndex instance + * + * @param LoggerInterface $logger + * @param SolrIndex $indexInstance + * @param int $batchSize + * @param string $taskName + * @param string $classes + */ + protected function processIndex( + LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null + ) { + // Filter classes for this index + $indexClasses = $this->getClassesForIndex($indexInstance, $classes); - public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { - foreach (Solr::get_indexes() as $indexInstance) { - $this->processIndex($logger, $indexInstance, $batchSize, $taskName, $classes); - } - } - - /** - * Process index for a single SolrIndex instance - * - * @param LoggerInterface $logger - * @param SolrIndex $indexInstance - * @param int $batchSize - * @param string $taskName - * @param string $classes - */ - protected function processIndex( - LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null - ) { - // Filter classes for this index - $indexClasses = $this->getClassesForIndex($indexInstance, $classes); + // Clear all records in this index which do not contain the given classes + $logger->info("Clearing obsolete classes from ".$indexInstance->getIndexName()); + $indexInstance->clearObsoleteClasses($indexClasses); - // Clear all records in this index which do not contain the given classes - $logger->info("Clearing obsolete classes from ".$indexInstance->getIndexName()); - $indexInstance->clearObsoleteClasses($indexClasses); + // Build queue for each class + foreach ($indexClasses as $class => $options) { + $includeSubclasses = $options['include_children']; - // Build queue for each class - foreach ($indexClasses as $class => $options) { - $includeSubclasses = $options['include_children']; + foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) { + $this->processVariant($logger, $indexInstance, $state, $class, $includeSubclasses, $batchSize, $taskName); + } + } + } - foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) { - $this->processVariant($logger, $indexInstance, $state, $class, $includeSubclasses, $batchSize, $taskName); - } - } - } + /** + * Get valid classes and options for an index with an optional filter + * + * @param SolrIndex $index + * @param string|array $filterClasses Optional class or classes to limit to + * @return array List of classes, where the key is the classname and value is list of options + */ + protected function getClassesForIndex(SolrIndex $index, $filterClasses = null) + { + // Get base classes + $classes = $index->getClasses(); + if (!$filterClasses) { + return $classes; + } - /** - * Get valid classes and options for an index with an optional filter - * - * @param SolrIndex $index - * @param string|array $filterClasses Optional class or classes to limit to - * @return array List of classes, where the key is the classname and value is list of options - */ - protected function getClassesForIndex(SolrIndex $index, $filterClasses = null) { - // Get base classes - $classes = $index->getClasses(); - if(!$filterClasses) { - return $classes; - } + // Apply filter + if (!is_array($filterClasses)) { + $filterClasses = explode(',', $filterClasses); + } + return array_intersect_key($classes, array_combine($filterClasses, $filterClasses)); + } - // Apply filter - if(!is_array($filterClasses)) { - $filterClasses = explode(',', $filterClasses); - } - return array_intersect_key($classes, array_combine($filterClasses, $filterClasses)); - } + /** + * Process re-index for a given variant state and class + * + * @param LoggerInterface $logger + * @param SolrIndex $indexInstance + * @param array $state Variant state + * @param string $class + * @param bool $includeSubclasses + * @param int $batchSize + * @param string $taskName + */ + protected function processVariant( + LoggerInterface $logger, SolrIndex $indexInstance, $state, + $class, $includeSubclasses, $batchSize, $taskName + ) { + // Set state + SearchVariant::activate_state($state); - /** - * Process re-index for a given variant state and class - * - * @param LoggerInterface $logger - * @param SolrIndex $indexInstance - * @param array $state Variant state - * @param string $class - * @param bool $includeSubclasses - * @param int $batchSize - * @param string $taskName - */ - protected function processVariant( - LoggerInterface $logger, SolrIndex $indexInstance, $state, - $class, $includeSubclasses, $batchSize, $taskName - ) { - // Set state - SearchVariant::activate_state($state); + // Count records + $query = $class::get(); + if (!$includeSubclasses) { + $query = $query->filter('ClassName', $class); + } + $total = $query->count(); - // Count records - $query = $class::get(); - if(!$includeSubclasses) { - $query = $query->filter('ClassName', $class); - } - $total = $query->count(); + // Skip this variant if nothing to process, or if there are no records + if ($total == 0 || $indexInstance->variantStateExcluded($state)) { + // Remove all records in the current state, since there are no groups to process + $logger->info("Clearing all records of type {$class} in the current state: " . json_encode($state)); + $this->clearRecords($indexInstance, $class); + return; + } - // Skip this variant if nothing to process, or if there are no records - if ($total == 0 || $indexInstance->variantStateExcluded($state)) { - // Remove all records in the current state, since there are no groups to process - $logger->info("Clearing all records of type {$class} in the current state: " . json_encode($state)); - $this->clearRecords($indexInstance, $class); - return; - } + // For each group, run processing + $groups = (int)(($total + $batchSize - 1) / $batchSize); + for ($group = 0; $group < $groups; $group++) { + $this->processGroup($logger, $indexInstance, $state, $class, $groups, $group, $taskName); + } + } - // For each group, run processing - $groups = (int)(($total + $batchSize - 1) / $batchSize); - for ($group = 0; $group < $groups; $group++) { - $this->processGroup($logger, $indexInstance, $state, $class, $groups, $group, $taskName); - } - } + /** + * Initiate the processing of a single group + * + * @param LoggerInterface $logger + * @param SolrIndex $indexInstance Index instance + * @param array $state Variant state + * @param string $class Class to index + * @param int $groups Total groups + * @param int $group Index of group to process + * @param string $taskName Name of task script to run + */ + abstract protected function processGroup( + LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName + ); - /** - * Initiate the processing of a single group - * - * @param LoggerInterface $logger - * @param SolrIndex $indexInstance Index instance - * @param array $state Variant state - * @param string $class Class to index - * @param int $groups Total groups - * @param int $group Index of group to process - * @param string $taskName Name of task script to run - */ - abstract protected function processGroup( - LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName - ); + /** + * Explicitly invoke the process that performs the group + * processing. Can be run either by a background task or a queuedjob. + * + * Does not commit changes to the index, so this must be controlled externally. + * + * @param LoggerInterface $logger + * @param SolrIndex $indexInstance + * @param array $state + * @param string $class + * @param int $groups + * @param int $group + */ + public function runGroup( + LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group + ) { + // Set time limit and state + increase_time_limit_to(); + SearchVariant::activate_state($state); + $logger->info("Adding $class"); - /** - * Explicitly invoke the process that performs the group - * processing. Can be run either by a background task or a queuedjob. - * - * Does not commit changes to the index, so this must be controlled externally. - * - * @param LoggerInterface $logger - * @param SolrIndex $indexInstance - * @param array $state - * @param string $class - * @param int $groups - * @param int $group - */ - public function runGroup( - LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group - ) { - // Set time limit and state - increase_time_limit_to(); - SearchVariant::activate_state($state); - $logger->info("Adding $class"); + // Prior to adding these records to solr, delete existing solr records + $this->clearRecords($indexInstance, $class, $groups, $group); - // Prior to adding these records to solr, delete existing solr records - $this->clearRecords($indexInstance, $class, $groups, $group); + // Process selected records in this class + $items = $this->getRecordsInGroup($indexInstance, $class, $groups, $group); + $processed = array(); + foreach ($items as $item) { + $processed[] = $item->ID; - // Process selected records in this class - $items = $this->getRecordsInGroup($indexInstance, $class, $groups, $group); - $processed = array(); - foreach ($items as $item) { - $processed[] = $item->ID; + // By this point, obsolete classes/states have been removed in processVariant + // and obsolete records have been removed in clearRecords + $indexInstance->add($item); + $item->destroy(); + } + $logger->info("Updated ".implode(',', $processed)); - // By this point, obsolete classes/states have been removed in processVariant - // and obsolete records have been removed in clearRecords - $indexInstance->add($item); - $item->destroy(); - } - $logger->info("Updated ".implode(',', $processed)); + // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex + DB::query('SELECT 1'); + + $logger->info("Done"); + } - // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex - DB::query('SELECT 1'); - - $logger->info("Done"); - } + /** + * Gets the datalist of records in the given group in the current state + * + * Assumes that the desired variant state is in effect. + * + * @param SolrIndex $indexInstance + * @param string $class + * @param int $groups + * @param int $group + * @return DataList + */ + protected function getRecordsInGroup(SolrIndex $indexInstance, $class, $groups, $group) + { + // Generate filtered list of local records + $baseClass = ClassInfo::baseDataClass($class); + $items = DataList::create($class) + ->where(sprintf( + '"%s"."ID" %% \'%d\' = \'%d\'', + $baseClass, + intval($groups), + intval($group) + )) + ->sort("ID"); - /** - * Gets the datalist of records in the given group in the current state - * - * Assumes that the desired variant state is in effect. - * - * @param SolrIndex $indexInstance - * @param string $class - * @param int $groups - * @param int $group - * @return DataList - */ - protected function getRecordsInGroup(SolrIndex $indexInstance, $class, $groups, $group) { - // Generate filtered list of local records - $baseClass = ClassInfo::baseDataClass($class); - $items = DataList::create($class) - ->where(sprintf( - '"%s"."ID" %% \'%d\' = \'%d\'', - $baseClass, - intval($groups), - intval($group) - )) - ->sort("ID"); + // Add child filter + $classes = $indexInstance->getClasses(); + $options = $classes[$class]; + if (!$options['include_children']) { + $items = $items->filter('ClassName', $class); + } - // Add child filter - $classes = $indexInstance->getClasses(); - $options = $classes[$class]; - if(!$options['include_children']) { - $items = $items->filter('ClassName', $class); - } + return $items; + } - return $items; - } + /** + * Clear all records of the given class in the current state ONLY. + * + * Optionally delete from a given group (where the group is defined as the ID % total groups) + * + * @param SolrIndex $indexInstance Index instance + * @param string $class Class name + * @param int $groups Number of groups, if clearing from a striped group + * @param int $group Group number, if clearing from a striped group + */ + protected function clearRecords(SolrIndex $indexInstance, $class, $groups = null, $group = null) + { + // Clear by classname + $conditions = array("+(ClassHierarchy:{$class})"); - /** - * Clear all records of the given class in the current state ONLY. - * - * Optionally delete from a given group (where the group is defined as the ID % total groups) - * - * @param SolrIndex $indexInstance Index instance - * @param string $class Class name - * @param int $groups Number of groups, if clearing from a striped group - * @param int $group Group number, if clearing from a striped group - */ - protected function clearRecords(SolrIndex $indexInstance, $class, $groups = null, $group = null) { - // Clear by classname - $conditions = array("+(ClassHierarchy:{$class})"); + // If grouping, delete from this group only + if ($groups) { + $conditions[] = "+_query_:\"{!frange l={$group} u={$group}}mod(ID, {$groups})\""; + } - // If grouping, delete from this group only - if($groups) { - $conditions[] = "+_query_:\"{!frange l={$group} u={$group}}mod(ID, {$groups})\""; - } + // Also filter by state (suffix on document ID) + $query = new SearchQuery(); + SearchVariant::with($class) + ->call('alterQuery', $query, $indexInstance); + if ($query->isfiltered()) { + $conditions = array_merge($conditions, $indexInstance->getFiltersComponent($query)); + } - // Also filter by state (suffix on document ID) - $query = new SearchQuery(); - SearchVariant::with($class) - ->call('alterQuery', $query, $indexInstance); - if($query->isfiltered()) { - $conditions = array_merge($conditions, $indexInstance->getFiltersComponent($query)); - } - - // Invoke delete on index - $deleteQuery = implode(' ', $conditions); - $indexInstance - ->getService() - ->deleteByQuery($deleteQuery); - } + // Invoke delete on index + $deleteQuery = implode(' ', $conditions); + $indexInstance + ->getService() + ->deleteByQuery($deleteQuery); + } } diff --git a/code/solr/reindex/handlers/SolrReindexHandler.php b/code/solr/reindex/handlers/SolrReindexHandler.php index e57bb2b..c7b6be4 100644 --- a/code/solr/reindex/handlers/SolrReindexHandler.php +++ b/code/solr/reindex/handlers/SolrReindexHandler.php @@ -5,38 +5,38 @@ use Psr\Log\LoggerInterface; /** * Provides interface for queueing a solr reindex */ -interface SolrReindexHandler { +interface SolrReindexHandler +{ + /** + * Trigger a solr-reindex + * + * @param LoggerInterface $logger + * @param int $batchSize Records to run each process + * @param string $taskName Name of devtask to run + * @param string|array|null $classes Optional class or classes to limit index to + */ + public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null); - /** - * Trigger a solr-reindex - * - * @param LoggerInterface $logger - * @param int $batchSize Records to run each process - * @param string $taskName Name of devtask to run - * @param string|array|null $classes Optional class or classes to limit index to - */ - public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null); + /** + * Begin an immediate re-index + * + * @param LoggerInterface $logger + * @param int $batchSize Records to run each process + * @param string $taskName Name of devtask to run + * @param string|array|null $classes Optional class or classes to limit index to + */ + public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null); - /** - * Begin an immediate re-index - * - * @param LoggerInterface $logger - * @param int $batchSize Records to run each process - * @param string $taskName Name of devtask to run - * @param string|array|null $classes Optional class or classes to limit index to - */ - public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null); - - /** - * Do an immediate re-index on the given group, where the group is defined as the list of items - * where ID mod $groups = $group, in the given $state and optional $class filter. - * - * @param LoggerInterface $logger - * @param SolrIndex $indexInstance - * @param array $state - * @param string $class - * @param int $groups - * @param int $group - */ - public function runGroup(LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group); + /** + * Do an immediate re-index on the given group, where the group is defined as the list of items + * where ID mod $groups = $group, in the given $state and optional $class filter. + * + * @param LoggerInterface $logger + * @param SolrIndex $indexInstance + * @param array $state + * @param string $class + * @param int $groups + * @param int $group + */ + public function runGroup(LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group); } diff --git a/code/solr/reindex/handlers/SolrReindexImmediateHandler.php b/code/solr/reindex/handlers/SolrReindexImmediateHandler.php index 3ddc3ed..fc1bf59 100644 --- a/code/solr/reindex/handlers/SolrReindexImmediateHandler.php +++ b/code/solr/reindex/handlers/SolrReindexImmediateHandler.php @@ -7,68 +7,69 @@ use Psr\Log\LoggerInterface; * * Internally batches of records will be invoked via shell tasks in the background */ -class SolrReindexImmediateHandler extends SolrReindexBase { +class SolrReindexImmediateHandler extends SolrReindexBase +{ + public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) + { + $this->runReindex($logger, $batchSize, $taskName, $classes); + } - public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { - $this->runReindex($logger, $batchSize, $taskName, $classes); - } + protected function processIndex( + LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null + ) { + parent::processIndex($logger, $indexInstance, $batchSize, $taskName, $classes); - protected function processIndex( - LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null - ) { - parent::processIndex($logger, $indexInstance, $batchSize, $taskName, $classes); + // Immediate processor needs to immediately commit after each index + $indexInstance->getService()->commit(); + } - // Immediate processor needs to immediately commit after each index - $indexInstance->getService()->commit(); - } + /** + * Process a single group. + * + * Without queuedjobs, it's necessary to shell this out to a background task as this is + * very memory intensive. + * + * The sub-process will then invoke $processor->runGroup() in {@see Solr_Reindex::doReindex} + * + * @param LoggerInterface $logger + * @param SolrIndex $indexInstance Index instance + * @param array $state Variant state + * @param string $class Class to index + * @param int $groups Total groups + * @param int $group Index of group to process + * @param string $taskName Name of task script to run + */ + protected function processGroup( + LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName + ) { + // Build state + $statevar = json_encode($state); + if (strpos(PHP_OS, "WIN") !== false) { + $statevar = '"'.str_replace('"', '\\"', $statevar).'"'; + } else { + $statevar = "'".$statevar."'"; + } - /** - * Process a single group. - * - * Without queuedjobs, it's necessary to shell this out to a background task as this is - * very memory intensive. - * - * The sub-process will then invoke $processor->runGroup() in {@see Solr_Reindex::doReindex} - * - * @param LoggerInterface $logger - * @param SolrIndex $indexInstance Index instance - * @param array $state Variant state - * @param string $class Class to index - * @param int $groups Total groups - * @param int $group Index of group to process - * @param string $taskName Name of task script to run - */ - protected function processGroup( - LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName - ) { - // Build state - $statevar = json_encode($state); - if (strpos(PHP_OS, "WIN") !== false) { - $statevar = '"'.str_replace('"', '\\"', $statevar).'"'; - } else { - $statevar = "'".$statevar."'"; - } + // Build script + $indexName = $indexInstance->getIndexName(); + $scriptPath = sprintf("%s%sframework%scli-script.php", BASE_PATH, DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR); + $scriptTask = "php {$scriptPath} dev/tasks/{$taskName}"; + $cmd = "{$scriptTask} index={$indexName} class={$class} group={$group} groups={$groups} variantstate={$statevar}"; + $cmd .= " verbose=1 2>&1"; + $logger->info("Running '$cmd'"); - // Build script - $indexName = $indexInstance->getIndexName(); - $scriptPath = sprintf("%s%sframework%scli-script.php", BASE_PATH, DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR); - $scriptTask = "php {$scriptPath} dev/tasks/{$taskName}"; - $cmd = "{$scriptTask} index={$indexName} class={$class} group={$group} groups={$groups} variantstate={$statevar}"; - $cmd .= " verbose=1 2>&1"; - $logger->info("Running '$cmd'"); + // Execute script via shell + $res = $logger ? passthru($cmd) : `$cmd`; + if ($logger) { + $logger->info(preg_replace('/\r\n|\n/', '$0 ', $res)); + } - // Execute script via shell - $res = $logger ? passthru($cmd) : `$cmd`; - if($logger) { - $logger->info(preg_replace('/\r\n|\n/', '$0 ', $res)); - } + // If we're in dev mode, commit more often for fun and profit + if (Director::isDev()) { + Solr::service($indexName)->commit(); + } - // If we're in dev mode, commit more often for fun and profit - if (Director::isDev()) { - Solr::service($indexName)->commit(); - } - - // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex - DB::query('SELECT 1'); - } + // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex + DB::query('SELECT 1'); + } } diff --git a/code/solr/reindex/handlers/SolrReindexMessageHandler.php b/code/solr/reindex/handlers/SolrReindexMessageHandler.php index 0631e2c..9ffd4ea 100644 --- a/code/solr/reindex/handlers/SolrReindexMessageHandler.php +++ b/code/solr/reindex/handlers/SolrReindexMessageHandler.php @@ -2,39 +2,43 @@ use Psr\Log\LoggerInterface; -if(!class_exists('MessageQueue')) return; - -class SolrReindexMessageHandler extends SolrReindexImmediateHandler { - - /** - * The MessageQueue to use when processing updates - * @config - * @var string - */ - private static $reindex_queue = "search_indexing"; - - public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { - $queue = Config::inst()->get(__CLASS__, 'reindex_queue'); - - $logger->info('Queuing message'); - MessageQueue::send( - $queue, - new MethodInvocationMessage('SolrReindexMessageHandler', 'run_reindex', $batchSize, $taskName, $classes) - ); - } - - /** - * Entry point for message queue - * - * @param int $batchSize - * @param string $taskName - * @param array|string|null $classes - */ - public static function run_reindex($batchSize, $taskName, $classes = null) { - // @todo Logger for message queue? - $logger = Injector::inst()->createWithArgs('Monolog\Logger', array(strtolower(get_class()))); - - $inst = Injector::inst()->get(get_class()); - $inst->runReindex($logger, $batchSize, $taskName, $classes); - } +if (!class_exists('MessageQueue')) { + return; +} + +class SolrReindexMessageHandler extends SolrReindexImmediateHandler +{ + /** + * The MessageQueue to use when processing updates + * @config + * @var string + */ + private static $reindex_queue = "search_indexing"; + + public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) + { + $queue = Config::inst()->get(__CLASS__, 'reindex_queue'); + + $logger->info('Queuing message'); + MessageQueue::send( + $queue, + new MethodInvocationMessage('SolrReindexMessageHandler', 'run_reindex', $batchSize, $taskName, $classes) + ); + } + + /** + * Entry point for message queue + * + * @param int $batchSize + * @param string $taskName + * @param array|string|null $classes + */ + public static function run_reindex($batchSize, $taskName, $classes = null) + { + // @todo Logger for message queue? + $logger = Injector::inst()->createWithArgs('Monolog\Logger', array(strtolower(get_class()))); + + $inst = Injector::inst()->get(get_class()); + $inst->runReindex($logger, $batchSize, $taskName, $classes); + } } diff --git a/code/solr/reindex/handlers/SolrReindexQueuedHandler.php b/code/solr/reindex/handlers/SolrReindexQueuedHandler.php index 388f0de..25e4b8f 100644 --- a/code/solr/reindex/handlers/SolrReindexQueuedHandler.php +++ b/code/solr/reindex/handlers/SolrReindexQueuedHandler.php @@ -2,93 +2,97 @@ use Psr\Log\LoggerInterface; -if(!interface_exists('QueuedJob')) return; +if (!interface_exists('QueuedJob')) { + return; +} /** * Represents a queued task to start the reindex job */ -class SolrReindexQueuedHandler extends SolrReindexBase { +class SolrReindexQueuedHandler extends SolrReindexBase +{ + /** + * @return QueuedJobService + */ + protected function getQueuedJobService() + { + return singleton('QueuedJobService'); + } - /** - * @return QueuedJobService - */ - protected function getQueuedJobService() { - return singleton('QueuedJobService'); - } + /** + * Cancel any cancellable jobs + * + * @param string $type Type of job to cancel + * @return int Number of jobs cleared + */ + protected function cancelExistingJobs($type) + { + $clearable = array( + // Paused jobs need to be discarded + QueuedJob::STATUS_PAUSED, + + // These types would be automatically started + QueuedJob::STATUS_NEW, + QueuedJob::STATUS_WAIT, - /** - * Cancel any cancellable jobs - * - * @param string $type Type of job to cancel - * @return int Number of jobs cleared - */ - protected function cancelExistingJobs($type) { - $clearable = array( - // Paused jobs need to be discarded - QueuedJob::STATUS_PAUSED, - - // These types would be automatically started - QueuedJob::STATUS_NEW, - QueuedJob::STATUS_WAIT, + // Cancel any in-progress job + QueuedJob::STATUS_INIT, + QueuedJob::STATUS_RUN + ); + DB::query(sprintf( + 'UPDATE "QueuedJobDescriptor" ' + . ' SET "JobStatus" = \'%s\'' + . ' WHERE "JobStatus" IN (\'%s\')' + . ' AND "Implementation" = \'%s\'', + Convert::raw2sql(QueuedJob::STATUS_CANCELLED), + implode("','", Convert::raw2sql($clearable)), + Convert::raw2sql($type) + )); + return DB::affectedRows(); + } - // Cancel any in-progress job - QueuedJob::STATUS_INIT, - QueuedJob::STATUS_RUN - ); - DB::query(sprintf( - 'UPDATE "QueuedJobDescriptor" ' - . ' SET "JobStatus" = \'%s\'' - . ' WHERE "JobStatus" IN (\'%s\')' - . ' AND "Implementation" = \'%s\'', - Convert::raw2sql(QueuedJob::STATUS_CANCELLED), - implode("','", Convert::raw2sql($clearable)), - Convert::raw2sql($type) - )); - return DB::affectedRows(); - } + public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) + { + // Cancel existing jobs + $queues = $this->cancelExistingJobs('SolrReindexQueuedJob'); + $groups = $this->cancelExistingJobs('SolrReindexGroupQueuedJob'); + $logger->info("Cancelled {$queues} re-index tasks and {$groups} re-index groups"); - public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { - // Cancel existing jobs - $queues = $this->cancelExistingJobs('SolrReindexQueuedJob'); - $groups = $this->cancelExistingJobs('SolrReindexGroupQueuedJob'); - $logger->info("Cancelled {$queues} re-index tasks and {$groups} re-index groups"); + // Although this class is used as a service (singleton) it may also be instantiated + // as a queuedjob + $job = Injector::inst()->create('SolrReindexQueuedJob', $batchSize, $taskName, $classes); + $this + ->getQueuedJobService() + ->queueJob($job); - // Although this class is used as a service (singleton) it may also be instantiated - // as a queuedjob - $job = Injector::inst()->create('SolrReindexQueuedJob', $batchSize, $taskName, $classes); - $this - ->getQueuedJobService() - ->queueJob($job); + $title = $job->getTitle(); + $logger->info("Queued {$title}"); + } - $title = $job->getTitle(); - $logger->info("Queued {$title}"); - } + protected function processGroup( + LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName + ) { + // Trigger another job for this group + $job = Injector::inst()->create( + 'SolrReindexGroupQueuedJob', + $indexInstance->getIndexName(), $state, $class, $groups, $group + ); + $this + ->getQueuedJobService() + ->queueJob($job); + + $title = $job->getTitle(); + $logger->info("Queued {$title}"); + } - protected function processGroup( - LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName - ) { - // Trigger another job for this group - $job = Injector::inst()->create( - 'SolrReindexGroupQueuedJob', - $indexInstance->getIndexName(), $state, $class, $groups, $group - ); - $this - ->getQueuedJobService() - ->queueJob($job); - - $title = $job->getTitle(); - $logger->info("Queued {$title}"); - } - - public function runGroup( - LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group - ) { - parent::runGroup($logger, $indexInstance, $state, $class, $groups, $group); - - // After any changes have been made, mark all indexes as dirty for commit - // see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers - $logger->info("Queuing commit on all changes"); - SearchUpdateCommitJobProcessor::queue(); - } + public function runGroup( + LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group + ) { + parent::runGroup($logger, $indexInstance, $state, $class, $groups, $group); + // After any changes have been made, mark all indexes as dirty for commit + // see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers + $logger->info("Queuing commit on all changes"); + SearchUpdateCommitJobProcessor::queue(); + } } diff --git a/code/solr/reindex/jobs/SolrReindexGroupQueuedJob.php b/code/solr/reindex/jobs/SolrReindexGroupQueuedJob.php index 666f24d..7a0eb6b 100644 --- a/code/solr/reindex/jobs/SolrReindexGroupQueuedJob.php +++ b/code/solr/reindex/jobs/SolrReindexGroupQueuedJob.php @@ -1,6 +1,8 @@ indexName = $indexName; + $this->state = $state; + $this->class = $class; + $this->groups = $groups; + $this->group = $group; + } - public function __construct($indexName = null, $state = null, $class = null, $groups = null, $group = null) { - parent::__construct(); - $this->indexName = $indexName; - $this->state = $state; - $this->class = $class; - $this->groups = $groups; - $this->group = $group; - } + public function getJobData() + { + $data = parent::getJobData(); - public function getJobData() { - $data = parent::getJobData(); + // Custom data + $data->jobData->indexName = $this->indexName; + $data->jobData->state = $this->state; + $data->jobData->class = $this->class; + $data->jobData->groups = $this->groups; + $data->jobData->group = $this->group; + + return $data; + } - // Custom data - $data->jobData->indexName = $this->indexName; - $data->jobData->state = $this->state; - $data->jobData->class = $this->class; - $data->jobData->groups = $this->groups; - $data->jobData->group = $this->group; - - return $data; - } + public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) + { + parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages); - public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { - parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages); + // Custom data + $this->indexName = $jobData->indexName; + $this->state = $jobData->state; + $this->class = $jobData->class; + $this->groups = $jobData->groups; + $this->group = $jobData->group; + } - // Custom data - $this->indexName = $jobData->indexName; - $this->state = $jobData->state; - $this->class = $jobData->class; - $this->groups = $jobData->groups; - $this->group = $jobData->group; - } + public function getSignature() + { + return md5(get_class($this) . time() . mt_rand(0, 100000)); + } - public function getSignature() { - return md5(get_class($this) . time() . mt_rand(0, 100000)); - } + public function getTitle() + { + return sprintf( + 'Solr Reindex Group (%d/%d) of %s in %s', + ($this->group+1), + $this->groups, + $this->class, + json_encode($this->state) + ); + } - public function getTitle() { - return sprintf( - 'Solr Reindex Group (%d/%d) of %s in %s', - ($this->group+1), - $this->groups, - $this->class, - json_encode($this->state) - ); - } + public function process() + { + $logger = $this->getLogger(); + if ($this->jobFinished()) { + $logger->notice("reindex group already complete"); + return; + } - public function process() { - $logger = $this->getLogger(); - if($this->jobFinished()) { - $logger->notice("reindex group already complete"); - return; - } - - // Get instance of index - $indexInstance = singleton($this->indexName); - - // Send back to processor - $logger->info("Beginning reindex group"); - $this - ->getHandler() - ->runGroup($logger, $indexInstance, $this->state, $this->class, $this->groups, $this->group); - $logger->info("Completed reindex group"); - $this->isComplete = true; - } + // Get instance of index + $indexInstance = singleton($this->indexName); + // Send back to processor + $logger->info("Beginning reindex group"); + $this + ->getHandler() + ->runGroup($logger, $indexInstance, $this->state, $this->class, $this->groups, $this->group); + $logger->info("Completed reindex group"); + $this->isComplete = true; + } } diff --git a/code/solr/reindex/jobs/SolrReindexQueuedJob.php b/code/solr/reindex/jobs/SolrReindexQueuedJob.php index b9c4250..ce46783 100644 --- a/code/solr/reindex/jobs/SolrReindexQueuedJob.php +++ b/code/solr/reindex/jobs/SolrReindexQueuedJob.php @@ -1,91 +1,100 @@ batchSize = $batchSize; + $this->taskName = $taskName; + $this->classes = $classes; + parent::__construct(); + } - public function __construct($batchSize = null, $taskName = null, $classes = null) { - $this->batchSize = $batchSize; - $this->taskName = $taskName; - $this->classes = $classes; - parent::__construct(); - } + public function getJobData() + { + $data = parent::getJobData(); - public function getJobData() { - $data = parent::getJobData(); + // Custom data + $data->jobData->batchSize = $this->batchSize; + $data->jobData->taskName = $this->taskName; + $data->jobData->classes = $this->classes; + + return $data; + } - // Custom data - $data->jobData->batchSize = $this->batchSize; - $data->jobData->taskName = $this->taskName; - $data->jobData->classes = $this->classes; - - return $data; - } + public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) + { + parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages); - public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { - parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages); + // Custom data + $this->batchSize = $jobData->batchSize; + $this->taskName = $jobData->taskName; + $this->classes = $jobData->classes; + } - // Custom data - $this->batchSize = $jobData->batchSize; - $this->taskName = $jobData->taskName; - $this->classes = $jobData->classes; - } + public function getSignature() + { + return __CLASS__; + } - public function getSignature() { - return __CLASS__; - } + public function getTitle() + { + return 'Solr Reindex Job'; + } - public function getTitle() { - return 'Solr Reindex Job'; - } + public function process() + { + $logger = $this->getLogger(); + if ($this->jobFinished()) { + $logger->notice("reindex already complete"); + return; + } - public function process() { - $logger = $this->getLogger(); - if($this->jobFinished()) { - $logger->notice("reindex already complete"); - return; - } + // Send back to processor + $logger->info("Beginning init of reindex"); + $this + ->getHandler() + ->runReindex($logger, $this->batchSize, $this->taskName, $this->classes); + $logger->info("Completed init of reindex"); + $this->isComplete = true; + } - // Send back to processor - $logger->info("Beginning init of reindex"); - $this - ->getHandler() - ->runReindex($logger, $this->batchSize, $this->taskName, $this->classes); - $logger->info("Completed init of reindex"); - $this->isComplete = true; - } - - /** - * Get size of batch - * - * @return int - */ - public function getBatchSize() { - return $this->batchSize; - } + /** + * Get size of batch + * + * @return int + */ + public function getBatchSize() + { + return $this->batchSize; + } } diff --git a/code/solr/reindex/jobs/SolrReindexQueuedJobBase.php b/code/solr/reindex/jobs/SolrReindexQueuedJobBase.php index 74b569f..ef72c1d 100644 --- a/code/solr/reindex/jobs/SolrReindexQueuedJobBase.php +++ b/code/solr/reindex/jobs/SolrReindexQueuedJobBase.php @@ -3,121 +3,136 @@ use Monolog\Logger; use Psr\Log\LoggerInterface; -if(!interface_exists('QueuedJob')) return; +if (!interface_exists('QueuedJob')) { + return; +} /** * Base class for jobs which perform re-index */ -abstract class SolrReindexQueuedJobBase implements QueuedJob { +abstract class SolrReindexQueuedJobBase implements QueuedJob +{ + /** + * Flag whether this job is done + * + * @var bool + */ + protected $isComplete; - /** - * Flag whether this job is done - * - * @var bool - */ - protected $isComplete; + /** + * List of messages + * + * @var array + */ + protected $messages; - /** - * List of messages - * - * @var array - */ - protected $messages; + /** + * Logger to use for this job + * + * @var LoggerInterface + */ + protected $logger; - /** - * Logger to use for this job - * - * @var LoggerInterface - */ - protected $logger; + public function __construct() + { + $this->isComplete = false; + $this->messages = array(); + } - public function __construct() { - $this->isComplete = false; - $this->messages = array(); - } + /** + * @return SearchLogFactory + */ + protected function getLoggerFactory() + { + return Injector::inst()->get('SearchLogFactory'); + } - /** - * @return SearchLogFactory - */ - protected function getLoggerFactory() { - return Injector::inst()->get('SearchLogFactory'); - } + /** + * Gets a logger for this job + * + * @return LoggerInterface + */ + protected function getLogger() + { + if ($this->logger) { + return $this->logger; + } - /** - * Gets a logger for this job - * - * @return LoggerInterface - */ - protected function getLogger() { - if($this->logger) { - return $this->logger; - } + // Set logger for this job + $this->logger = $this + ->getLoggerFactory() + ->getQueuedJobLogger($this); + return $this->logger; + } - // Set logger for this job - $this->logger = $this - ->getLoggerFactory() - ->getQueuedJobLogger($this); - return $this->logger; - } + /** + * Assign custom logger for this job + * + * @param LoggerInterface $logger + */ + public function setLogger($logger) + { + $this->logger = $logger; + } - /** - * Assign custom logger for this job - * - * @param LoggerInterface $logger - */ - public function setLogger($logger) { - $this->logger = $logger; - } + public function getJobData() + { + $data = new stdClass(); - public function getJobData() { - $data = new stdClass(); + // Standard fields + $data->totalSteps = 1; + $data->currentStep = $this->isComplete ? 0 : 1; + $data->isComplete = $this->isComplete; + $data->messages = $this->messages; - // Standard fields - $data->totalSteps = 1; - $data->currentStep = $this->isComplete ? 0 : 1; - $data->isComplete = $this->isComplete; - $data->messages = $this->messages; + // Custom data + $data->jobData = new stdClass(); + return $data; + } - // Custom data - $data->jobData = new stdClass(); - return $data; - } + public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) + { + $this->isComplete = $isComplete; + $this->messages = $messages; + } - public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { - $this->isComplete = $isComplete; - $this->messages = $messages; - } + /** + * Get the reindex handler + * + * @return SolrReindexHandler + */ + protected function getHandler() + { + return Injector::inst()->get('SolrReindexHandler'); + } - /** - * Get the reindex handler - * - * @return SolrReindexHandler - */ - protected function getHandler() { - return Injector::inst()->get('SolrReindexHandler'); - } + public function jobFinished() + { + return $this->isComplete; + } - public function jobFinished() { - return $this->isComplete; - } + public function prepareForRestart() + { + // NOOP + } - public function prepareForRestart() { - // NOOP - } + public function setup() + { + // NOOP + } - public function setup() { - // NOOP - } + public function afterComplete() + { + // NOOP + } - public function afterComplete() { - // NOOP - } + public function getJobType() + { + return QueuedJob::QUEUED; + } - public function getJobType() { - return QueuedJob::QUEUED; - } - - public function addMessage($message) { - $this->messages[] = $message; - } + public function addMessage($message) + { + $this->messages[] = $message; + } } diff --git a/code/utils/CombinationsArrayIterator.php b/code/utils/CombinationsArrayIterator.php index 4e038f7..69e5727 100644 --- a/code/utils/CombinationsArrayIterator.php +++ b/code/utils/CombinationsArrayIterator.php @@ -1,66 +1,80 @@ arrays = array(); - $this->keys = array(); + public function __construct($args) + { + $this->arrays = array(); + $this->keys = array(); - $keys = array_keys($args); - $values = array_values($args); + $keys = array_keys($args); + $values = array_values($args); - foreach ($values as $i => $arg) { - if (is_array($arg) && count($arg)) { - $this->arrays[] = $arg; - $this->keys[] = $keys[$i]; - } - } + foreach ($values as $i => $arg) { + if (is_array($arg) && count($arg)) { + $this->arrays[] = $arg; + $this->keys[] = $keys[$i]; + } + } - $this->numArrays = count($this->arrays); - $this->rewind(); - } + $this->numArrays = count($this->arrays); + $this->rewind(); + } - function rewind() { - if (!$this->numArrays) { - $this->isValid = false; - } - else { - $this->isValid = true; - $this->k = 0; - - for ($i = 0; $i < $this->numArrays; $i++) reset($this->arrays[$i]); - } - } + public function rewind() + { + if (!$this->numArrays) { + $this->isValid = false; + } else { + $this->isValid = true; + $this->k = 0; + + for ($i = 0; $i < $this->numArrays; $i++) { + reset($this->arrays[$i]); + } + } + } - function valid() { - return $this->isValid; - } + public function valid() + { + return $this->isValid; + } - function next() { - $this->k++; + public function next() + { + $this->k++; - for ($i = 0; $i < $this->numArrays; $i++) { - if (next($this->arrays[$i]) === false) { - if ($i == $this->numArrays-1) $this->isValid = false; - else reset($this->arrays[$i]); - } - else break; - } - } + for ($i = 0; $i < $this->numArrays; $i++) { + if (next($this->arrays[$i]) === false) { + if ($i == $this->numArrays-1) { + $this->isValid = false; + } else { + reset($this->arrays[$i]); + } + } else { + break; + } + } + } - function current() { - $res = array(); - for ($i = 0; $i < $this->numArrays; $i++) $res[$this->keys[$i]] = current($this->arrays[$i]); - return $res; - } + public function current() + { + $res = array(); + for ($i = 0; $i < $this->numArrays; $i++) { + $res[$this->keys[$i]] = current($this->arrays[$i]); + } + return $res; + } - function key() { - return $this->k; - } -} \ No newline at end of file + public function key() + { + return $this->k; + } +} diff --git a/code/utils/MultipleArrayIterator.php b/code/utils/MultipleArrayIterator.php index 9963db3..7b79850 100644 --- a/code/utils/MultipleArrayIterator.php +++ b/code/utils/MultipleArrayIterator.php @@ -1,44 +1,58 @@ arrays = array(); + foreach ($args as $arg) { + if (is_array($arg) && count($arg)) { + $this->arrays[] = $arg; + } + } - $this->arrays = array(); - foreach ($args as $arg) { - if (is_array($arg) && count($arg)) $this->arrays[] = $arg; - } + $this->rewind(); + } - $this->rewind(); - } + public function rewind() + { + $this->active = $this->arrays; + if ($this->active) { + reset($this->active[0]); + } + } - function rewind() { - $this->active = $this->arrays; - if ($this->active) reset($this->active[0]); - } + public function current() + { + return $this->active ? current($this->active[0]) : false; + } - function current() { - return $this->active ? current($this->active[0]) : false; - } + public function key() + { + return $this->active ? key($this->active[0]) : false; + } - function key() { - return $this->active ? key($this->active[0]) : false; - } + public function next() + { + if (!$this->active) { + return; + } - function next() { - if (!$this->active) return; + if (next($this->active[0]) === false) { + array_shift($this->active); + if ($this->active) { + reset($this->active[0]); + } + } + } - if (next($this->active[0]) === false) { - array_shift($this->active); - if ($this->active) reset($this->active[0]); - } - } - - function valid() { - return $this->active && (current($this->active[0]) !== false); - } + public function valid() + { + return $this->active && (current($this->active[0]) !== false); + } } diff --git a/code/utils/WebDAV.php b/code/utils/WebDAV.php index 268787d..8c9a92c 100644 --- a/code/utils/WebDAV.php +++ b/code/utils/WebDAV.php @@ -1,68 +1,76 @@ getLoggerFor($name); + $formatter = $this->getFormatter(); - public function getOutputLogger($name, $verbose) { - $logger = $this->getLoggerFor($name); - $formatter = $this->getFormatter(); + // Notice handling + if ($verbose) { + $messageHandler = $this->getStreamHandler($formatter, 'php://stdout', Logger::INFO); + $logger->pushHandler($messageHandler); + } - // Notice handling - if($verbose) { - $messageHandler = $this->getStreamHandler($formatter, 'php://stdout', Logger::INFO); - $logger->pushHandler($messageHandler); - } + // Error handling. buble is false so that errors aren't logged twice + $errorHandler = $this->getStreamHandler($formatter, 'php://stderr', Logger::ERROR, false); + $logger->pushHandler($errorHandler); + return $logger; + } - // Error handling. buble is false so that errors aren't logged twice - $errorHandler = $this->getStreamHandler($formatter, 'php://stderr', Logger::ERROR, false); - $logger->pushHandler($errorHandler); - return $logger; - } + public function getQueuedJobLogger($job) + { + $logger = $this->getLoggerFor(get_class($job)); + $handler = $this->getJobHandler($job); + $logger->pushHandler($handler); + return $logger; + } - public function getQueuedJobLogger($job) { - $logger = $this->getLoggerFor(get_class($job)); - $handler = $this->getJobHandler($job); - $logger->pushHandler($handler); - return $logger; - } + /** + * Generate a handler for the given stream + * + * @param FormatterInterface $formatter + * @param string $stream Name of preferred stream + * @param int $level + * @param bool $bubble + * @return HandlerInterface + */ + protected function getStreamHandler(FormatterInterface $formatter, $stream, $level = Logger::DEBUG, $bubble = true) + { + // Unless cli, force output to php://output + $stream = Director::is_cli() ? $stream : 'php://output'; + $handler = Injector::inst()->createWithArgs( + 'Monolog\Handler\StreamHandler', + array($stream, $level, $bubble) + ); + $handler->setFormatter($formatter); + return $handler; + } - /** - * Generate a handler for the given stream - * - * @param FormatterInterface $formatter - * @param string $stream Name of preferred stream - * @param int $level - * @param bool $bubble - * @return HandlerInterface - */ - protected function getStreamHandler(FormatterInterface $formatter, $stream, $level = Logger::DEBUG, $bubble = true) { - // Unless cli, force output to php://output - $stream = Director::is_cli() ? $stream : 'php://output'; - $handler = Injector::inst()->createWithArgs( - 'Monolog\Handler\StreamHandler', - array($stream, $level, $bubble) - ); - $handler->setFormatter($formatter); - return $handler; - } + /** + * Gets a formatter for standard output + * + * @return FormatterInterface + */ + protected function getFormatter() + { + // Get formatter + $format = LineFormatter::SIMPLE_FORMAT; + if (!Director::is_cli()) { + $format = "

$format

"; + } + return Injector::inst()->createWithArgs( + 'Monolog\Formatter\LineFormatter', + array($format) + ); + } - /** - * Gets a formatter for standard output - * - * @return FormatterInterface - */ - protected function getFormatter() { - // Get formatter - $format = LineFormatter::SIMPLE_FORMAT; - if(!Director::is_cli()) { - $format = "

$format

"; - } - return Injector::inst()->createWithArgs( - 'Monolog\Formatter\LineFormatter', - array($format) - ); - } + /** + * Get a logger for a named class + * + * @param string $name + * @return Logger + */ + protected function getLoggerFor($name) + { + return Injector::inst()->createWithArgs( + 'Monolog\Logger', + array(strtolower($name)) + ); + } - /** - * Get a logger for a named class - * - * @param string $name - * @return Logger - */ - protected function getLoggerFor($name) { - return Injector::inst()->createWithArgs( - 'Monolog\Logger', - array(strtolower($name)) - ); - } - - /** - * Generate handler for a job object - * - * @param QueuedJob $job - * @return HandlerInterface - */ - protected function getJobHandler($job) { - return Injector::inst()->createWithArgs( - 'QueuedJobLogHandler', - array($job, Logger::INFO) - ); - } + /** + * Generate handler for a job object + * + * @param QueuedJob $job + * @return HandlerInterface + */ + protected function getJobHandler($job) + { + return Injector::inst()->createWithArgs( + 'QueuedJobLogHandler', + array($job, Logger::INFO) + ); + } } diff --git a/code/utils/logging/QueuedJobLogHandler.php b/code/utils/logging/QueuedJobLogHandler.php index 666cdfd..9b90b51 100644 --- a/code/utils/logging/QueuedJobLogHandler.php +++ b/code/utils/logging/QueuedJobLogHandler.php @@ -3,51 +3,56 @@ use Monolog\Handler\AbstractProcessingHandler; use Monolog\Logger; -if(!interface_exists('QueuedJob')) return; +if (!interface_exists('QueuedJob')) { + return; +} /** * Handler for logging events into QueuedJob message data */ -class QueuedJobLogHandler extends AbstractProcessingHandler { +class QueuedJobLogHandler extends AbstractProcessingHandler +{ + /** + * Job to log to + * + * @var QueuedJob + */ + protected $queuedJob; - /** - * Job to log to - * - * @var QueuedJob - */ - protected $queuedJob; + /** + * @param QueuedJob $queuedJob Job to log to + * @param integer $level The minimum logging level at which this handler will be triggered + * @param Boolean $bubble Whether the messages that are handled can bubble up the stack or not + */ + public function __construct(QueuedJob $queuedJob, $level = Logger::DEBUG, $bubble = true) + { + parent::__construct($level, $bubble); + $this->setQueuedJob($queuedJob); + } - /** - * @param QueuedJob $queuedJob Job to log to - * @param integer $level The minimum logging level at which this handler will be triggered - * @param Boolean $bubble Whether the messages that are handled can bubble up the stack or not - */ - public function __construct(QueuedJob $queuedJob, $level = Logger::DEBUG, $bubble = true) { - parent::__construct($level, $bubble); - $this->setQueuedJob($queuedJob); - } + /** + * Set a new queuedjob + * + * @param QueuedJob $queuedJob + */ + public function setQueuedJob(QueuedJob $queuedJob) + { + $this->queuedJob = $queuedJob; + } - /** - * Set a new queuedjob - * - * @param QueuedJob $queuedJob - */ - public function setQueuedJob(QueuedJob $queuedJob) { - $this->queuedJob = $queuedJob; - } - - /** - * Get queuedjob - * - * @return QueuedJob - */ - public function getQueuedJob() { - return $this->queuedJob; - } - - protected function write(array $record) { - // Write formatted message - $this->getQueuedJob()->addMessage($record['formatted']); - } + /** + * Get queuedjob + * + * @return QueuedJob + */ + public function getQueuedJob() + { + return $this->queuedJob; + } + protected function write(array $record) + { + // Write formatted message + $this->getQueuedJob()->addMessage($record['formatted']); + } } diff --git a/code/utils/logging/SearchLogFactory.php b/code/utils/logging/SearchLogFactory.php index 6296691..ac56df8 100644 --- a/code/utils/logging/SearchLogFactory.php +++ b/code/utils/logging/SearchLogFactory.php @@ -2,22 +2,22 @@ use Psr\Log; -interface SearchLogFactory { +interface SearchLogFactory +{ + /** + * Make a logger for a queuedjob + * + * @param QueuedJob $job + * @return Log + */ + public function getQueuedJobLogger($job); - /** - * Make a logger for a queuedjob - * - * @param QueuedJob $job - * @return Log - */ - public function getQueuedJobLogger($job); - - /** - * Get an output logger with the given verbosity - * - * @param string $name - * @param bool $verbose - * @return Log - */ - public function getOutputLogger($name, $verbose); + /** + * Get an output logger with the given verbosity + * + * @param string $name + * @param bool $verbose + * @return Log + */ + public function getOutputLogger($name, $verbose); } diff --git a/tests/BatchedProcessorTest.php b/tests/BatchedProcessorTest.php index 76e4460..65d7c55 100644 --- a/tests/BatchedProcessorTest.php +++ b/tests/BatchedProcessorTest.php @@ -1,246 +1,259 @@ 'Varchar' - ); +class BatchedProcessorTest_Object extends SiteTree implements TestOnly +{ + private static $db = array( + 'TestText' => 'Varchar' + ); } -class BatchedProcessorTest_Index extends SearchIndex_Recording implements TestOnly { - function init() { - $this->addClass('BatchedProcessorTest_Object'); - $this->addFilterField('TestText'); - } +class BatchedProcessorTest_Index extends SearchIndex_Recording implements TestOnly +{ + public function init() + { + $this->addClass('BatchedProcessorTest_Object'); + $this->addFilterField('TestText'); + } } -class BatchedProcessor_QueuedJobService { - protected $jobs = array(); +class BatchedProcessor_QueuedJobService +{ + protected $jobs = array(); - public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null) { - $this->jobs[] = array( - 'job' => $job, - 'startAfter' => $startAfter - ); - return $job; - } + public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null) + { + $this->jobs[] = array( + 'job' => $job, + 'startAfter' => $startAfter + ); + return $job; + } - public function getJobs() { - return $this->jobs; - } + public function getJobs() + { + return $this->jobs; + } } /** * Tests {@see SearchUpdateQueuedJobProcessor} */ -class BatchedProcessorTest extends SapphireTest { - - protected $oldProcessor; - - protected $extraDataObjects = array( - 'BatchedProcessorTest_Object' - ); +class BatchedProcessorTest extends SapphireTest +{ + protected $oldProcessor; + + protected $extraDataObjects = array( + 'BatchedProcessorTest_Object' + ); - protected $illegalExtensions = array( - 'SiteTree' => array( - 'SiteTreeSubsites', - 'Translatable' - ) - ); + protected $illegalExtensions = array( + 'SiteTree' => array( + 'SiteTreeSubsites', + 'Translatable' + ) + ); - public function setUpOnce() { - // Disable illegal extensions if skipping this test - if(class_exists('Subsite') || !interface_exists('QueuedJob')) { - $this->illegalExtensions = array(); - } - parent::setUpOnce(); - } + public function setUpOnce() + { + // Disable illegal extensions if skipping this test + if (class_exists('Subsite') || !interface_exists('QueuedJob')) { + $this->illegalExtensions = array(); + } + parent::setUpOnce(); + } - public function setUp() { - parent::setUp(); - Config::nest(); - - if (!interface_exists('QueuedJob')) { - $this->skipTest = true; - $this->markTestSkipped("These tests need the QueuedJobs module installed to run"); - } + public function setUp() + { + parent::setUp(); + Config::nest(); + + if (!interface_exists('QueuedJob')) { + $this->skipTest = true; + $this->markTestSkipped("These tests need the QueuedJobs module installed to run"); + } - if(class_exists('Subsite')) { - $this->skipTest = true; - $this->markTestSkipped(get_class() . ' skipped when running with subsites'); - } + if (class_exists('Subsite')) { + $this->skipTest = true; + $this->markTestSkipped(get_class() . ' skipped when running with subsites'); + } - SS_Datetime::set_mock_now('2015-05-07 06:00:00'); - - Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_size', 5); - Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 0); - Config::inst()->update('SearchUpdateCommitJobProcessor', 'cooldown', 600); - - Versioned::reading_stage("Stage"); + SS_Datetime::set_mock_now('2015-05-07 06:00:00'); + + Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_size', 5); + Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 0); + Config::inst()->update('SearchUpdateCommitJobProcessor', 'cooldown', 600); + + Versioned::reading_stage("Stage"); - Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), 'QueuedJobService'); + Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), 'QueuedJobService'); - FullTextSearch::force_index_list('BatchedProcessorTest_Index'); + FullTextSearch::force_index_list('BatchedProcessorTest_Index'); - SearchUpdateCommitJobProcessor::$dirty_indexes = array(); - SearchUpdateCommitJobProcessor::$has_run = false; - - $this->oldProcessor = SearchUpdater::$processor; - SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor(); - } + SearchUpdateCommitJobProcessor::$dirty_indexes = array(); + SearchUpdateCommitJobProcessor::$has_run = false; + + $this->oldProcessor = SearchUpdater::$processor; + SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor(); + } - public function tearDown() { - if($this->oldProcessor) { - SearchUpdater::$processor = $this->oldProcessor; - } - Config::unnest(); - Injector::inst()->unregisterNamedObject('QueuedJobService'); - FullTextSearch::force_index_list(); - parent::tearDown(); - } + public function tearDown() + { + if ($this->oldProcessor) { + SearchUpdater::$processor = $this->oldProcessor; + } + Config::unnest(); + Injector::inst()->unregisterNamedObject('QueuedJobService'); + FullTextSearch::force_index_list(); + parent::tearDown(); + } - /** - * @return SearchUpdateQueuedJobProcessor - */ - protected function generateDirtyIds() { - $processor = SearchUpdater::$processor; - for($id = 1; $id <= 42; $id++) { - // Save to db - $object = new BatchedProcessorTest_Object(); - $object->TestText = 'Object ' . $id; - $object->write(); - // Add to index manually - $processor->addDirtyIDs( - 'BatchedProcessorTest_Object', - array(array( - 'id' => $id, - 'state' => array('SearchVariantVersioned' => 'Stage') - )), - 'BatchedProcessorTest_Index' - ); - } - $processor->batchData(); - return $processor; - } - - /** - * Tests that large jobs are broken up into a suitable number of batches - */ - public function testBatching() { - $index = singleton('BatchedProcessorTest_Index'); - $index->reset(); - $processor = $this->generateDirtyIds(); - - // Check initial state - $data = $processor->getJobData(); - $this->assertEquals(9, $data->totalSteps); - $this->assertEquals(0, $data->currentStep); - $this->assertEmpty($data->isComplete); - $this->assertEquals(0, count($index->getAdded())); - - // Advance state - for($pass = 1; $pass <= 8; $pass++) { - $processor->process(); - $data = $processor->getJobData(); - $this->assertEquals($pass, $data->currentStep); - $this->assertEquals($pass * 5, count($index->getAdded())); - } - - // Last run should have two hanging items - $processor->process(); - $data = $processor->getJobData(); - $this->assertEquals(9, $data->currentStep); - $this->assertEquals(42, count($index->getAdded())); - $this->assertTrue($data->isComplete); + /** + * @return SearchUpdateQueuedJobProcessor + */ + protected function generateDirtyIds() + { + $processor = SearchUpdater::$processor; + for ($id = 1; $id <= 42; $id++) { + // Save to db + $object = new BatchedProcessorTest_Object(); + $object->TestText = 'Object ' . $id; + $object->write(); + // Add to index manually + $processor->addDirtyIDs( + 'BatchedProcessorTest_Object', + array(array( + 'id' => $id, + 'state' => array('SearchVariantVersioned' => 'Stage') + )), + 'BatchedProcessorTest_Index' + ); + } + $processor->batchData(); + return $processor; + } + + /** + * Tests that large jobs are broken up into a suitable number of batches + */ + public function testBatching() + { + $index = singleton('BatchedProcessorTest_Index'); + $index->reset(); + $processor = $this->generateDirtyIds(); + + // Check initial state + $data = $processor->getJobData(); + $this->assertEquals(9, $data->totalSteps); + $this->assertEquals(0, $data->currentStep); + $this->assertEmpty($data->isComplete); + $this->assertEquals(0, count($index->getAdded())); + + // Advance state + for ($pass = 1; $pass <= 8; $pass++) { + $processor->process(); + $data = $processor->getJobData(); + $this->assertEquals($pass, $data->currentStep); + $this->assertEquals($pass * 5, count($index->getAdded())); + } + + // Last run should have two hanging items + $processor->process(); + $data = $processor->getJobData(); + $this->assertEquals(9, $data->currentStep); + $this->assertEquals(42, count($index->getAdded())); + $this->assertTrue($data->isComplete); - // Check any additional queued jobs - $processor->afterComplete(); - $service = singleton('QueuedJobService'); - $jobs = $service->getJobs(); - $this->assertEquals(1, count($jobs)); - $this->assertInstanceOf('SearchUpdateCommitJobProcessor', $jobs[0]['job']); - } + // Check any additional queued jobs + $processor->afterComplete(); + $service = singleton('QueuedJobService'); + $jobs = $service->getJobs(); + $this->assertEquals(1, count($jobs)); + $this->assertInstanceOf('SearchUpdateCommitJobProcessor', $jobs[0]['job']); + } - /** - * Test creation of multiple commit jobs - */ - public function testMultipleCommits() { - $index = singleton('BatchedProcessorTest_Index'); - $index->reset(); + /** + * Test creation of multiple commit jobs + */ + public function testMultipleCommits() + { + $index = singleton('BatchedProcessorTest_Index'); + $index->reset(); - // Test that running a commit immediately after submitting to the indexes - // correctly commits - $first = SearchUpdateCommitJobProcessor::queue(); - $second = SearchUpdateCommitJobProcessor::queue(); + // Test that running a commit immediately after submitting to the indexes + // correctly commits + $first = SearchUpdateCommitJobProcessor::queue(); + $second = SearchUpdateCommitJobProcessor::queue(); - $this->assertFalse($index->getIsCommitted()); + $this->assertFalse($index->getIsCommitted()); - // First process will cause the commit - $this->assertFalse($first->jobFinished()); - $first->process(); - $allMessages = $first->getMessages(); - $this->assertTrue($index->getIsCommitted()); - $this->assertTrue($first->jobFinished()); - $this->assertStringEndsWith('All indexes committed', $allMessages[2]); + // First process will cause the commit + $this->assertFalse($first->jobFinished()); + $first->process(); + $allMessages = $first->getMessages(); + $this->assertTrue($index->getIsCommitted()); + $this->assertTrue($first->jobFinished()); + $this->assertStringEndsWith('All indexes committed', $allMessages[2]); - // Executing the subsequent processor should not re-trigger a commit - $index->reset(); - $this->assertFalse($second->jobFinished()); - $second->process(); - $allMessages = $second->getMessages(); - $this->assertFalse($index->getIsCommitted()); - $this->assertTrue($second->jobFinished()); - $this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]); + // Executing the subsequent processor should not re-trigger a commit + $index->reset(); + $this->assertFalse($second->jobFinished()); + $second->process(); + $allMessages = $second->getMessages(); + $this->assertFalse($index->getIsCommitted()); + $this->assertTrue($second->jobFinished()); + $this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]); - // Given that a third job is created, and the indexes are dirtied, attempting to run this job - // should result in a delay - $index->reset(); - $third = SearchUpdateCommitJobProcessor::queue(); - $this->assertFalse($third->jobFinished()); - $third->process(); - $this->assertTrue($third->jobFinished()); - $allMessages = $third->getMessages(); - $this->assertStringEndsWith( - 'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00', - $allMessages[0] - ); - } + // Given that a third job is created, and the indexes are dirtied, attempting to run this job + // should result in a delay + $index->reset(); + $third = SearchUpdateCommitJobProcessor::queue(); + $this->assertFalse($third->jobFinished()); + $third->process(); + $this->assertTrue($third->jobFinished()); + $allMessages = $third->getMessages(); + $this->assertStringEndsWith( + 'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00', + $allMessages[0] + ); + } - - /** - * Tests that the batch_soft_cap setting is properly respected - */ - public function testSoftCap() { - $index = singleton('BatchedProcessorTest_Index'); - $index->reset(); - $processor = $this->generateDirtyIds(); - - // Test that increasing the soft cap to 2 will reduce the number of batches - Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 2); - $processor->batchData(); - $data = $processor->getJobData(); - //Debug::dump($data);die; - $this->assertEquals(8, $data->totalSteps); - - // A soft cap of 1 should not fit in the hanging two items - Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 1); - $processor->batchData(); - $data = $processor->getJobData(); - $this->assertEquals(9, $data->totalSteps); - - // Extra large soft cap should fit both items - Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 4); - $processor->batchData(); - $data = $processor->getJobData(); - $this->assertEquals(8, $data->totalSteps); - - // Process all data and ensure that all are processed adequately - for($pass = 1; $pass <= 8; $pass++) { - $processor->process(); - } - $data = $processor->getJobData(); - $this->assertEquals(8, $data->currentStep); - $this->assertEquals(42, count($index->getAdded())); - $this->assertTrue($data->isComplete); - } + + /** + * Tests that the batch_soft_cap setting is properly respected + */ + public function testSoftCap() + { + $index = singleton('BatchedProcessorTest_Index'); + $index->reset(); + $processor = $this->generateDirtyIds(); + + // Test that increasing the soft cap to 2 will reduce the number of batches + Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 2); + $processor->batchData(); + $data = $processor->getJobData(); + //Debug::dump($data);die; + $this->assertEquals(8, $data->totalSteps); + + // A soft cap of 1 should not fit in the hanging two items + Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 1); + $processor->batchData(); + $data = $processor->getJobData(); + $this->assertEquals(9, $data->totalSteps); + + // Extra large soft cap should fit both items + Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 4); + $processor->batchData(); + $data = $processor->getJobData(); + $this->assertEquals(8, $data->totalSteps); + + // Process all data and ensure that all are processed adequately + for ($pass = 1; $pass <= 8; $pass++) { + $processor->process(); + } + $data = $processor->getJobData(); + $this->assertEquals(8, $data->currentStep); + $this->assertEquals(42, count($index->getAdded())); + $this->assertTrue($data->isComplete); + } } diff --git a/tests/SearchUpdaterTest.php b/tests/SearchUpdaterTest.php index ea1c371..dd0e006 100644 --- a/tests/SearchUpdaterTest.php +++ b/tests/SearchUpdaterTest.php @@ -1,215 +1,229 @@ 'Varchar', - 'Field2' => 'Varchar', - 'MyDate' => 'Date', - ); +class SearchUpdaterTest_Container extends DataObject +{ + private static $db = array( + 'Field1' => 'Varchar', + 'Field2' => 'Varchar', + 'MyDate' => 'Date', + ); - private static $has_one = array( - 'HasOneObject' => 'SearchUpdaterTest_HasOne' - ); + private static $has_one = array( + 'HasOneObject' => 'SearchUpdaterTest_HasOne' + ); - private static $has_many = array( - 'HasManyObjects' => 'SearchUpdaterTest_HasMany' - ); + private static $has_many = array( + 'HasManyObjects' => 'SearchUpdaterTest_HasMany' + ); - private static $many_many = array( - 'ManyManyObjects' => 'SearchUpdaterTest_ManyMany' - ); + private static $many_many = array( + 'ManyManyObjects' => 'SearchUpdaterTest_ManyMany' + ); } -class SearchUpdaterTest_HasOne extends DataObject { - private static $db = array( - 'Field1' => 'Varchar', - 'Field2' => 'Varchar' - ); +class SearchUpdaterTest_HasOne extends DataObject +{ + private static $db = array( + 'Field1' => 'Varchar', + 'Field2' => 'Varchar' + ); - private static $has_many = array( - 'HasManyContainers' => 'SearchUpdaterTest_Container' - ); + private static $has_many = array( + 'HasManyContainers' => 'SearchUpdaterTest_Container' + ); } -class SearchUpdaterTest_HasMany extends DataObject { - private static $db = array( - 'Field1' => 'Varchar', - 'Field2' => 'Varchar' - ); +class SearchUpdaterTest_HasMany extends DataObject +{ + private static $db = array( + 'Field1' => 'Varchar', + 'Field2' => 'Varchar' + ); - private static $has_one = array( - 'HasManyContainer' => 'SearchUpdaterTest_Container' - ); + private static $has_one = array( + 'HasManyContainer' => 'SearchUpdaterTest_Container' + ); } -class SearchUpdaterTest_ManyMany extends DataObject { - private static $db = array( - 'Field1' => 'Varchar', - 'Field2' => 'Varchar' - ); +class SearchUpdaterTest_ManyMany extends DataObject +{ + private static $db = array( + 'Field1' => 'Varchar', + 'Field2' => 'Varchar' + ); - private static $belongs_many_many = array( - 'ManyManyContainer' => 'SearchUpdaterTest_Container' - ); + private static $belongs_many_many = array( + 'ManyManyContainer' => 'SearchUpdaterTest_Container' + ); } -class SearchUpdaterTest_Index extends SearchIndex_Recording { - function init() { - $this->addClass('SearchUpdaterTest_Container'); +class SearchUpdaterTest_Index extends SearchIndex_Recording +{ + public function init() + { + $this->addClass('SearchUpdaterTest_Container'); - $this->addFilterField('Field1'); - $this->addFilterField('HasOneObject.Field1'); - $this->addFilterField('HasManyObjects.Field1'); - } + $this->addFilterField('Field1'); + $this->addFilterField('HasOneObject.Field1'); + $this->addFilterField('HasManyObjects.Field1'); + } } -class SearchUpdaterTest extends SapphireTest { +class SearchUpdaterTest extends SapphireTest +{ + protected $usesDatabase = true; - protected $usesDatabase = true; + private static $index = null; + + public function setUp() + { + parent::setUp(); - private static $index = null; - - function setUp() { - parent::setUp(); + if (self::$index === null) { + self::$index = singleton(get_class($this).'_Index'); + } else { + self::$index->reset(); + } - if (self::$index === null) self::$index = singleton(get_class($this).'_Index'); - else self::$index->reset(); + SearchUpdater::bind_manipulation_capture(); - SearchUpdater::bind_manipulation_capture(); + Config::nest(); - Config::nest(); + Config::inst()->update('Injector', 'SearchUpdateProcessor', array( + 'class' => 'SearchUpdateImmediateProcessor' + )); - Config::inst()->update('Injector', 'SearchUpdateProcessor', array( - 'class' => 'SearchUpdateImmediateProcessor' - )); + FullTextSearch::force_index_list(self::$index); + SearchUpdater::clear_dirty_indexes(); + } - FullTextSearch::force_index_list(self::$index); - SearchUpdater::clear_dirty_indexes(); - } + public function tearDown() + { + Config::unnest(); - function tearDown() { - Config::unnest(); + parent::tearDown(); + } - parent::tearDown(); - } + public function testBasic() + { + $item = new SearchUpdaterTest_Container(); + $item->write(); - function testBasic() { - $item = new SearchUpdaterTest_Container(); - $item->write(); + // TODO: Make sure changing field1 updates item. + // TODO: Get updating just field2 to not update item (maybe not possible - variants complicate) + } - // TODO: Make sure changing field1 updates item. - // TODO: Get updating just field2 to not update item (maybe not possible - variants complicate) - } + public function testHasOneHook() + { + $hasOne = new SearchUpdaterTest_HasOne(); + $hasOne->write(); - function testHasOneHook() { - $hasOne = new SearchUpdaterTest_HasOne(); - $hasOne->write(); + $alternateHasOne = new SearchUpdaterTest_HasOne(); + $alternateHasOne->write(); - $alternateHasOne = new SearchUpdaterTest_HasOne(); - $alternateHasOne->write(); + $container1 = new SearchUpdaterTest_Container(); + $container1->HasOneObjectID = $hasOne->ID; + $container1->write(); - $container1 = new SearchUpdaterTest_Container(); - $container1->HasOneObjectID = $hasOne->ID; - $container1->write(); + $container2 = new SearchUpdaterTest_Container(); + $container2->HasOneObjectID = $hasOne->ID; + $container2->write(); - $container2 = new SearchUpdaterTest_Container(); - $container2->HasOneObjectID = $hasOne->ID; - $container2->write(); + $container3 = new SearchUpdaterTest_Container(); + $container3->HasOneObjectID = $alternateHasOne->ID; + $container3->write(); - $container3 = new SearchUpdaterTest_Container(); - $container3->HasOneObjectID = $alternateHasOne->ID; - $container3->write(); - - // Check the default "writing a document updates the document" - SearchUpdater::flush_dirty_indexes(); + // Check the default "writing a document updates the document" + SearchUpdater::flush_dirty_indexes(); - $added = self::$index->getAdded(array('ID')); - // Some databases don't output $added in a consistent order; that's okay - usort($added, function($a,$b) {return $a['ID']-$b['ID']; }); + $added = self::$index->getAdded(array('ID')); + // Some databases don't output $added in a consistent order; that's okay + usort($added, function ($a, $b) {return $a['ID']-$b['ID']; }); - $this->assertEquals($added, array( - array('ID' => $container1->ID), - array('ID' => $container2->ID), - array('ID' => $container3->ID) - )); + $this->assertEquals($added, array( + array('ID' => $container1->ID), + array('ID' => $container2->ID), + array('ID' => $container3->ID) + )); - // Check writing a has_one tracks back to the origin documents + // Check writing a has_one tracks back to the origin documents - self::$index->reset(); + self::$index->reset(); - $hasOne->Field1 = "Updated"; - $hasOne->write(); + $hasOne->Field1 = "Updated"; + $hasOne->write(); - SearchUpdater::flush_dirty_indexes(); - $added = self::$index->getAdded(array('ID')); - // Some databases don't output $added in a consistent order; that's okay - usort($added, function($a,$b) {return $a['ID']-$b['ID']; }); + SearchUpdater::flush_dirty_indexes(); + $added = self::$index->getAdded(array('ID')); + // Some databases don't output $added in a consistent order; that's okay + usort($added, function ($a, $b) {return $a['ID']-$b['ID']; }); - $this->assertEquals($added, array( - array('ID' => $container1->ID), - array('ID' => $container2->ID) - )); + $this->assertEquals($added, array( + array('ID' => $container1->ID), + array('ID' => $container2->ID) + )); - // Check updating an unrelated field doesn't track back + // Check updating an unrelated field doesn't track back - self::$index->reset(); + self::$index->reset(); - $hasOne->Field2 = "Updated"; - $hasOne->write(); + $hasOne->Field2 = "Updated"; + $hasOne->write(); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID')), array()); + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID')), array()); - // Check writing a has_one tracks back to the origin documents + // Check writing a has_one tracks back to the origin documents - self::$index->reset(); + self::$index->reset(); - $alternateHasOne->Field1= "Updated"; - $alternateHasOne->write(); + $alternateHasOne->Field1= "Updated"; + $alternateHasOne->write(); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID')), array( - array('ID' => $container3->ID) - )); - } + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID')), array( + array('ID' => $container3->ID) + )); + } - function testHasManyHook() { - $container1 = new SearchUpdaterTest_Container(); - $container1->write(); + public function testHasManyHook() + { + $container1 = new SearchUpdaterTest_Container(); + $container1->write(); - $container2 = new SearchUpdaterTest_Container(); - $container2->write(); + $container2 = new SearchUpdaterTest_Container(); + $container2->write(); - //self::$index->reset(); - //SearchUpdater::clear_dirty_indexes(); + //self::$index->reset(); + //SearchUpdater::clear_dirty_indexes(); - $hasMany1 = new SearchUpdaterTest_HasMany(); - $hasMany1->HasManyContainerID = $container1->ID; - $hasMany1->write(); + $hasMany1 = new SearchUpdaterTest_HasMany(); + $hasMany1->HasManyContainerID = $container1->ID; + $hasMany1->write(); - $hasMany2 = new SearchUpdaterTest_HasMany(); - $hasMany2->HasManyContainerID = $container1->ID; - $hasMany2->write(); + $hasMany2 = new SearchUpdaterTest_HasMany(); + $hasMany2->HasManyContainerID = $container1->ID; + $hasMany2->write(); - SearchUpdater::flush_dirty_indexes(); + SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID')), array( - array('ID' => $container1->ID), - array('ID' => $container2->ID) - )); + $this->assertEquals(self::$index->getAdded(array('ID')), array( + array('ID' => $container1->ID), + array('ID' => $container2->ID) + )); - self::$index->reset(); + self::$index->reset(); - $hasMany1->Field1 = 'Updated'; - $hasMany1->write(); + $hasMany1->Field1 = 'Updated'; + $hasMany1->write(); - $hasMany2->Field1 = 'Updated'; - $hasMany2->write(); + $hasMany2->Field1 = 'Updated'; + $hasMany2->write(); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID')), array( - array('ID' => $container1->ID) - )); - } + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID')), array( + array('ID' => $container1->ID) + )); + } } diff --git a/tests/SearchVariantSiteTreeSubsitesPolyhomeTest.php b/tests/SearchVariantSiteTreeSubsitesPolyhomeTest.php index 7337291..20e1d9a 100644 --- a/tests/SearchVariantSiteTreeSubsitesPolyhomeTest.php +++ b/tests/SearchVariantSiteTreeSubsitesPolyhomeTest.php @@ -1,76 +1,83 @@ 'Varchar' - ); +class SearchVariantSiteTreeSubsitesPolyhomeTest_Item extends SiteTree +{ + // TODO: Currently theres a failure if you addClass a non-table class + private static $db = array( + 'TestText' => 'Varchar' + ); } -class SearchVariantSiteTreeSubsitesPolyhomeTest_Index extends SearchIndex_Recording { - function init() { - $this->addClass('SearchVariantSiteTreeSubsitesPolyhomeTest_Item'); - $this->addFilterField('TestText'); - } +class SearchVariantSiteTreeSubsitesPolyhomeTest_Index extends SearchIndex_Recording +{ + public function init() + { + $this->addClass('SearchVariantSiteTreeSubsitesPolyhomeTest_Item'); + $this->addFilterField('TestText'); + } } -class SearchVariantSiteTreeSubsitesPolyhomeTest extends SapphireTest { +class SearchVariantSiteTreeSubsitesPolyhomeTest extends SapphireTest +{ + private static $index = null; - private static $index = null; + private static $subsite_a = null; + private static $subsite_b = null; - private static $subsite_a = null; - private static $subsite_b = null; + public function setUp() + { + parent::setUp(); - function setUp() { - parent::setUp(); + // Check subsites installed + if (!class_exists('Subsite') || !class_exists('SubsitePolyhome')) { + return $this->markTestSkipped('The subsites polyhome module is not installed'); + } - // Check subsites installed - if(!class_exists('Subsite') || !class_exists('SubsitePolyhome')) { - return $this->markTestSkipped('The subsites polyhome module is not installed'); - } + if (self::$index === null) { + self::$index = singleton('SearchVariantSiteTreeSubsitesPolyhomeTest_Index'); + } - if (self::$index === null) self::$index = singleton('SearchVariantSiteTreeSubsitesPolyhomeTest_Index'); + if (self::$subsite_a === null) { + self::$subsite_a = new Subsite(); + self::$subsite_a->write(); + self::$subsite_b = new Subsite(); + self::$subsite_b->write(); + } - if (self::$subsite_a === null) { - self::$subsite_a = new Subsite(); self::$subsite_a->write(); - self::$subsite_b = new Subsite(); self::$subsite_b->write(); - } + FullTextSearch::force_index_list(self::$index); + SearchUpdater::clear_dirty_indexes(); + } - FullTextSearch::force_index_list(self::$index); - SearchUpdater::clear_dirty_indexes(); - } + public function testSavingDirect() + { + // Initial add - function testSavingDirect() { - // Initial add + $item = new SearchVariantSiteTreeSubsitesPolyhomeTest_Item(); + $item->write(); - $item = new SearchVariantSiteTreeSubsitesPolyhomeTest_Item(); - $item->write(); + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array( + array('ID' => $item->ID, '_subsite' => 0) + )); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array( - array('ID' => $item->ID, '_subsite' => 0) - )); + // Check that adding to subsites works - // Check that adding to subsites works + self::$index->reset(); - self::$index->reset(); + $item->setField('AddToSubsite[0]', 1); + $item->setField('AddToSubsite['.(self::$subsite_a->ID).']', 1); - $item->setField('AddToSubsite[0]', 1); - $item->setField('AddToSubsite['.(self::$subsite_a->ID).']', 1); + $item->write(); - $item->write(); - - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array( - array('ID' => $item->ID, '_subsite' => 0), - array('ID' => $item->ID, '_subsite' => self::$subsite_a->ID) - )); - $this->assertEquals(self::$index->deleted, array( - array('base' => 'SiteTree', 'id' => $item->ID, 'state' => array( - 'SearchVariantVersioned' => 'Stage', 'SearchVariantSiteTreeSubsitesPolyhome' => self::$subsite_b->ID - )) - )); - - - } -} \ No newline at end of file + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array( + array('ID' => $item->ID, '_subsite' => 0), + array('ID' => $item->ID, '_subsite' => self::$subsite_a->ID) + )); + $this->assertEquals(self::$index->deleted, array( + array('base' => 'SiteTree', 'id' => $item->ID, 'state' => array( + 'SearchVariantVersioned' => 'Stage', 'SearchVariantSiteTreeSubsitesPolyhome' => self::$subsite_b->ID + )) + )); + } +} diff --git a/tests/SearchVariantVersionedTest.php b/tests/SearchVariantVersionedTest.php index e10cab0..2774901 100644 --- a/tests/SearchVariantVersionedTest.php +++ b/tests/SearchVariantVersionedTest.php @@ -1,114 +1,125 @@ markTestSkipped('The versioned decorator is not installed'); + } - // Check versioned available - if(!class_exists('Versioned')) { - return $this->markTestSkipped('The versioned decorator is not installed'); - } + if (self::$index === null) { + self::$index = singleton('SearchVariantVersionedTest_Index'); + } - if (self::$index === null) self::$index = singleton('SearchVariantVersionedTest_Index'); + SearchUpdater::bind_manipulation_capture(); - SearchUpdater::bind_manipulation_capture(); + Config::nest(); - Config::nest(); + Config::inst()->update('Injector', 'SearchUpdateProcessor', array( + 'class' => 'SearchUpdateImmediateProcessor' + )); - Config::inst()->update('Injector', 'SearchUpdateProcessor', array( - 'class' => 'SearchUpdateImmediateProcessor' - )); + FullTextSearch::force_index_list(self::$index); + SearchUpdater::clear_dirty_indexes(); + } - FullTextSearch::force_index_list(self::$index); - SearchUpdater::clear_dirty_indexes(); - } + public function tearDown() + { + Config::unnest(); - function tearDown() { - Config::unnest(); + parent::tearDown(); + } - parent::tearDown(); - } + public function testPublishing() + { + // Check that write updates Stage - function testPublishing() { - // Check that write updates Stage - - $item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo')); - $item->write(); + $item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo')); + $item->write(); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( - array('ID' => $item->ID, '_versionedstage' => 'Stage') - )); + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( + array('ID' => $item->ID, '_versionedstage' => 'Stage') + )); - // Check that publish updates Live + // Check that publish updates Live - self::$index->reset(); + self::$index->reset(); - $item->publish("Stage", "Live"); + $item->publish("Stage", "Live"); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( - array('ID' => $item->ID, '_versionedstage' => 'Live') - )); + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( + array('ID' => $item->ID, '_versionedstage' => 'Live') + )); - // Just update a SiteTree field, and check it updates Stage + // Just update a SiteTree field, and check it updates Stage - self::$index->reset(); + self::$index->reset(); - $item->Title = "Pow!"; - $item->write(); + $item->Title = "Pow!"; + $item->write(); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( - array('ID' => $item->ID, '_versionedstage' => 'Stage') - )); - } + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( + array('ID' => $item->ID, '_versionedstage' => 'Stage') + )); + } - function testExcludeVariantState() { - $index = singleton('SearchVariantVersionedTest_IndexNoStage'); - FullTextSearch::force_index_list($index); + public function testExcludeVariantState() + { + $index = singleton('SearchVariantVersionedTest_IndexNoStage'); + FullTextSearch::force_index_list($index); - // Check that write doesn't update stage - $item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo')); - $item->write(); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array()); + // Check that write doesn't update stage + $item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo')); + $item->write(); + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array()); - // Check that publish updates Live - $index->reset(); - $item->publish("Stage", "Live"); - SearchUpdater::flush_dirty_indexes(); - $this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array( - array('ID' => $item->ID, '_versionedstage' => 'Live') - )); - } + // Check that publish updates Live + $index->reset(); + $item->publish("Stage", "Live"); + SearchUpdater::flush_dirty_indexes(); + $this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array( + array('ID' => $item->ID, '_versionedstage' => 'Live') + )); + } } -class SearchVariantVersionedTest_Item extends SiteTree implements TestOnly { - // TODO: Currently theres a failure if you addClass a non-table class - private static $db = array( - 'TestText' => 'Varchar' - ); +class SearchVariantVersionedTest_Item extends SiteTree implements TestOnly +{ + // TODO: Currently theres a failure if you addClass a non-table class + private static $db = array( + 'TestText' => 'Varchar' + ); } -class SearchVariantVersionedTest_Index extends SearchIndex_Recording { - function init() { - $this->addClass('SearchVariantVersionedTest_Item'); - $this->addFilterField('TestText'); - } +class SearchVariantVersionedTest_Index extends SearchIndex_Recording +{ + public function init() + { + $this->addClass('SearchVariantVersionedTest_Item'); + $this->addFilterField('TestText'); + } } -class SearchVariantVersionedTest_IndexNoStage extends SearchIndex_Recording { - function init() { - $this->addClass('SearchVariantVersionedTest_Item'); - $this->addFilterField('TestText'); - $this->excludeVariantState(array('SearchVariantVersioned' => 'Stage')); - } -} \ No newline at end of file +class SearchVariantVersionedTest_IndexNoStage extends SearchIndex_Recording +{ + public function init() + { + $this->addClass('SearchVariantVersionedTest_Item'); + $this->addFilterField('TestText'); + $this->excludeVariantState(array('SearchVariantVersioned' => 'Stage')); + } +} diff --git a/tests/Solr4ServiceTest.php b/tests/Solr4ServiceTest.php index 0ba2ca3..1359d6e 100644 --- a/tests/Solr4ServiceTest.php +++ b/tests/Solr4ServiceTest.php @@ -3,64 +3,71 @@ /** * Test solr 4.0 compatibility */ -class Solr4ServiceTest extends SapphireTest { - - /** - * - * @return Solr4ServiceTest_RecordingService - */ - protected function getMockService() { - return new Solr4ServiceTest_RecordingService(); - } - - protected function getMockDocument($id) { - $document = new Apache_Solr_Document(); - $document->setField('id', $id); - $document->setField('title', "Item $id"); - return $document; - } - - public function testAddDocument() { - $service = $this->getMockService(); - $sent = $service->addDocument($this->getMockDocument('A'), false); - $this->assertEquals( - 'AItem A', - $sent - ); - $sent = $service->addDocument($this->getMockDocument('B'), true); - $this->assertEquals( - 'BItem B', - $sent - ); - } - - public function testAddDocuments() { - $service = $this->getMockService(); - $sent = $service->addDocuments(array( - $this->getMockDocument('C'), - $this->getMockDocument('D') - ), false); - $this->assertEquals( - 'CItem CDItem D', - $sent - ); - $sent = $service->addDocuments(array( - $this->getMockDocument('E'), - $this->getMockDocument('F') - ), true); - $this->assertEquals( - 'EItem EFItem F', - $sent - ); - } +class Solr4ServiceTest extends SapphireTest +{ + /** + * + * @return Solr4ServiceTest_RecordingService + */ + protected function getMockService() + { + return new Solr4ServiceTest_RecordingService(); + } + + protected function getMockDocument($id) + { + $document = new Apache_Solr_Document(); + $document->setField('id', $id); + $document->setField('title', "Item $id"); + return $document; + } + + public function testAddDocument() + { + $service = $this->getMockService(); + $sent = $service->addDocument($this->getMockDocument('A'), false); + $this->assertEquals( + 'AItem A', + $sent + ); + $sent = $service->addDocument($this->getMockDocument('B'), true); + $this->assertEquals( + 'BItem B', + $sent + ); + } + + public function testAddDocuments() + { + $service = $this->getMockService(); + $sent = $service->addDocuments(array( + $this->getMockDocument('C'), + $this->getMockDocument('D') + ), false); + $this->assertEquals( + 'CItem CDItem D', + $sent + ); + $sent = $service->addDocuments(array( + $this->getMockDocument('E'), + $this->getMockDocument('F') + ), true); + $this->assertEquals( + 'EItem EFItem F', + $sent + ); + } } -class Solr4ServiceTest_RecordingService extends Solr4Service_Core { - protected function _sendRawPost($url, $rawPost, $timeout = FALSE, $contentType = 'text/xml; charset=UTF-8') { - return $rawPost; - } - - protected function _sendRawGet($url, $timeout = FALSE) { - return $url; - } +class Solr4ServiceTest_RecordingService extends Solr4Service_Core +{ + protected function _sendRawPost($url, $rawPost, $timeout = false, $contentType = 'text/xml; charset=UTF-8') + { + return $rawPost; + } + + protected function _sendRawGet($url, $timeout = false) + { + return $url; + } } diff --git a/tests/SolrIndexTest.php b/tests/SolrIndexTest.php index f68160a..3ba417a 100644 --- a/tests/SolrIndexTest.php +++ b/tests/SolrIndexTest.php @@ -1,307 +1,330 @@ markTestSkipped("These tests need the Phockito module installed to run"); + $this->skipTest = true; + } - function setUp() { - if (!class_exists('Phockito')) { - $this->markTestSkipped("These tests need the Phockito module installed to run"); - $this->skipTest = true; - } + parent::setUp(); + } - parent::setUp(); - } + public function testFieldDataHasOne() + { + $index = new SolrIndexTest_FakeIndex(); + $data = $index->fieldData('HasOneObject.Field1'); + $data = $data['SearchUpdaterTest_Container_HasOneObject_Field1']; - function testFieldDataHasOne() { - $index = new SolrIndexTest_FakeIndex(); - $data = $index->fieldData('HasOneObject.Field1'); - $data = $data['SearchUpdaterTest_Container_HasOneObject_Field1']; + $this->assertEquals('SearchUpdaterTest_Container', $data['origin']); + $this->assertEquals('SearchUpdaterTest_Container', $data['base']); + $this->assertEquals('SearchUpdaterTest_HasOne', $data['class']); + } - $this->assertEquals('SearchUpdaterTest_Container', $data['origin']); - $this->assertEquals('SearchUpdaterTest_Container', $data['base']); - $this->assertEquals('SearchUpdaterTest_HasOne', $data['class']); - } + public function testFieldDataHasMany() + { + $index = new SolrIndexTest_FakeIndex(); + $data = $index->fieldData('HasManyObjects.Field1'); + $data = $data['SearchUpdaterTest_Container_HasManyObjects_Field1']; - function testFieldDataHasMany() { - $index = new SolrIndexTest_FakeIndex(); - $data = $index->fieldData('HasManyObjects.Field1'); - $data = $data['SearchUpdaterTest_Container_HasManyObjects_Field1']; + $this->assertEquals('SearchUpdaterTest_Container', $data['origin']); + $this->assertEquals('SearchUpdaterTest_Container', $data['base']); + $this->assertEquals('SearchUpdaterTest_HasMany', $data['class']); + } - $this->assertEquals('SearchUpdaterTest_Container', $data['origin']); - $this->assertEquals('SearchUpdaterTest_Container', $data['base']); - $this->assertEquals('SearchUpdaterTest_HasMany', $data['class']); - } + public function testFieldDataManyMany() + { + $index = new SolrIndexTest_FakeIndex(); + $data = $index->fieldData('ManyManyObjects.Field1'); + $data = $data['SearchUpdaterTest_Container_ManyManyObjects_Field1']; - function testFieldDataManyMany() { - $index = new SolrIndexTest_FakeIndex(); - $data = $index->fieldData('ManyManyObjects.Field1'); - $data = $data['SearchUpdaterTest_Container_ManyManyObjects_Field1']; + $this->assertEquals('SearchUpdaterTest_Container', $data['origin']); + $this->assertEquals('SearchUpdaterTest_Container', $data['base']); + $this->assertEquals('SearchUpdaterTest_ManyMany', $data['class']); + } - $this->assertEquals('SearchUpdaterTest_Container', $data['origin']); - $this->assertEquals('SearchUpdaterTest_Container', $data['base']); - $this->assertEquals('SearchUpdaterTest_ManyMany', $data['class']); - } + /** + * Test boosting on SearchQuery + */ + public function testBoostedQuery() + { + $serviceMock = $this->getServiceMock(); + Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse()); - /** - * Test boosting on SearchQuery - */ - function testBoostedQuery() { - $serviceMock = $this->getServiceMock(); - Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse()); + $index = new SolrIndexTest_FakeIndex(); + $index->setService($serviceMock); - $index = new SolrIndexTest_FakeIndex(); - $index->setService($serviceMock); + $query = new SearchQuery(); + $query->search( + 'term', + null, + array('Field1' => 1.5, 'HasOneObject_Field1' => 3) + ); + $index->search($query); - $query = new SearchQuery(); - $query->search( - 'term', - null, - array('Field1' => 1.5, 'HasOneObject_Field1' => 3) - ); - $index->search($query); + Phockito::verify($serviceMock)->search('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', anything(), anything(), anything(), anything()); + } + + /** + * Test boosting on field schema (via queried fields parameter) + */ + public function testBoostedField() + { + $serviceMock = $this->getServiceMock(); + Phockito::when($serviceMock) + ->search(anything(), anything(), anything(), anything(), anything()) + ->return($this->getFakeRawSolrResponse()); - Phockito::verify($serviceMock)->search('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', anything(), anything(), anything(), anything()); - } - - /** - * Test boosting on field schema (via queried fields parameter) - */ - public function testBoostedField() { - $serviceMock = $this->getServiceMock(); - Phockito::when($serviceMock) - ->search(anything(), anything(), anything(), anything(), anything()) - ->return($this->getFakeRawSolrResponse()); + $index = new SolrIndexTest_BoostedIndex(); + $index->setService($serviceMock); - $index = new SolrIndexTest_BoostedIndex(); - $index->setService($serviceMock); + $query = new SearchQuery(); + $query->search('term'); + $index->search($query); - $query = new SearchQuery(); - $query->search('term'); - $index->search($query); + // Ensure matcher contains correct boost in 'qf' parameter + $matcher = new Hamcrest_Array_IsArrayContainingKeyValuePair( + new Hamcrest_Core_IsEqual('qf'), + new Hamcrest_Core_IsEqual('SearchUpdaterTest_Container_Field1^1.5 SearchUpdaterTest_Container_Field2^2.1 _text') + ); + Phockito::verify($serviceMock) + ->search('+term', anything(), anything(), $matcher, anything()); + } - // Ensure matcher contains correct boost in 'qf' parameter - $matcher = new Hamcrest_Array_IsArrayContainingKeyValuePair( - new Hamcrest_Core_IsEqual('qf'), - new Hamcrest_Core_IsEqual('SearchUpdaterTest_Container_Field1^1.5 SearchUpdaterTest_Container_Field2^2.1 _text') - ); - Phockito::verify($serviceMock) - ->search('+term', anything(), anything(), $matcher, anything()); - } + public function testHighlightQueryOnBoost() + { + $serviceMock = $this->getServiceMock(); + Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse()); - function testHighlightQueryOnBoost() { - $serviceMock = $this->getServiceMock(); - Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse()); + $index = new SolrIndexTest_FakeIndex(); + $index->setService($serviceMock); - $index = new SolrIndexTest_FakeIndex(); - $index->setService($serviceMock); + // Search without highlighting + $query = new SearchQuery(); + $query->search( + 'term', + null, + array('Field1' => 1.5, 'HasOneObject_Field1' => 3) + ); + $index->search($query); + Phockito::verify( + $serviceMock)->search( + '+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', + anything(), + anything(), + not(hasKeyInArray('hl.q')), + anything() + ); - // Search without highlighting - $query = new SearchQuery(); - $query->search( - 'term', - null, - array('Field1' => 1.5, 'HasOneObject_Field1' => 3) - ); - $index->search($query); - Phockito::verify( - $serviceMock)->search( - '+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', - anything(), - anything(), - not(hasKeyInArray('hl.q')), - anything() - ); + // Search with highlighting + $query = new SearchQuery(); + $query->search( + 'term', + null, + array('Field1' => 1.5, 'HasOneObject_Field1' => 3) + ); + $index->search($query, -1, -1, array('hl' => true)); + Phockito::verify( + $serviceMock)->search( + '+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', + anything(), + anything(), + hasKeyInArray('hl.q'), + anything() + ); + } - // Search with highlighting - $query = new SearchQuery(); - $query->search( - 'term', - null, - array('Field1' => 1.5, 'HasOneObject_Field1' => 3) - ); - $index->search($query, -1, -1, array('hl' => true)); - Phockito::verify( - $serviceMock)->search( - '+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', - anything(), - anything(), - hasKeyInArray('hl.q'), - anything() - ); - } + public function testIndexExcludesNullValues() + { + $serviceMock = $this->getServiceMock(); + $index = new SolrIndexTest_FakeIndex(); + $index->setService($serviceMock); + $obj = new SearchUpdaterTest_Container(); - function testIndexExcludesNullValues() { - $serviceMock = $this->getServiceMock(); - $index = new SolrIndexTest_FakeIndex(); - $index->setService($serviceMock); - $obj = new SearchUpdaterTest_Container(); + $obj->Field1 = 'Field1 val'; + $obj->Field2 = null; + $obj->MyDate = null; + $docs = $index->add($obj); + $value = $docs[0]->getField('SearchUpdaterTest_Container_Field1'); + $this->assertEquals('Field1 val', $value['value'], 'Writes non-NULL string fields'); + $value = $docs[0]->getField('SearchUpdaterTest_Container_Field2'); + $this->assertFalse($value, 'Ignores string fields if they are NULL'); + $value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate'); + $this->assertFalse($value, 'Ignores date fields if they are NULL'); - $obj->Field1 = 'Field1 val'; - $obj->Field2 = null; - $obj->MyDate = null; - $docs = $index->add($obj); - $value = $docs[0]->getField('SearchUpdaterTest_Container_Field1'); - $this->assertEquals('Field1 val', $value['value'], 'Writes non-NULL string fields'); - $value = $docs[0]->getField('SearchUpdaterTest_Container_Field2'); - $this->assertFalse($value, 'Ignores string fields if they are NULL'); - $value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate'); - $this->assertFalse($value, 'Ignores date fields if they are NULL'); + $obj->MyDate = '2010-12-30'; + $docs = $index->add($obj); + $value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate'); + $this->assertEquals('2010-12-30T00:00:00Z', $value['value'], 'Writes non-NULL dates'); + } - $obj->MyDate = '2010-12-30'; - $docs = $index->add($obj); - $value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate'); - $this->assertEquals('2010-12-30T00:00:00Z', $value['value'], 'Writes non-NULL dates'); - } + public function testAddFieldExtraOptions() + { + Config::inst()->nest(); + Config::inst()->update('Director', 'environment_type', 'live'); // dev mode sets stored=true for everything - function testAddFieldExtraOptions() { - Config::inst()->nest(); - Config::inst()->update('Director', 'environment_type', 'live'); // dev mode sets stored=true for everything + $index = new SolrIndexTest_FakeIndex(); - $index = new SolrIndexTest_FakeIndex(); + $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); + $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); + $this->assertEquals((string)$defField1[0]['stored'], 'false'); - $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); - $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); - $this->assertEquals((string)$defField1[0]['stored'], 'false'); + $index->addFilterField('Field1', null, array('stored' => 'true')); + $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); + $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); + $this->assertEquals((string)$defField1[0]['stored'], 'true'); - $index->addFilterField('Field1', null, array('stored' => 'true')); - $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); - $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); - $this->assertEquals((string)$defField1[0]['stored'], 'true'); + Config::inst()->unnest(); + } - Config::inst()->unnest(); - } + public function testAddAnalyzer() + { + $index = new SolrIndexTest_FakeIndex(); - function testAddAnalyzer() { - $index = new SolrIndexTest_FakeIndex(); + $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); + $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); + $analyzers = $defField1[0]->analyzer; + $this->assertFalse((bool)$analyzers); - $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); - $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); - $analyzers = $defField1[0]->analyzer; - $this->assertFalse((bool)$analyzers); + $index->addAnalyzer('Field1', 'charFilter', array('class' => 'solr.HTMLStripCharFilterFactory')); + $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); + $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); + $analyzers = $defField1[0]->analyzer; + $this->assertTrue((bool)$analyzers); + $this->assertEquals('solr.HTMLStripCharFilterFactory', $analyzers[0]->charFilter[0]['class']); + } - $index->addAnalyzer('Field1', 'charFilter', array('class' => 'solr.HTMLStripCharFilterFactory')); - $defs = simplexml_load_string('' . $index->getFieldDefinitions() . ''); - $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); - $analyzers = $defField1[0]->analyzer; - $this->assertTrue((bool)$analyzers); - $this->assertEquals('solr.HTMLStripCharFilterFactory', $analyzers[0]->charFilter[0]['class']); - } + public function testAddCopyField() + { + $index = new SolrIndexTest_FakeIndex(); + $index->addCopyField('sourceField', 'destField'); - function testAddCopyField() { - $index = new SolrIndexTest_FakeIndex(); - $index->addCopyField('sourceField', 'destField'); + $defs = simplexml_load_string('' . $index->getCopyFieldDefinitions() . ''); + $copyField = $defs->xpath('copyField'); - $defs = simplexml_load_string('' . $index->getCopyFieldDefinitions() . ''); - $copyField = $defs->xpath('copyField'); + $this->assertEquals('sourceField', $copyField[0]['source']); + $this->assertEquals('destField', $copyField[0]['dest']); + } - $this->assertEquals('sourceField', $copyField[0]['source']); - $this->assertEquals('destField', $copyField[0]['dest']); - } + /** + * Tests the setting of the 'stored' flag + */ + public function testStoredFields() + { + // Test two fields + $index = new SolrIndexTest_FakeIndex2(); + $index->addStoredField('Field1'); + $index->addFulltextField('Field2'); + $schema = $index->getFieldDefinitions(); + $this->assertContains( + "assertContains( + "addStoredField('Field1'); - $index->addFulltextField('Field2'); - $schema = $index->getFieldDefinitions(); - $this->assertContains( - "assertContains( - "addAllFulltextFields(); + $index2->addStoredField('Field2'); + $schema2 = $index2->getFieldDefinitions(); + $this->assertContains( + "assertContains( + "addAllFulltextFields(); - $index2->addStoredField('Field2'); - $schema2 = $index2->getFieldDefinitions(); - $this->assertContains( - "assertContains( - "_sendRawPost()->return($this->getFakeRawSolrResponse()); - protected function getServiceSpy() { - $serviceSpy = Phockito::spy('Solr3Service'); - Phockito::when($serviceSpy)->_sendRawPost()->return($this->getFakeRawSolrResponse()); + return $serviceSpy; + } - return $serviceSpy; - } - - protected function getFakeRawSolrResponse() { - return new Apache_Solr_Response( - new Apache_Solr_HttpTransport_Response( - null, - null, - '{}' - ) - ); - } + protected function getFakeRawSolrResponse() + { + return new Apache_Solr_Response( + new Apache_Solr_HttpTransport_Response( + null, + null, + '{}' + ) + ); + } } -class SolrIndexTest_FakeIndex extends SolrIndex { - function init() { - $this->addClass('SearchUpdaterTest_Container'); +class SolrIndexTest_FakeIndex extends SolrIndex +{ + public function init() + { + $this->addClass('SearchUpdaterTest_Container'); - $this->addFilterField('Field1'); - $this->addFilterField('MyDate', 'Date'); - $this->addFilterField('HasOneObject.Field1'); - $this->addFilterField('HasManyObjects.Field1'); - $this->addFilterField('ManyManyObjects.Field1'); - } + $this->addFilterField('Field1'); + $this->addFilterField('MyDate', 'Date'); + $this->addFilterField('HasOneObject.Field1'); + $this->addFilterField('HasManyObjects.Field1'); + $this->addFilterField('ManyManyObjects.Field1'); + } } -class SolrIndexTest_FakeIndex2 extends SolrIndex { - - protected function getStoredDefault() { - // Override isDev defaulting to stored - return 'false'; - } +class SolrIndexTest_FakeIndex2 extends SolrIndex +{ + protected function getStoredDefault() + { + // Override isDev defaulting to stored + return 'false'; + } - function init() { - $this->addClass('SearchUpdaterTest_Container'); - $this->addFilterField('MyDate', 'Date'); - $this->addFilterField('HasOneObject.Field1'); - $this->addFilterField('HasManyObjects.Field1'); - $this->addFilterField('ManyManyObjects.Field1'); - } + public function init() + { + $this->addClass('SearchUpdaterTest_Container'); + $this->addFilterField('MyDate', 'Date'); + $this->addFilterField('HasOneObject.Field1'); + $this->addFilterField('HasManyObjects.Field1'); + $this->addFilterField('ManyManyObjects.Field1'); + } } -class SolrIndexTest_BoostedIndex extends SolrIndex { - - protected function getStoredDefault() { - // Override isDev defaulting to stored - return 'false'; - } +class SolrIndexTest_BoostedIndex extends SolrIndex +{ + protected function getStoredDefault() + { + // Override isDev defaulting to stored + return 'false'; + } - function init() { - $this->addClass('SearchUpdaterTest_Container'); - $this->addAllFulltextFields(); - $this->setFieldBoosting('SearchUpdaterTest_Container_Field1', 1.5); - $this->addBoostedField('Field2', null, array(), 2.1); - } + public function init() + { + $this->addClass('SearchUpdaterTest_Container'); + $this->addAllFulltextFields(); + $this->setFieldBoosting('SearchUpdaterTest_Container_Field1', 1.5); + $this->addBoostedField('Field2', null, array(), 2.1); + } } - diff --git a/tests/SolrIndexVersionedTest.php b/tests/SolrIndexVersionedTest.php index d51a023..19b8383 100644 --- a/tests/SolrIndexVersionedTest.php +++ b/tests/SolrIndexVersionedTest.php @@ -1,165 +1,183 @@ skipTest = true; + return $this->markTestSkipped("These tests need the Phockito module installed to run"); + } - parent::setUp(); - - if (!class_exists('Phockito')) { - $this->skipTest = true; - return $this->markTestSkipped("These tests need the Phockito module installed to run"); - } + // Check versioned available + if (!class_exists('Versioned')) { + $this->skipTest = true; + return $this->markTestSkipped('The versioned decorator is not installed'); + } - // Check versioned available - if(!class_exists('Versioned')) { - $this->skipTest = true; - return $this->markTestSkipped('The versioned decorator is not installed'); - } + if (self::$index === null) { + self::$index = singleton('SolrVersionedTest_Index'); + } - if (self::$index === null) self::$index = singleton('SolrVersionedTest_Index'); + SearchUpdater::bind_manipulation_capture(); - SearchUpdater::bind_manipulation_capture(); + Config::nest(); - Config::nest(); + Config::inst()->update('Injector', 'SearchUpdateProcessor', array( + 'class' => 'SearchUpdateImmediateProcessor' + )); - Config::inst()->update('Injector', 'SearchUpdateProcessor', array( - 'class' => 'SearchUpdateImmediateProcessor' - )); + FullTextSearch::force_index_list(self::$index); + SearchUpdater::clear_dirty_indexes(); + + $this->oldMode = Versioned::get_reading_mode(); + Versioned::reading_stage('Stage'); + } + + public function tearDown() + { + Versioned::set_reading_mode($this->oldMode); + Config::unnest(); + parent::tearDown(); + } - FullTextSearch::force_index_list(self::$index); - SearchUpdater::clear_dirty_indexes(); - - $this->oldMode = Versioned::get_reading_mode(); - Versioned::reading_stage('Stage'); - } - - public function tearDown() { - Versioned::set_reading_mode($this->oldMode); - Config::unnest(); - parent::tearDown(); - } - - protected function getServiceMock() { - return Phockito::mock('Solr3Service'); - } - - protected function getExpectedDocumentId($id, $stage) { - // Prevent subsites from breaking tests - $subsites = class_exists('Subsite') ? '"SearchVariantSubsites":"0",' : ''; - return $id.'-SiteTree-{'.$subsites.'"SearchVariantVersioned":"'.$stage.'"}'; - } - - public function testPublishing() { - - // Setup mocks - $serviceMock = $this->getServiceMock(); - self::$index->setService($serviceMock); - - // Check that write updates Stage - Versioned::reading_stage('Stage'); - Phockito::reset($serviceMock); - $item = new SearchVariantVersionedTest_Item(array('Title' => 'Foo')); - $item->write(); - SearchUpdater::flush_dirty_indexes(); - $doc = new SolrDocumentMatcher(array( - '_documentid' => $this->getExpectedDocumentId($item->ID, 'Stage'), - 'ClassName' => 'SearchVariantVersionedTest_Item' - )); - Phockito::verify($serviceMock)->addDocument($doc); - - // Check that write updates Live - Versioned::reading_stage('Stage'); - Phockito::reset($serviceMock); - $item = new SearchVariantVersionedTest_Item(array('Title' => 'Bar')); - $item->write(); - $item->publish('Stage', 'Live'); - SearchUpdater::flush_dirty_indexes(); - $doc = new SolrDocumentMatcher(array( - '_documentid' => $this->getExpectedDocumentId($item->ID, 'Live'), - 'ClassName' => 'SearchVariantVersionedTest_Item' - )); - Phockito::verify($serviceMock)->addDocument($doc); - } - - public function testDelete() { - - // Setup mocks - $serviceMock = $this->getServiceMock(); - self::$index->setService($serviceMock); - - // Delete the live record (not the stage) - Versioned::reading_stage('Stage'); - Phockito::reset($serviceMock); - $item = new SearchVariantVersionedTest_Item(array('Title' => 'Too')); - $item->write(); - $item->publish('Stage', 'Live'); - Versioned::reading_stage('Live'); - $id = $item->ID; - $item->delete(); - SearchUpdater::flush_dirty_indexes(); - Phockito::verify($serviceMock, 1) - ->deleteById($this->getExpectedDocumentId($id, 'Live')); - Phockito::verify($serviceMock, 0) - ->deleteById($this->getExpectedDocumentId($id, 'Stage')); - - // Delete the stage record - Versioned::reading_stage('Stage'); - Phockito::reset($serviceMock); - $item = new SearchVariantVersionedTest_Item(array('Title' => 'Too')); - $item->write(); - $item->publish('Stage', 'Live'); - $id = $item->ID; - $item->delete(); - SearchUpdater::flush_dirty_indexes(); - Phockito::verify($serviceMock, 1) - ->deleteById($this->getExpectedDocumentId($id, 'Stage')); - Phockito::verify($serviceMock, 0) - ->deleteById($this->getExpectedDocumentId($id, 'Live')); - } + protected function getServiceMock() + { + return Phockito::mock('Solr3Service'); + } + + protected function getExpectedDocumentId($id, $stage) + { + // Prevent subsites from breaking tests + $subsites = class_exists('Subsite') ? '"SearchVariantSubsites":"0",' : ''; + return $id.'-SiteTree-{'.$subsites.'"SearchVariantVersioned":"'.$stage.'"}'; + } + + public function testPublishing() + { + + // Setup mocks + $serviceMock = $this->getServiceMock(); + self::$index->setService($serviceMock); + + // Check that write updates Stage + Versioned::reading_stage('Stage'); + Phockito::reset($serviceMock); + $item = new SearchVariantVersionedTest_Item(array('Title' => 'Foo')); + $item->write(); + SearchUpdater::flush_dirty_indexes(); + $doc = new SolrDocumentMatcher(array( + '_documentid' => $this->getExpectedDocumentId($item->ID, 'Stage'), + 'ClassName' => 'SearchVariantVersionedTest_Item' + )); + Phockito::verify($serviceMock)->addDocument($doc); + + // Check that write updates Live + Versioned::reading_stage('Stage'); + Phockito::reset($serviceMock); + $item = new SearchVariantVersionedTest_Item(array('Title' => 'Bar')); + $item->write(); + $item->publish('Stage', 'Live'); + SearchUpdater::flush_dirty_indexes(); + $doc = new SolrDocumentMatcher(array( + '_documentid' => $this->getExpectedDocumentId($item->ID, 'Live'), + 'ClassName' => 'SearchVariantVersionedTest_Item' + )); + Phockito::verify($serviceMock)->addDocument($doc); + } + + public function testDelete() + { + + // Setup mocks + $serviceMock = $this->getServiceMock(); + self::$index->setService($serviceMock); + + // Delete the live record (not the stage) + Versioned::reading_stage('Stage'); + Phockito::reset($serviceMock); + $item = new SearchVariantVersionedTest_Item(array('Title' => 'Too')); + $item->write(); + $item->publish('Stage', 'Live'); + Versioned::reading_stage('Live'); + $id = $item->ID; + $item->delete(); + SearchUpdater::flush_dirty_indexes(); + Phockito::verify($serviceMock, 1) + ->deleteById($this->getExpectedDocumentId($id, 'Live')); + Phockito::verify($serviceMock, 0) + ->deleteById($this->getExpectedDocumentId($id, 'Stage')); + + // Delete the stage record + Versioned::reading_stage('Stage'); + Phockito::reset($serviceMock); + $item = new SearchVariantVersionedTest_Item(array('Title' => 'Too')); + $item->write(); + $item->publish('Stage', 'Live'); + $id = $item->ID; + $item->delete(); + SearchUpdater::flush_dirty_indexes(); + Phockito::verify($serviceMock, 1) + ->deleteById($this->getExpectedDocumentId($id, 'Stage')); + Phockito::verify($serviceMock, 0) + ->deleteById($this->getExpectedDocumentId($id, 'Live')); + } } -class SolrVersionedTest_Index extends SolrIndex { - function init() { - $this->addClass('SearchVariantVersionedTest_Item'); - $this->addFilterField('TestText'); - } +class SolrVersionedTest_Index extends SolrIndex +{ + public function init() + { + $this->addClass('SearchVariantVersionedTest_Item'); + $this->addFilterField('TestText'); + } } -if (!class_exists('Phockito')) return; - -class SolrDocumentMatcher extends Hamcrest_BaseMatcher { - - protected $properties; - - public function __construct($properties) { - $this->properties = $properties; - } - - public function describeTo(\Hamcrest_Description $description) { - $description->appendText('Apache_Solr_Document with properties '.var_export($this->properties, true)); - } - - public function matches($item) { - - if(! ($item instanceof Apache_Solr_Document)) return false; - - foreach($this->properties as $key => $value) { - if($item->{$key} != $value) return false; - } - - return true; - } - +if (!class_exists('Phockito')) { + return; +} + +class SolrDocumentMatcher extends Hamcrest_BaseMatcher +{ + protected $properties; + + public function __construct($properties) + { + $this->properties = $properties; + } + + public function describeTo(\Hamcrest_Description $description) + { + $description->appendText('Apache_Solr_Document with properties '.var_export($this->properties, true)); + } + + public function matches($item) + { + if (! ($item instanceof Apache_Solr_Document)) { + return false; + } + + foreach ($this->properties as $key => $value) { + if ($item->{$key} != $value) { + return false; + } + } + + return true; + } } diff --git a/tests/SolrReindexQueuedTest.php b/tests/SolrReindexQueuedTest.php index 5c6539f..31cb4b8 100644 --- a/tests/SolrReindexQueuedTest.php +++ b/tests/SolrReindexQueuedTest.php @@ -3,217 +3,227 @@ /** * Additional tests of solr reindexing processes when run with queuedjobs */ -class SolrReindexQueuedTest extends SapphireTest { +class SolrReindexQueuedTest extends SapphireTest +{ + protected $usesDatabase = true; - protected $usesDatabase = true; + protected $extraDataObjects = array( + 'SolrReindexTest_Item' + ); - protected $extraDataObjects = array( - 'SolrReindexTest_Item' - ); + /** + * Forced index for testing + * + * @var SolrReindexTest_Index + */ + protected $index = null; - /** - * Forced index for testing - * - * @var SolrReindexTest_Index - */ - protected $index = null; + /** + * Mock service + * + * @var SolrService + */ + protected $service = null; - /** - * Mock service - * - * @var SolrService - */ - protected $service = null; + public function setUp() + { + parent::setUp(); - public function setUp() { - parent::setUp(); + if (!class_exists('Phockito')) { + $this->skipTest = true; + return $this->markTestSkipped("These tests need the Phockito module installed to run"); + } - if (!class_exists('Phockito')) { - $this->skipTest = true; - return $this->markTestSkipped("These tests need the Phockito module installed to run"); - } + if (!interface_exists('QueuedJob')) { + $this->skipTest = true; + return $this->markTestSkipped("These tests need the QueuedJobs module installed to run"); + } - if(!interface_exists('QueuedJob')) { - $this->skipTest = true; - return $this->markTestSkipped("These tests need the QueuedJobs module installed to run"); - } + // Set queued handler for reindex + Config::inst()->update('Injector', 'SolrReindexHandler', array( + 'class' => 'SolrReindexQueuedHandler' + )); + Injector::inst()->registerService(new SolrReindexQueuedHandler(), 'SolrReindexHandler'); - // Set queued handler for reindex - Config::inst()->update('Injector', 'SolrReindexHandler', array( - 'class' => 'SolrReindexQueuedHandler' - )); - Injector::inst()->registerService(new SolrReindexQueuedHandler(), 'SolrReindexHandler'); + // Set test variant + SolrReindexTest_Variant::enable(); - // Set test variant - SolrReindexTest_Variant::enable(); + // Set index list + $this->service = $this->getServiceMock(); + $this->index = singleton('SolrReindexTest_Index'); + $this->index->setService($this->service); + FullTextSearch::force_index_list($this->index); + } - // Set index list - $this->service = $this->getServiceMock(); - $this->index = singleton('SolrReindexTest_Index'); - $this->index->setService($this->service); - FullTextSearch::force_index_list($this->index); - } + /** + * Populate database with dummy dataset + * + * @param int $number Number of records to create in each variant + */ + protected function createDummyData($number) + { + // Populate dataobjects. Use truncate to generate predictable IDs + DB::query('TRUNCATE "SolrReindexTest_Item"'); - /** - * Populate database with dummy dataset - * - * @param int $number Number of records to create in each variant - */ - protected function createDummyData($number) { - // Populate dataobjects. Use truncate to generate predictable IDs - DB::query('TRUNCATE "SolrReindexTest_Item"'); + // Note that we don't create any records in variant = 2, to represent a variant + // that should be cleared without any re-indexes performed + foreach (array(0, 1) as $variant) { + for ($i = 1; $i <= $number; $i++) { + $item = new SolrReindexTest_Item(); + $item->Variant = $variant; + $item->Title = "Item $variant / $i"; + $item->write(); + } + } + } - // Note that we don't create any records in variant = 2, to represent a variant - // that should be cleared without any re-indexes performed - foreach(array(0, 1) as $variant) { - for($i = 1; $i <= $number; $i++) { - $item = new SolrReindexTest_Item(); - $item->Variant = $variant; - $item->Title = "Item $variant / $i"; - $item->write(); - } - } - } + /** + * Mock service + * + * @return SolrService + */ + protected function getServiceMock() + { + return Phockito::mock('Solr4Service'); + } - /** - * Mock service - * - * @return SolrService - */ - protected function getServiceMock() { - return Phockito::mock('Solr4Service'); - } + public function tearDown() + { + FullTextSearch::force_index_list(); + SolrReindexTest_Variant::disable(); + parent::tearDown(); + } - public function tearDown() { - FullTextSearch::force_index_list(); - SolrReindexTest_Variant::disable(); - parent::tearDown(); - } + /** + * Get the reindex handler + * + * @return SolrReindexHandler + */ + protected function getHandler() + { + return Injector::inst()->get('SolrReindexHandler'); + } - /** - * Get the reindex handler - * - * @return SolrReindexHandler - */ - protected function getHandler() { - return Injector::inst()->get('SolrReindexHandler'); - } + /** + * @return SolrReindexQueuedTest_Service + */ + protected function getQueuedJobService() + { + return singleton('SolrReindexQueuedTest_Service'); + } - /** - * @return SolrReindexQueuedTest_Service - */ - protected function getQueuedJobService() { - return singleton('SolrReindexQueuedTest_Service'); - } + /** + * Test that reindex will generate a top top level queued job, and executing this will perform + * the necessary initialisation of the grouped queued jobs + */ + public function testReindexSegmentsGroups() + { + $this->createDummyData(18); - /** - * Test that reindex will generate a top top level queued job, and executing this will perform - * the necessary initialisation of the grouped queued jobs - */ - public function testReindexSegmentsGroups() { - $this->createDummyData(18); + // Create pre-existing jobs + $this->getQueuedJobService()->queueJob(new SolrReindexQueuedJob()); + $this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob()); + $this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob()); - // Create pre-existing jobs - $this->getQueuedJobService()->queueJob(new SolrReindexQueuedJob()); - $this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob()); - $this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob()); + // Initiate re-index + $logger = new SolrReindexTest_RecordingLogger(); + $this->getHandler()->triggerReindex($logger, 6, 'Solr_Reindex'); - // Initiate re-index - $logger = new SolrReindexTest_RecordingLogger(); - $this->getHandler()->triggerReindex($logger, 6, 'Solr_Reindex'); + // Old jobs should be cancelled + $this->assertEquals(1, $logger->countMessages('Cancelled 1 re-index tasks and 2 re-index groups')); + $this->assertEquals(1, $logger->countMessages('Queued Solr Reindex Job')); - // Old jobs should be cancelled - $this->assertEquals(1, $logger->countMessages('Cancelled 1 re-index tasks and 2 re-index groups')); - $this->assertEquals(1, $logger->countMessages('Queued Solr Reindex Job')); + // Next job should be queue job + $job = $this->getQueuedJobService()->getNextJob(); + $this->assertInstanceOf('SolrReindexQueuedJob', $job); + $this->assertEquals(6, $job->getBatchSize()); - // Next job should be queue job - $job = $this->getQueuedJobService()->getNextJob(); - $this->assertInstanceOf('SolrReindexQueuedJob', $job); - $this->assertEquals(6, $job->getBatchSize()); + // Test that necessary items are created + $logger->clear(); + $job->setLogger($logger); + $job->process(); - // Test that necessary items are created - $logger->clear(); - $job->setLogger($logger); - $job->process(); + // Deletes are performed in the main task prior to individual groups being processed + // 18 records means 3 groups of 6 in each variant (6 total) + Phockito::verify($this->service, 2) + ->deleteByQuery(anything()); + $this->assertEquals(1, $logger->countMessages('Beginning init of reindex')); + $this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group ')); + $this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}')); + $this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"1"}')); + $this->assertEquals(1, $logger->countMessages('Completed init of reindex')); - // Deletes are performed in the main task prior to individual groups being processed - // 18 records means 3 groups of 6 in each variant (6 total) - Phockito::verify($this->service, 2) - ->deleteByQuery(anything()); - $this->assertEquals(1, $logger->countMessages('Beginning init of reindex')); - $this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group ')); - $this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}')); - $this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"1"}')); - $this->assertEquals(1, $logger->countMessages('Completed init of reindex')); + + // Test that invalid classes are removed + $this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index')); + Phockito::verify($this->service, 1) + ->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)'); - - // Test that invalid classes are removed - $this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index')); - Phockito::verify($this->service, 1) - ->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)'); + // Test that valid classes in invalid variants are removed + $this->assertNotEmpty($logger->getMessages( + 'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}' + )); + Phockito::verify($this->service, 1) + ->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")'); + } - // Test that valid classes in invalid variants are removed - $this->assertNotEmpty($logger->getMessages( - 'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}' - )); - Phockito::verify($this->service, 1) - ->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")'); - } + /** + * Test index processing on individual groups + */ + public function testRunGroup() + { + $this->createDummyData(18); - /** - * Test index processing on individual groups - */ - public function testRunGroup() { - $this->createDummyData(18); + // Just do what the SolrReindexQueuedJob would do to create each sub + $logger = new SolrReindexTest_RecordingLogger(); + $this->getHandler()->runReindex($logger, 6, 'Solr_Reindex'); - // Just do what the SolrReindexQueuedJob would do to create each sub - $logger = new SolrReindexTest_RecordingLogger(); - $this->getHandler()->runReindex($logger, 6, 'Solr_Reindex'); + // Assert jobs are created + $this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group')); - // Assert jobs are created - $this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group')); + // Check next job is a group queued job + $job = $this->getQueuedJobService()->getNextJob(); + $this->assertInstanceOf('SolrReindexGroupQueuedJob', $job); + $this->assertEquals( + 'Solr Reindex Group (1/3) of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}', + $job->getTitle() + ); - // Check next job is a group queued job - $job = $this->getQueuedJobService()->getNextJob(); - $this->assertInstanceOf('SolrReindexGroupQueuedJob', $job); - $this->assertEquals( - 'Solr Reindex Group (1/3) of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}', - $job->getTitle() - ); + // Running this job performs the necessary reindex + $logger->clear(); + $job->setLogger($logger); + $job->process(); - // Running this job performs the necessary reindex - $logger->clear(); - $job->setLogger($logger); - $job->process(); + // Check tasks completed (as per non-queuedjob version) + $this->assertEquals(1, $logger->countMessages('Beginning reindex group')); + $this->assertEquals(1, $logger->countMessages('Adding SolrReindexTest_Item')); + $this->assertEquals(1, $logger->countMessages('Queuing commit on all changes')); + $this->assertEquals(1, $logger->countMessages('Completed reindex group')); - // Check tasks completed (as per non-queuedjob version) - $this->assertEquals(1, $logger->countMessages('Beginning reindex group')); - $this->assertEquals(1, $logger->countMessages('Adding SolrReindexTest_Item')); - $this->assertEquals(1, $logger->countMessages('Queuing commit on all changes')); - $this->assertEquals(1, $logger->countMessages('Completed reindex group')); - - // Check IDs - $idMessage = $logger->filterMessages('Updated '); - $this->assertNotEmpty(preg_match('/^Updated (?[,\d]+)/i', $idMessage[0], $matches)); - $ids = array_unique(explode(',', $matches['ids'])); - $this->assertEquals(6, count($ids)); - foreach($ids as $id) { - // Each id should be % 3 == 0 - $this->assertEquals(0, $id % 3, "ID $id Should match pattern ID % 3 = 0"); - } - } + // Check IDs + $idMessage = $logger->filterMessages('Updated '); + $this->assertNotEmpty(preg_match('/^Updated (?[,\d]+)/i', $idMessage[0], $matches)); + $ids = array_unique(explode(',', $matches['ids'])); + $this->assertEquals(6, count($ids)); + foreach ($ids as $id) { + // Each id should be % 3 == 0 + $this->assertEquals(0, $id % 3, "ID $id Should match pattern ID % 3 = 0"); + } + } } -if(!class_exists('QueuedJobService')) return; +if (!class_exists('QueuedJobService')) { + return; +} -class SolrReindexQueuedTest_Service extends QueuedJobService implements TestOnly { - - /** - * @return QueuedJob - */ - public function getNextJob() { - $job = $this->getNextPendingJob(); - return $this->initialiseJob($job); - } - -} \ No newline at end of file +class SolrReindexQueuedTest_Service extends QueuedJobService implements TestOnly +{ + /** + * @return QueuedJob + */ + public function getNextJob() + { + $job = $this->getNextPendingJob(); + return $this->initialiseJob($job); + } +} diff --git a/tests/SolrReindexTest.php b/tests/SolrReindexTest.php index f352308..70c44f5 100644 --- a/tests/SolrReindexTest.php +++ b/tests/SolrReindexTest.php @@ -5,335 +5,348 @@ use Monolog\Handler\HandlerInterface; use Monolog\Logger; use Psr\Log\LoggerInterface; -if (class_exists('Phockito')) Phockito::include_hamcrest(); +if (class_exists('Phockito')) { + Phockito::include_hamcrest(); +} -class SolrReindexTest extends SapphireTest { +class SolrReindexTest extends SapphireTest +{ + protected $usesDatabase = true; - protected $usesDatabase = true; + protected $extraDataObjects = array( + 'SolrReindexTest_Item' + ); - protected $extraDataObjects = array( - 'SolrReindexTest_Item' - ); + /** + * Forced index for testing + * + * @var SolrReindexTest_Index + */ + protected $index = null; - /** - * Forced index for testing - * - * @var SolrReindexTest_Index - */ - protected $index = null; + /** + * Mock service + * + * @var SolrService + */ + protected $service = null; - /** - * Mock service - * - * @var SolrService - */ - protected $service = null; + public function setUp() + { + parent::setUp(); - public function setUp() { - parent::setUp(); + if (!class_exists('Phockito')) { + $this->skipTest = true; + return $this->markTestSkipped("These tests need the Phockito module installed to run"); + } + + // Set test handler for reindex + Config::inst()->update('Injector', 'SolrReindexHandler', array( + 'class' => 'SolrReindexTest_TestHandler' + )); + Injector::inst()->registerService(new SolrReindexTest_TestHandler(), 'SolrReindexHandler'); - if (!class_exists('Phockito')) { - $this->skipTest = true; - return $this->markTestSkipped("These tests need the Phockito module installed to run"); - } - - // Set test handler for reindex - Config::inst()->update('Injector', 'SolrReindexHandler', array( - 'class' => 'SolrReindexTest_TestHandler' - )); - Injector::inst()->registerService(new SolrReindexTest_TestHandler(), 'SolrReindexHandler'); + // Set test variant + SolrReindexTest_Variant::enable(); - // Set test variant - SolrReindexTest_Variant::enable(); + // Set index list + $this->service = $this->getServiceMock(); + $this->index = singleton('SolrReindexTest_Index'); + $this->index->setService($this->service); + FullTextSearch::force_index_list($this->index); + } - // Set index list - $this->service = $this->getServiceMock(); - $this->index = singleton('SolrReindexTest_Index'); - $this->index->setService($this->service); - FullTextSearch::force_index_list($this->index); - } + /** + * Populate database with dummy dataset + * + * @param int $number Number of records to create in each variant + */ + protected function createDummyData($number) + { + // Populate dataobjects. Use truncate to generate predictable IDs + DB::query('TRUNCATE "SolrReindexTest_Item"'); - /** - * Populate database with dummy dataset - * - * @param int $number Number of records to create in each variant - */ - protected function createDummyData($number) { - // Populate dataobjects. Use truncate to generate predictable IDs - DB::query('TRUNCATE "SolrReindexTest_Item"'); + // Note that we don't create any records in variant = 2, to represent a variant + // that should be cleared without any re-indexes performed + foreach (array(0, 1) as $variant) { + for ($i = 1; $i <= $number; $i++) { + $item = new SolrReindexTest_Item(); + $item->Variant = $variant; + $item->Title = "Item $variant / $i"; + $item->write(); + } + } + } - // Note that we don't create any records in variant = 2, to represent a variant - // that should be cleared without any re-indexes performed - foreach(array(0, 1) as $variant) { - for($i = 1; $i <= $number; $i++) { - $item = new SolrReindexTest_Item(); - $item->Variant = $variant; - $item->Title = "Item $variant / $i"; - $item->write(); - } - } - } + /** + * Mock service + * + * @return SolrService + */ + protected function getServiceMock() + { + return Phockito::mock('Solr4Service'); + } - /** - * Mock service - * - * @return SolrService - */ - protected function getServiceMock() { - return Phockito::mock('Solr4Service'); - } + public function tearDown() + { + FullTextSearch::force_index_list(); + SolrReindexTest_Variant::disable(); + parent::tearDown(); + } - public function tearDown() { - FullTextSearch::force_index_list(); - SolrReindexTest_Variant::disable(); - parent::tearDown(); - } + /** + * Get the reindex handler + * + * @return SolrReindexHandler + */ + protected function getHandler() + { + return Injector::inst()->get('SolrReindexHandler'); + } - /** - * Get the reindex handler - * - * @return SolrReindexHandler - */ - protected function getHandler() { - return Injector::inst()->get('SolrReindexHandler'); - } + /** + * Ensure the test variant is up and running properly + */ + public function testVariant() + { + // State defaults to 0 + $variant = SearchVariant::current_state(); + $this->assertEquals( + array( + "SolrReindexTest_Variant" => "0" + ), + $variant + ); - /** - * Ensure the test variant is up and running properly - */ - public function testVariant() { - // State defaults to 0 - $variant = SearchVariant::current_state(); - $this->assertEquals( - array( - "SolrReindexTest_Variant" => "0" - ), - $variant - ); + // All states enumerated + $allStates = iterator_to_array(SearchVariant::reindex_states()); + $this->assertEquals( + array( + array( + "SolrReindexTest_Variant" => "0" + ), + array( + "SolrReindexTest_Variant" => "1" + ), + array( + "SolrReindexTest_Variant" => "2" + ) + ), + $allStates + ); - // All states enumerated - $allStates = iterator_to_array(SearchVariant::reindex_states()); - $this->assertEquals( - array( - array( - "SolrReindexTest_Variant" => "0" - ), - array( - "SolrReindexTest_Variant" => "1" - ), - array( - "SolrReindexTest_Variant" => "2" - ) - ), - $allStates - ); - - // Check correct items created and that filtering on variant works - $this->createDummyData(120); - SolrReindexTest_Variant::set_current(2); - $this->assertEquals(0, SolrReindexTest_Item::get()->count()); - SolrReindexTest_Variant::set_current(1); - $this->assertEquals(120, SolrReindexTest_Item::get()->count()); - SolrReindexTest_Variant::set_current(0); - $this->assertEquals(120, SolrReindexTest_Item::get()->count()); - SolrReindexTest_Variant::disable(); - $this->assertEquals(240, SolrReindexTest_Item::get()->count()); - } + // Check correct items created and that filtering on variant works + $this->createDummyData(120); + SolrReindexTest_Variant::set_current(2); + $this->assertEquals(0, SolrReindexTest_Item::get()->count()); + SolrReindexTest_Variant::set_current(1); + $this->assertEquals(120, SolrReindexTest_Item::get()->count()); + SolrReindexTest_Variant::set_current(0); + $this->assertEquals(120, SolrReindexTest_Item::get()->count()); + SolrReindexTest_Variant::disable(); + $this->assertEquals(240, SolrReindexTest_Item::get()->count()); + } - /** - * Given the invocation of a new re-index with a given set of data, ensure that the necessary - * list of groups are created and segmented for each state - * - * Test should work fine with any variants (versioned, subsites, etc) specified - */ - public function testReindexSegmentsGroups() { - $this->createDummyData(120); + /** + * Given the invocation of a new re-index with a given set of data, ensure that the necessary + * list of groups are created and segmented for each state + * + * Test should work fine with any variants (versioned, subsites, etc) specified + */ + public function testReindexSegmentsGroups() + { + $this->createDummyData(120); - // Initiate re-index - $logger = new SolrReindexTest_RecordingLogger(); - $this->getHandler()->runReindex($logger, 21, 'Solr_Reindex'); + // Initiate re-index + $logger = new SolrReindexTest_RecordingLogger(); + $this->getHandler()->runReindex($logger, 21, 'Solr_Reindex'); - // Test that invalid classes are removed - $this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index')); - Phockito::verify($this->service, 1) - ->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)'); + // Test that invalid classes are removed + $this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index')); + Phockito::verify($this->service, 1) + ->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)'); - // Test that valid classes in invalid variants are removed - $this->assertNotEmpty($logger->getMessages( - 'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}' - )); - Phockito::verify($this->service, 1) - ->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")'); + // Test that valid classes in invalid variants are removed + $this->assertNotEmpty($logger->getMessages( + 'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}' + )); + Phockito::verify($this->service, 1) + ->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")'); - // 120x2 grouped into groups of 21 results in 12 groups - $this->assertEquals(12, $logger->countMessages('Called processGroup with ')); - $this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"0"}')); - $this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"1"}')); + // 120x2 grouped into groups of 21 results in 12 groups + $this->assertEquals(12, $logger->countMessages('Called processGroup with ')); + $this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"0"}')); + $this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"1"}')); - // Given that there are two variants, there should be two group ids of each number - $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 0 of 6')); - $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 1 of 6')); - $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 2 of 6')); - $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 3 of 6')); - $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 4 of 6')); - $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 5 of 6')); + // Given that there are two variants, there should be two group ids of each number + $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 0 of 6')); + $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 1 of 6')); + $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 2 of 6')); + $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 3 of 6')); + $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 4 of 6')); + $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 5 of 6')); - // Check various group sizes - $logger->clear(); - $this->getHandler()->runReindex($logger, 120, 'Solr_Reindex'); - $this->assertEquals(2, $logger->countMessages('Called processGroup with ')); - $logger->clear(); - $this->getHandler()->runReindex($logger, 119, 'Solr_Reindex'); - $this->assertEquals(4, $logger->countMessages('Called processGroup with ')); - $logger->clear(); - $this->getHandler()->runReindex($logger, 121, 'Solr_Reindex'); - $this->assertEquals(2, $logger->countMessages('Called processGroup with ')); - $logger->clear(); - $this->getHandler()->runReindex($logger, 2, 'Solr_Reindex'); - $this->assertEquals(120, $logger->countMessages('Called processGroup with ')); - } + // Check various group sizes + $logger->clear(); + $this->getHandler()->runReindex($logger, 120, 'Solr_Reindex'); + $this->assertEquals(2, $logger->countMessages('Called processGroup with ')); + $logger->clear(); + $this->getHandler()->runReindex($logger, 119, 'Solr_Reindex'); + $this->assertEquals(4, $logger->countMessages('Called processGroup with ')); + $logger->clear(); + $this->getHandler()->runReindex($logger, 121, 'Solr_Reindex'); + $this->assertEquals(2, $logger->countMessages('Called processGroup with ')); + $logger->clear(); + $this->getHandler()->runReindex($logger, 2, 'Solr_Reindex'); + $this->assertEquals(120, $logger->countMessages('Called processGroup with ')); + } - /** - * Test index processing on individual groups - */ - public function testRunGroup() { - $this->createDummyData(120); - $logger = new SolrReindexTest_RecordingLogger(); + /** + * Test index processing on individual groups + */ + public function testRunGroup() + { + $this->createDummyData(120); + $logger = new SolrReindexTest_RecordingLogger(); - // Initiate re-index of third group (index 2 of 6) - $state = array('SolrReindexTest_Variant' => '1'); - $this->getHandler()->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, 2); - $idMessage = $logger->filterMessages('Updated '); - $this->assertNotEmpty(preg_match('/^Updated (?[,\d]+)/i', $idMessage[0], $matches)); - $ids = array_unique(explode(',', $matches['ids'])); + // Initiate re-index of third group (index 2 of 6) + $state = array('SolrReindexTest_Variant' => '1'); + $this->getHandler()->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, 2); + $idMessage = $logger->filterMessages('Updated '); + $this->assertNotEmpty(preg_match('/^Updated (?[,\d]+)/i', $idMessage[0], $matches)); + $ids = array_unique(explode(',', $matches['ids'])); - // Test successful - $this->assertNotEmpty($logger->getMessages('Adding SolrReindexTest_Item')); - $this->assertNotEmpty($logger->getMessages('Done')); + // Test successful + $this->assertNotEmpty($logger->getMessages('Adding SolrReindexTest_Item')); + $this->assertNotEmpty($logger->getMessages('Done')); - // Test that items in this variant / group are cleared from solr - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")' - ); + // Test that items in this variant / group are cleared from solr + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")' + ); - // Test that items in this variant / group are re-indexed - // 120 divided into 6 groups should be 20 at least (max 21) - $this->assertEquals(21, count($ids), 'Group size is about 20', 1); - foreach($ids as $id) { - // Each id should be % 6 == 2 - $this->assertEquals(2, $id % 6, "ID $id Should match pattern ID % 6 = 2"); - } - } + // Test that items in this variant / group are re-indexed + // 120 divided into 6 groups should be 20 at least (max 21) + $this->assertEquals(21, count($ids), 'Group size is about 20', 1); + foreach ($ids as $id) { + // Each id should be % 6 == 2 + $this->assertEquals(2, $id % 6, "ID $id Should match pattern ID % 6 = 2"); + } + } - /** - * Test that running all groups covers the entire range of dataobject IDs - */ - public function testRunAllGroups() { - $this->createDummyData(120); - $logger = new SolrReindexTest_RecordingLogger(); + /** + * Test that running all groups covers the entire range of dataobject IDs + */ + public function testRunAllGroups() + { + $this->createDummyData(120); + $logger = new SolrReindexTest_RecordingLogger(); - // Test that running all groups covers the complete set of ids - $state = array('SolrReindexTest_Variant' => '1'); - for($i = 0; $i < 6; $i++) { - // See testReindexSegmentsGroups for test that each of these states is invoked during a full reindex - $this - ->getHandler() - ->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, $i); - } + // Test that running all groups covers the complete set of ids + $state = array('SolrReindexTest_Variant' => '1'); + for ($i = 0; $i < 6; $i++) { + // See testReindexSegmentsGroups for test that each of these states is invoked during a full reindex + $this + ->getHandler() + ->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, $i); + } - // Count all ids updated - $ids = array(); - foreach($logger->filterMessages('Updated ') as $message) { - $this->assertNotEmpty(preg_match('/^Updated (?[,\d]+)/', $message, $matches)); - $ids = array_unique(array_merge($ids, explode(',', $matches['ids']))); - } + // Count all ids updated + $ids = array(); + foreach ($logger->filterMessages('Updated ') as $message) { + $this->assertNotEmpty(preg_match('/^Updated (?[,\d]+)/', $message, $matches)); + $ids = array_unique(array_merge($ids, explode(',', $matches['ids']))); + } - // Check ids - $this->assertEquals(120, count($ids)); - Phockito::verify($this->service, 6)->deleteByQuery(anything()); - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=0 u=0}mod(ID, 6)" +(_testvariant:"1")' - ); - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=1 u=1}mod(ID, 6)" +(_testvariant:"1")' - ); - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")' - ); - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=3 u=3}mod(ID, 6)" +(_testvariant:"1")' - ); - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=4 u=4}mod(ID, 6)" +(_testvariant:"1")' - ); - Phockito::verify($this->service, 1)->deleteByQuery( - '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=5 u=5}mod(ID, 6)" +(_testvariant:"1")' - ); - } + // Check ids + $this->assertEquals(120, count($ids)); + Phockito::verify($this->service, 6)->deleteByQuery(anything()); + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=0 u=0}mod(ID, 6)" +(_testvariant:"1")' + ); + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=1 u=1}mod(ID, 6)" +(_testvariant:"1")' + ); + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")' + ); + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=3 u=3}mod(ID, 6)" +(_testvariant:"1")' + ); + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=4 u=4}mod(ID, 6)" +(_testvariant:"1")' + ); + Phockito::verify($this->service, 1)->deleteByQuery( + '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=5 u=5}mod(ID, 6)" +(_testvariant:"1")' + ); + } } /** * Provides a wrapper for testing SolrReindexBase */ -class SolrReindexTest_TestHandler extends SolrReindexBase { - - public function processGroup( - LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName - ) { - $indexName = $indexInstance->getIndexName(); - $stateName = json_encode($state); - $logger->info("Called processGroup with {$indexName}, {$stateName}, {$class}, group {$group} of {$groups}"); - } - - public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { - $logger->info("Called triggerReindex"); - } +class SolrReindexTest_TestHandler extends SolrReindexBase +{ + public function processGroup( + LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName + ) { + $indexName = $indexInstance->getIndexName(); + $stateName = json_encode($state); + $logger->info("Called processGroup with {$indexName}, {$stateName}, {$class}, group {$group} of {$groups}"); + } + public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) + { + $logger->info("Called triggerReindex"); + } } -class SolrReindexTest_Index extends SolrIndex implements TestOnly { - public function init() { - $this->addClass('SolrReindexTest_Item'); - $this->addAllFulltextFields(); - } +class SolrReindexTest_Index extends SolrIndex implements TestOnly +{ + public function init() + { + $this->addClass('SolrReindexTest_Item'); + $this->addAllFulltextFields(); + } } /** * Does not have any variant extensions */ -class SolrReindexTest_Item extends DataObject implements TestOnly { - - private static $extensions = array( - 'SolrReindexTest_ItemExtension' - ); - - private static $db = array( - 'Title' => 'Varchar(255)', - 'Variant' => 'Int(0)' - ); +class SolrReindexTest_Item extends DataObject implements TestOnly +{ + private static $extensions = array( + 'SolrReindexTest_ItemExtension' + ); + private static $db = array( + 'Title' => 'Varchar(255)', + 'Variant' => 'Int(0)' + ); } /** * Select only records in the current variant */ -class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly { - - /** - * Filter records on the current variant - * - * @param SQLQuery $query - * @param DataQuery $dataQuery - */ - public function augmentSQL(SQLQuery &$query, DataQuery &$dataQuery = null) { - $variant = SolrReindexTest_Variant::get_current(); - if($variant !== null && !$query->filtersOnID()) { - $sqlVariant = Convert::raw2sql($variant); - $query->addWhere("\"Variant\" = '{$sqlVariant}'"); - } - } +class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly +{ + /** + * Filter records on the current variant + * + * @param SQLQuery $query + * @param DataQuery $dataQuery + */ + public function augmentSQL(SQLQuery &$query, DataQuery &$dataQuery = null) + { + $variant = SolrReindexTest_Variant::get_current(); + if ($variant !== null && !$query->filtersOnID()) { + $sqlVariant = Convert::raw2sql($variant); + $query->addWhere("\"Variant\" = '{$sqlVariant}'"); + } + } } @@ -342,190 +355,208 @@ class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly { * * Variant states are 0 and 1, or null if disabled */ -class SolrReindexTest_Variant extends SearchVariant implements TestOnly { +class SolrReindexTest_Variant extends SearchVariant implements TestOnly +{ + /** + * Value of this variant (either null, 0, or 1) + * + * @var int|null + */ + protected static $current = null; - /** - * Value of this variant (either null, 0, or 1) - * - * @var int|null - */ - protected static $current = null; + /** + * Activate this variant + */ + public static function enable() + { + self::disable(); - /** - * Activate this variant - */ - public static function enable() { - self::disable(); + self::$current = 0; + self::$variants = array( + 'SolrReindexTest_Variant' => singleton('SolrReindexTest_Variant') + ); + } - self::$current = 0; - self::$variants = array( - 'SolrReindexTest_Variant' => singleton('SolrReindexTest_Variant') - ); - } + /** + * Disable this variant and reset + */ + public static function disable() + { + self::$current = null; + self::$variants = null; + self::$class_variants = array(); + self::$call_instances = array(); + } - /** - * Disable this variant and reset - */ - public static function disable() { - self::$current = null; - self::$variants = null; - self::$class_variants = array(); - self::$call_instances = array(); - } + public function activateState($state) + { + self::set_current($state); + } - public function activateState($state) { - self::set_current($state); - } + /** + * Set the current variant to the given state + * + * @param int $current 0, 1, 2, or null (disabled) + */ + public static function set_current($current) + { + self::$current = $current; + } - /** - * Set the current variant to the given state - * - * @param int $current 0, 1, 2, or null (disabled) - */ - public static function set_current($current) { - self::$current = $current; - } + /** + * Get the current state + * + * @return string|null + */ + public static function get_current() + { + // Always use string values for states for consistent json_encode value + if (isset(self::$current)) { + return (string)self::$current; + } + } - /** - * Get the current state - * - * @return string|null - */ - public static function get_current() { - // Always use string values for states for consistent json_encode value - if(isset(self::$current)) { - return (string)self::$current; - } - } + public function alterDefinition($base, $index) + { + $self = get_class($this); - function alterDefinition($base, $index) { - $self = get_class($this); + $index->filterFields['_testvariant'] = array( + 'name' => '_testvariant', + 'field' => '_testvariant', + 'fullfield' => '_testvariant', + 'base' => $base, + 'origin' => $base, + 'type' => 'Int', + 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) + ); + } - $index->filterFields['_testvariant'] = array( - 'name' => '_testvariant', - 'field' => '_testvariant', - 'fullfield' => '_testvariant', - 'base' => $base, - 'origin' => $base, - 'type' => 'Int', - 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) - ); - } + public function alterQuery($query, $index) + { + // I guess just calling it _testvariant is ok? + $query->filter('_testvariant', $this->currentState()); + } - public function alterQuery($query, $index) { - // I guess just calling it _testvariant is ok? - $query->filter('_testvariant', $this->currentState()); - } + public function appliesTo($class, $includeSubclasses) + { + return $class === 'SolrReindexTest_Item' || + ($includeSubclasses && is_subclass_of($class, 'SolrReindexTest_Item', true)); + } - public function appliesTo($class, $includeSubclasses) { - return $class === 'SolrReindexTest_Item' || - ($includeSubclasses && is_subclass_of($class, 'SolrReindexTest_Item', true)); - } + public function appliesToEnvironment() + { + // Set to null to disable + return self::$current !== null; + } - public function appliesToEnvironment() { - // Set to null to disable - return self::$current !== null; - } - - public function currentState() { - return self::get_current(); - } - - public function reindexStates() { - // Always use string values for states for consistent json_encode value - return array('0', '1', '2'); - } + public function currentState() + { + return self::get_current(); + } + public function reindexStates() + { + // Always use string values for states for consistent json_encode value + return array('0', '1', '2'); + } } /** * Test logger for recording messages */ -class SolrReindexTest_RecordingLogger extends Logger implements TestOnly { +class SolrReindexTest_RecordingLogger extends Logger implements TestOnly +{ + /** + * @var SolrReindexTest_Handler + */ + protected $testHandler = null; - /** - * @var SolrReindexTest_Handler - */ - protected $testHandler = null; + public function __construct($name = 'testlogger', array $handlers = array(), array $processors = array()) + { + parent::__construct($name, $handlers, $processors); - public function __construct($name = 'testlogger', array $handlers = array(), array $processors = array()) { - parent::__construct($name, $handlers, $processors); + $this->testHandler = new SolrReindexTest_Handler(); + $this->pushHandler($this->testHandler); + } - $this->testHandler = new SolrReindexTest_Handler(); - $this->pushHandler($this->testHandler); - } + /** + * @return array + */ + public function getMessages() + { + return $this->testHandler->getMessages(); + } - /** - * @return array - */ - public function getMessages() { - return $this->testHandler->getMessages(); - } + /** + * Clear all messages + */ + public function clear() + { + $this->testHandler->clear(); + } - /** - * Clear all messages - */ - public function clear() { - $this->testHandler->clear(); - } + /** + * Get messages with the given filter + * + * @param string $containing + * @return array Filtered array + */ + public function filterMessages($containing) + { + return array_values(array_filter( + $this->getMessages(), + function ($content) use ($containing) { + return stripos($content, $containing) !== false; + } + )); + } - /** - * Get messages with the given filter - * - * @param string $containing - * @return array Filtered array - */ - public function filterMessages($containing) { - return array_values(array_filter( - $this->getMessages(), - function($content) use ($containing) { - return stripos($content, $containing) !== false; - } - )); - } - - /** - * Count all messages containing the given substring - * - * @param string $containing Message to filter by - * @return int - */ - public function countMessages($containing = null) { - if($containing) { - $messages = $this->filterMessages($containing); - } else { - $messages = $this->getMessages(); - } - return count($messages); - } + /** + * Count all messages containing the given substring + * + * @param string $containing Message to filter by + * @return int + */ + public function countMessages($containing = null) + { + if ($containing) { + $messages = $this->filterMessages($containing); + } else { + $messages = $this->getMessages(); + } + return count($messages); + } } /** * Logger for recording messages for later retrieval */ -class SolrReindexTest_Handler extends AbstractProcessingHandler implements TestOnly { +class SolrReindexTest_Handler extends AbstractProcessingHandler implements TestOnly +{ + /** + * Messages + * + * @var array + */ + protected $messages = array(); - /** - * Messages - * - * @var array - */ - protected $messages = array(); + /** + * Get all messages + * + * @return array + */ + public function getMessages() + { + return $this->messages; + } - /** - * Get all messages - * - * @return array - */ - public function getMessages() { - return $this->messages; - } + public function clear() + { + $this->messages = array(); + } - public function clear() { - $this->messages = array(); - } - - protected function write(array $record) { - $this->messages[] = $record['message']; - } -} \ No newline at end of file + protected function write(array $record) + { + $this->messages[] = $record['message']; + } +}