tonyair/silverstripe-fulltextsearch
1
0
Fork 0
mirror of https://github.com/silverstripe/silverstripe-fulltextsearch synced 2024-10-22 12:05:29 +00:00
Code Issues Projects Releases Wiki Activity

Converted to PSR-2

Browse Source
This commit is contained in:
helpfulrobot 2015-11-21 19:19:20 +13:00
parent c96d4bf749
commit 314feddd48
44 changed files with 6459 additions and 5854 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
code/search/FullTextSearch.php
View File

@ -3,124 +3,134 @@
/** /**
* Base class to manage active search indexes. * Base class to manage active search indexes.
*/ */
class FullTextSearch { class FullTextSearch
{
protected static $all_indexes = null;
static protected $all_indexes = null; protected static $indexes_by_subclass = array();
static protected $indexes_by_subclass = array(); /**
* Optional list of index names to limit to. If left empty, all subclasses of SearchIndex
* will be used
*
* @var array
* @config
*/
private static $indexes = array();
/** /**
* Optional list of index names to limit to. If left empty, all subclasses of SearchIndex * Get all the instantiable search indexes (so all the user created indexes, but not the connector or library level
* will be used * abstract indexes). Can optionally be filtered to only return indexes that are subclasses of some class
* *
* @var array * @static
* @config * @param String $class - Class name to filter indexes by, so that all returned indexes are subclasses of provided class
*/ * @param bool $rebuild - If true, don't use cached values
private static $indexes = array(); */
public static function get_indexes($class = null, $rebuild = false)
{
if ($rebuild) {
self::$all_indexes = null;
self::$indexes_by_subclass = array();
}
/** if (!$class) {
* Get all the instantiable search indexes (so all the user created indexes, but not the connector or library level if (self::$all_indexes === null) {
* abstract indexes). Can optionally be filtered to only return indexes that are subclasses of some class // Get declared indexes, or otherwise default to all subclasses of SearchIndex
* $classes = Config::inst()->get(__CLASS__, 'indexes')
* @static ?: ClassInfo::subclassesFor('SearchIndex');
* @param String $class - Class name to filter indexes by, so that all returned indexes are subclasses of provided class
* @param bool $rebuild - If true, don't use cached values
*/
static function get_indexes($class = null, $rebuild = false) {
if ($rebuild) {
self::$all_indexes = null;
self::$indexes_by_subclass = array();
}
if (!$class) { $hidden = array();
if (self::$all_indexes === null) { $candidates = array();
// Get declared indexes, or otherwise default to all subclasses of SearchIndex foreach ($classes as $class) {
$classes = Config::inst()->get(__CLASS__, 'indexes') // Check if this index is disabled
?: ClassInfo::subclassesFor('SearchIndex'); $hides = $class::config()->hide_ancestor;
if ($hides) {
$hidden[] = $hides;
}
$hidden = array(); // Check if this index is abstract
$candidates = array(); $ref = new ReflectionClass($class);
foreach ($classes as $class) { if (!$ref->isInstantiable()) {
// Check if this index is disabled continue;
$hides = $class::config()->hide_ancestor; }
if($hides) {
$hidden[] = $hides;
}
// Check if this index is abstract $candidates[] = $class;
$ref = new ReflectionClass($class); }
if (!$ref->isInstantiable()) {
continue;
}
$candidates[] = $class; if ($hidden) {
} $candidates = array_diff($candidates, $hidden);
}
if($hidden) { // Create all indexes
$candidates = array_diff($candidates, $hidden); $concrete = array();
} foreach ($candidates as $class) {
$concrete[$class] = singleton($class);
}
// Create all indexes self::$all_indexes = $concrete;
$concrete = array(); }
foreach($candidates as $class) {
$concrete[$class] = singleton($class);
}
self::$all_indexes = $concrete; return self::$all_indexes;
} } else {
if (!isset(self::$indexes_by_subclass[$class])) {
$all = self::get_indexes();
return self::$all_indexes; $valid = array();
} foreach ($all as $indexclass => $instance) {
else { if (is_subclass_of($indexclass, $class)) {
if (!isset(self::$indexes_by_subclass[$class])) { $valid[$indexclass] = $instance;
$all = self::get_indexes(); }
}
$valid = array(); self::$indexes_by_subclass[$class] = $valid;
foreach ($all as $indexclass => $instance) { }
if (is_subclass_of($indexclass, $class)) $valid[$indexclass] = $instance;
}
self::$indexes_by_subclass[$class] = $valid; return self::$indexes_by_subclass[$class];
} }
}
return self::$indexes_by_subclass[$class]; /**
} * Sometimes, like when in tests, you want to restrain the actual indexes to a subset
} *
* Call with one argument - an array of class names, index instances or classname => indexinstance pairs (can be mixed).
* Alternatively call with multiple arguments, each of which is a class name or index instance
*
* From then on, fulltext search system will only see those indexes passed in this most recent call.
*
* Passing in no arguments resets back to automatic index list
*
* Alternatively you can use `FullTextSearch.indexes` to configure a list of indexes via config.
*/
public static function force_index_list()
{
$indexes = func_get_args();
/** // No arguments = back to automatic
* Sometimes, like when in tests, you want to restrain the actual indexes to a subset if (!$indexes) {
* self::get_indexes(null, true);
* Call with one argument - an array of class names, index instances or classname => indexinstance pairs (can be mixed). return;
* Alternatively call with multiple arguments, each of which is a class name or index instance }
*
* From then on, fulltext search system will only see those indexes passed in this most recent call.
*
* Passing in no arguments resets back to automatic index list
*
* Alternatively you can use `FullTextSearch.indexes` to configure a list of indexes via config.
*/
static function force_index_list() {
$indexes = func_get_args();
// No arguments = back to automatic // Arguments can be a single array
if (!$indexes) { if (is_array($indexes[0])) {
self::get_indexes(null, true); $indexes = $indexes[0];
return; }
}
// Arguments can be a single array // Reset to empty first
if (is_array($indexes[0])) $indexes = $indexes[0]; self::$all_indexes = array();
self::$indexes_by_subclass = array();
// Reset to empty first // And parse out alternative type combos for arguments and add to allIndexes
self::$all_indexes = array(); self::$indexes_by_subclass = array(); foreach ($indexes as $class => $index) {
if (is_string($index)) {
// And parse out alternative type combos for arguments and add to allIndexes $class = $index;
foreach ($indexes as $class => $index) { $index = singleton($class);
if (is_string($index)) { $class = $index; $index = singleton($class); } }
if (is_numeric($class)) $class = get_class($index); if (is_numeric($class)) {
$class = get_class($index);
self::$all_indexes[$class] = $index; }
}
}
self::$all_indexes[$class] = $index;
}
}
} }

1169
code/search/SearchIndex.php
View File

@ -27,576 +27,647 @@
* - Specifying update rules that are not extractable from metadata (because the values come from functions for instance) * - Specifying update rules that are not extractable from metadata (because the values come from functions for instance)
* *
*/ */
abstract class SearchIndex extends ViewableData { abstract class SearchIndex extends ViewableData
{
/** /**
* Allows this index to hide a parent index. Specifies the name of a parent index to disable * Allows this index to hide a parent index. Specifies the name of a parent index to disable
* *
* @var string * @var string
* @config * @config
*/ */
private static $hide_ancestor; private static $hide_ancestor;
public function __construct() { public function __construct()
parent::__construct(); {
$this->init(); parent::__construct();
$this->init();
foreach ($this->getClasses() as $class => $options) {
SearchVariant::with($class, $options['include_children'])->call('alterDefinition', $class, $this); foreach ($this->getClasses() as $class => $options) {
} SearchVariant::with($class, $options['include_children'])->call('alterDefinition', $class, $this);
}
$this->buildDependancyList();
} $this->buildDependancyList();
}
public function __toString() {
return 'Search Index ' . get_class($this); public function __toString()
} {
return 'Search Index ' . get_class($this);
/** }
* Examines the classes this index is built on to try and find defined fields in the class hierarchy for those classes.
* Looks for db and viewable-data fields, although can't nessecarily find type for viewable-data fields. /**
*/ * Examines the classes this index is built on to try and find defined fields in the class hierarchy for those classes.
public function fieldData($field, $forceType = null, $extraOptions = array()) { * Looks for db and viewable-data fields, although can't nessecarily find type for viewable-data fields.
$fullfield = str_replace(".", "_", $field); */
$sources = $this->getClasses(); public function fieldData($field, $forceType = null, $extraOptions = array())
{
foreach ($sources as $source => $options) { $fullfield = str_replace(".", "_", $field);
$sources[$source]['base'] = ClassInfo::baseDataClass($source); $sources = $this->getClasses();
$sources[$source]['lookup_chain'] = array();
} foreach ($sources as $source => $options) {
$sources[$source]['base'] = ClassInfo::baseDataClass($source);
$found = array(); $sources[$source]['lookup_chain'] = array();
}
if (strpos($field, '.') !== false) {
$lookups = explode(".", $field); $found = array();
$field = array_pop($lookups);
if (strpos($field, '.') !== false) {
foreach ($lookups as $lookup) { $lookups = explode(".", $field);
$next = array(); $field = array_pop($lookups);
foreach ($sources as $source => $options) { foreach ($lookups as $lookup) {
$class = null; $next = array();
foreach (SearchIntrospection::hierarchy($source, $options['include_children']) as $dataclass) { foreach ($sources as $source => $options) {
$singleton = singleton($dataclass); $class = null;
if ($hasOne = $singleton->has_one($lookup)) { foreach (SearchIntrospection::hierarchy($source, $options['include_children']) as $dataclass) {
$class = $hasOne; $singleton = singleton($dataclass);
$options['lookup_chain'][] = array(
'call' => 'method', 'method' => $lookup, if ($hasOne = $singleton->has_one($lookup)) {
'through' => 'has_one', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => "{$lookup}ID" $class = $hasOne;
); $options['lookup_chain'][] = array(
} 'call' => 'method', 'method' => $lookup,
else if ($hasMany = $singleton->has_many($lookup)) { 'through' => 'has_one', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => "{$lookup}ID"
$class = $hasMany; );
$options['multi_valued'] = true; } elseif ($hasMany = $singleton->has_many($lookup)) {
$options['lookup_chain'][] = array( $class = $hasMany;
'call' => 'method', 'method' => $lookup, $options['multi_valued'] = true;
'through' => 'has_many', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => $singleton->getRemoteJoinField($lookup, 'has_many') $options['lookup_chain'][] = array(
); 'call' => 'method', 'method' => $lookup,
} 'through' => 'has_many', 'class' => $dataclass, 'otherclass' => $class, 'foreignkey' => $singleton->getRemoteJoinField($lookup, 'has_many')
else if ($manyMany = $singleton->many_many($lookup)) { );
$class = $manyMany[1]; } elseif ($manyMany = $singleton->many_many($lookup)) {
$options['multi_valued'] = true; $class = $manyMany[1];
$options['lookup_chain'][] = array( $options['multi_valued'] = true;
'call' => 'method', 'method' => $lookup, $options['lookup_chain'][] = array(
'through' => 'many_many', 'class' => $dataclass, 'otherclass' => $class, 'details' => $manyMany 'call' => 'method', 'method' => $lookup,
); 'through' => 'many_many', 'class' => $dataclass, 'otherclass' => $class, 'details' => $manyMany
} );
}
if ($class) {
if (!isset($options['origin'])) $options['origin'] = $dataclass; if ($class) {
$next[$class] = $options; if (!isset($options['origin'])) {
continue 2; $options['origin'] = $dataclass;
} }
} $next[$class] = $options;
} continue 2;
}
if (!$next) return $next; // Early out to avoid excessive empty looping }
$sources = $next; }
}
} if (!$next) {
return $next;
foreach ($sources as $class => $options) { } // Early out to avoid excessive empty looping
$dataclasses = SearchIntrospection::hierarchy($class, $options['include_children']); $sources = $next;
}
while (count($dataclasses)) { }
$dataclass = array_shift($dataclasses);
$type = null; $fieldoptions = $options; foreach ($sources as $class => $options) {
$dataclasses = SearchIntrospection::hierarchy($class, $options['include_children']);
$fields = DataObject::database_fields($dataclass);
while (count($dataclasses)) {
if (isset($fields[$field])) { $dataclass = array_shift($dataclasses);
$type = $fields[$field]; $type = null;
$fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field); $fieldoptions = $options;
}
else { $fields = DataObject::database_fields($dataclass);
$singleton = singleton($dataclass);
if (isset($fields[$field])) {
if ($singleton->hasMethod("get$field") || $singleton->hasField($field)) { $type = $fields[$field];
$type = $singleton->castingClass($field); $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field);
if (!$type) $type = 'String'; } else {
$singleton = singleton($dataclass);
if ($singleton->hasMethod("get$field")) $fieldoptions['lookup_chain'][] = array('call' => 'method', 'method' => "get$field");
else $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field); if ($singleton->hasMethod("get$field") || $singleton->hasField($field)) {
} $type = $singleton->castingClass($field);
} if (!$type) {
$type = 'String';
if ($type) { }
// Don't search through child classes of a class we matched on. TODO: Should we?
$dataclasses = array_diff($dataclasses, array_values(ClassInfo::subclassesFor($dataclass))); if ($singleton->hasMethod("get$field")) {
// Trim arguments off the type string $fieldoptions['lookup_chain'][] = array('call' => 'method', 'method' => "get$field");
if (preg_match('/^(\w+)\(/', $type, $match)) $type = $match[1]; } else {
// Get the origin $fieldoptions['lookup_chain'][] = array('call' => 'property', 'property' => $field);
$origin = isset($fieldoptions['origin']) ? $fieldoptions['origin'] : $dataclass; }
}
$found["{$origin}_{$fullfield}"] = array( }
'name' => "{$origin}_{$fullfield}",
'field' => $field, if ($type) {
'fullfield' => $fullfield, // Don't search through child classes of a class we matched on. TODO: Should we?
'base' => $fieldoptions['base'], $dataclasses = array_diff($dataclasses, array_values(ClassInfo::subclassesFor($dataclass)));
'origin' => $origin, // Trim arguments off the type string
'class' => $dataclass, if (preg_match('/^(\w+)\(/', $type, $match)) {
'lookup_chain' => $fieldoptions['lookup_chain'], $type = $match[1];
'type' => $forceType ? $forceType : $type, }
'multi_valued' => isset($fieldoptions['multi_valued']) ? true : false, // Get the origin
'extra_options' => $extraOptions $origin = isset($fieldoptions['origin']) ? $fieldoptions['origin'] : $dataclass;
);
} $found["{$origin}_{$fullfield}"] = array(
} 'name' => "{$origin}_{$fullfield}",
} 'field' => $field,
'fullfield' => $fullfield,
return $found; 'base' => $fieldoptions['base'],
} 'origin' => $origin,
'class' => $dataclass,
/** Public, but should only be altered by variants */ 'lookup_chain' => $fieldoptions['lookup_chain'],
'type' => $forceType ? $forceType : $type,
protected $classes = array(); 'multi_valued' => isset($fieldoptions['multi_valued']) ? true : false,
'extra_options' => $extraOptions
protected $fulltextFields = array(); );
}
public $filterFields = array(); }
}
protected $sortFields = array();
return $found;
protected $excludedVariantStates = array(); }
/** /** Public, but should only be altered by variants */
* Add a DataObject subclass whose instances should be included in this index
* protected $classes = array();
* Can only be called when addFulltextField, addFilterField, addSortField and addAllFulltextFields have not
* yet been called for this index instance protected $fulltextFields = array();
*
* @throws Exception public $filterFields = array();
* @param String $class - The class to include
* @param array $options - TODO: Remove protected $sortFields = array();
*/
public function addClass($class, $options = array()) { protected $excludedVariantStates = array();
if ($this->fulltextFields || $this->filterFields || $this->sortFields) {
throw new Exception('Can\'t add class to Index after fields have already been added'); /**
} * Add a DataObject subclass whose instances should be included in this index
*
if (!DataObject::has_own_table($class)) { * Can only be called when addFulltextField, addFilterField, addSortField and addAllFulltextFields have not
throw new InvalidArgumentException('Can\'t add classes which don\'t have data tables (no $db or $has_one set on the class)'); * yet been called for this index instance
} *
* @throws Exception
$options = array_merge(array( * @param String $class - The class to include
'include_children' => true * @param array $options - TODO: Remove
), $options); */
public function addClass($class, $options = array())
$this->classes[$class] = $options; {
} if ($this->fulltextFields || $this->filterFields || $this->sortFields) {
throw new Exception('Can\'t add class to Index after fields have already been added');
/** }
* Get the classes added by addClass
*/ if (!DataObject::has_own_table($class)) {
public function getClasses() { return $this->classes; } throw new InvalidArgumentException('Can\'t add classes which don\'t have data tables (no $db or $has_one set on the class)');
}
/**
* Add a field that should be fulltext searchable $options = array_merge(array(
* @param String $field - The field to add 'include_children' => true
* @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) ), $options);
* @param String $extraOptions - Dependent on search implementation
*/ $this->classes[$class] = $options;
public function addFulltextField($field, $forceType = null, $extraOptions = array()) { }
$this->fulltextFields = array_merge($this->fulltextFields, $this->fieldData($field, $forceType, $extraOptions));
} /**
* Get the classes added by addClass
public function getFulltextFields() { return $this->fulltextFields; } */
public function getClasses()
/** {
* Add a field that should be filterable return $this->classes;
* @param String $field - The field to add }
* @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata)
* @param String $extraOptions - Dependent on search implementation /**
*/ * Add a field that should be fulltext searchable
public function addFilterField($field, $forceType = null, $extraOptions = array()) { * @param String $field - The field to add
$this->filterFields = array_merge($this->filterFields, $this->fieldData($field, $forceType, $extraOptions)); * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata)
} * @param String $extraOptions - Dependent on search implementation
*/
public function getFilterFields() { return $this->filterFields; } public function addFulltextField($field, $forceType = null, $extraOptions = array())
{
/** $this->fulltextFields = array_merge($this->fulltextFields, $this->fieldData($field, $forceType, $extraOptions));
* Add a field that should be sortable }
* @param String $field - The field to add
* @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata) public function getFulltextFields()
* @param String $extraOptions - Dependent on search implementation {
*/ return $this->fulltextFields;
public function addSortField($field, $forceType = null, $extraOptions = array()) { }
$this->sortFields = array_merge($this->sortFields, $this->fieldData($field, $forceType, $extraOptions));
} /**
* Add a field that should be filterable
public function getSortFields() { return $this->sortFields; } * @param String $field - The field to add
* @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata)
/** * @param String $extraOptions - Dependent on search implementation
* Add all database-backed text fields as fulltext searchable fields. */
* public function addFilterField($field, $forceType = null, $extraOptions = array())
* For every class included in the index, examines those classes and all subclasses looking for "Text" database {
* fields (Varchar, Text, HTMLText, etc) and adds them all as fulltext searchable fields. $this->filterFields = array_merge($this->filterFields, $this->fieldData($field, $forceType, $extraOptions));
*/ }
public function addAllFulltextFields($includeSubclasses = true) {
foreach ($this->getClasses() as $class => $options) { public function getFilterFields()
foreach (SearchIntrospection::hierarchy($class, $includeSubclasses, true) as $dataclass) { {
$fields = DataObject::database_fields($dataclass); return $this->filterFields;
}
foreach ($fields as $field => $type) {
if (preg_match('/^(\w+)\(/', $type, $match)) $type = $match[1]; /**
if (is_subclass_of($type, 'StringField')) $this->addFulltextField($field); * Add a field that should be sortable
} * @param String $field - The field to add
} * @param String $forceType - The type to force this field as (required in some cases, when not detectable from metadata)
} * @param String $extraOptions - Dependent on search implementation
} */
public function addSortField($field, $forceType = null, $extraOptions = array())
/** {
* Returns an interator that will let you interate through all added fields, regardless of whether they $this->sortFields = array_merge($this->sortFields, $this->fieldData($field, $forceType, $extraOptions));
* were added as fulltext, filter or sort fields. }
*
* @return MultipleArrayIterator public function getSortFields()
*/ {
public function getFieldsIterator() { return $this->sortFields;
return new MultipleArrayIterator($this->fulltextFields, $this->filterFields, $this->sortFields); }
}
/**
public function excludeVariantState($state) { * Add all database-backed text fields as fulltext searchable fields.
$this->excludedVariantStates[] = $state; *
} * For every class included in the index, examines those classes and all subclasses looking for "Text" database
* fields (Varchar, Text, HTMLText, etc) and adds them all as fulltext searchable fields.
/** Returns true if some variant state should be ignored */ */
public function variantStateExcluded($state) { public function addAllFulltextFields($includeSubclasses = true)
foreach ($this->excludedVariantStates as $excludedstate) { {
$matches = true; foreach ($this->getClasses() as $class => $options) {
foreach (SearchIntrospection::hierarchy($class, $includeSubclasses, true) as $dataclass) {
foreach ($excludedstate as $variant => $variantstate) { $fields = DataObject::database_fields($dataclass);
if (!isset($state[$variant]) || $state[$variant] != $variantstate) { $matches = false; break; }
} foreach ($fields as $field => $type) {
if (preg_match('/^(\w+)\(/', $type, $match)) {
if ($matches) return true; $type = $match[1];
} }
} if (is_subclass_of($type, 'StringField')) {
$this->addFulltextField($field);
public $dependancyList = array(); }
}
public function buildDependancyList() { }
$this->dependancyList = array_keys($this->getClasses()); }
}
foreach ($this->getFieldsIterator() as $name => $field) {
if (!isset($field['class'])) continue; /**
SearchIntrospection::add_unique_by_ancestor($this->dependancyList, $field['class']); * Returns an interator that will let you interate through all added fields, regardless of whether they
} * were added as fulltext, filter or sort fields.
} *
* @return MultipleArrayIterator
public $derivedFields = null; */
public function getFieldsIterator()
/** {
* Returns an array where each member is all the fields and the classes that are at the end of some return new MultipleArrayIterator($this->fulltextFields, $this->filterFields, $this->sortFields);
* specific lookup chain from one of the base classes }
*/
public function getDerivedFields() { public function excludeVariantState($state)
if ($this->derivedFields === null) { {
$this->derivedFields = array(); $this->excludedVariantStates[] = $state;
}
foreach ($this->getFieldsIterator() as $name => $field) {
if (count($field['lookup_chain']) < 2) continue; /** Returns true if some variant state should be ignored */
public function variantStateExcluded($state)
$key = sha1($field['base'].serialize($field['lookup_chain'])); {
$fieldname = "{$field['class']}:{$field['field']}"; foreach ($this->excludedVariantStates as $excludedstate) {
$matches = true;
if (isset($this->derivedFields[$key])) {
$this->derivedFields[$key]['fields'][$fieldname] = $fieldname; foreach ($excludedstate as $variant => $variantstate) {
SearchIntrospection::add_unique_by_ancestor($this->derivedFields['classes'], $field['class']); if (!isset($state[$variant]) || $state[$variant] != $variantstate) {
} $matches = false;
else { break;
$chain = array_reverse($field['lookup_chain']); }
array_shift($chain); }
$this->derivedFields[$key] = array( if ($matches) {
'base' => $field['base'], return true;
'fields' => array($fieldname => $fieldname), }
'classes' => array($field['class']), }
'chain' => $chain }
);
} public $dependancyList = array();
}
} public function buildDependancyList()
{
return $this->derivedFields; $this->dependancyList = array_keys($this->getClasses());
}
foreach ($this->getFieldsIterator() as $name => $field) {
/** if (!isset($field['class'])) {
* Get the "document ID" (a database & variant unique id) given some "Base" class, DataObject ID and state array continue;
* }
* @param String $base - The base class of the object SearchIntrospection::add_unique_by_ancestor($this->dependancyList, $field['class']);
* @param Integer $id - The ID of the object }
* @param Array $state - The variant state of the object }
* @return string - The document ID as a string
*/ public $derivedFields = null;
public function getDocumentIDForState($base, $id, $state) {
ksort($state); /**
$parts = array('id' => $id, 'base' => $base, 'state' => json_encode($state)); * Returns an array where each member is all the fields and the classes that are at the end of some
return implode('-', array_values($parts)); * specific lookup chain from one of the base classes
} */
public function getDerivedFields()
/** {
* Get the "document ID" (a database & variant unique id) given some "Base" class and DataObject if ($this->derivedFields === null) {
* $this->derivedFields = array();
* @param DataObject $object - The object
* @param String $base - The base class of the object foreach ($this->getFieldsIterator() as $name => $field) {
* @param Boolean $includesubs - TODO: Probably going away if (count($field['lookup_chain']) < 2) {
* @return string - The document ID as a string continue;
*/ }
public function getDocumentID($object, $base, $includesubs) {
return $this->getDocumentIDForState($base, $object->ID, SearchVariant::current_state($base, $includesubs)); $key = sha1($field['base'].serialize($field['lookup_chain']));
} $fieldname = "{$field['class']}:{$field['field']}";
/** if (isset($this->derivedFields[$key])) {
* Given an object and a field definition (as returned by fieldData) get the current value of that field on that object $this->derivedFields[$key]['fields'][$fieldname] = $fieldname;
* SearchIntrospection::add_unique_by_ancestor($this->derivedFields['classes'], $field['class']);
* @param DataObject $object - The object to get the value from } else {
* @param Array $field - The field definition to use $chain = array_reverse($field['lookup_chain']);
* @return Mixed - The value of the field, or null if we couldn't look it up for some reason array_shift($chain);
*/
protected function _getFieldValue($object, $field) { $this->derivedFields[$key] = array(
set_error_handler(create_function('$no, $str', 'throw new Exception("HTML Parse Error: ".$str);'), E_ALL); 'base' => $field['base'],
'fields' => array($fieldname => $fieldname),
try { 'classes' => array($field['class']),
foreach ($field['lookup_chain'] as $step) { 'chain' => $chain
// Just fail if we've fallen off the end of the chain );
if (!$object) return null; }
}
// If we're looking up this step on an array or SS_List, do the step on every item, merge result }
if (is_array($object) || $object instanceof SS_List) {
$next = array(); return $this->derivedFields;
}
foreach ($object as $item) {
if ($step['call'] == 'method') { /**
$method = $step['method']; * Get the "document ID" (a database & variant unique id) given some "Base" class, DataObject ID and state array
$item = $item->$method(); *
} * @param String $base - The base class of the object
else { * @param Integer $id - The ID of the object
$property = $step['property']; * @param Array $state - The variant state of the object
$item = $item->$property; * @return string - The document ID as a string
} */
public function getDocumentIDForState($base, $id, $state)
if ($item instanceof SS_List) $next = array_merge($next, $item->toArray()); {
elseif (is_array($item)) $next = array_merge($next, $item); ksort($state);
else $next[] = $item; $parts = array('id' => $id, 'base' => $base, 'state' => json_encode($state));
} return implode('-', array_values($parts));
}
$object = $next;
} /**
// Otherwise, just call * Get the "document ID" (a database & variant unique id) given some "Base" class and DataObject
else { *
if ($step['call'] == 'method') { * @param DataObject $object - The object
$method = $step['method']; * @param String $base - The base class of the object
$object = $object->$method(); * @param Boolean $includesubs - TODO: Probably going away
} * @return string - The document ID as a string
elseif ($step['call'] == 'variant') { */
$variants = SearchVariant::variants($field['base'], true); public function getDocumentID($object, $base, $includesubs)
$variant = $variants[$step['variant']]; $method = $step['method']; {
$object = $variant->$method($object); return $this->getDocumentIDForState($base, $object->ID, SearchVariant::current_state($base, $includesubs));
} }
else {
$property = $step['property']; /**
$object = $object->$property; * Given an object and a field definition (as returned by fieldData) get the current value of that field on that object
} *
} * @param DataObject $object - The object to get the value from
} * @param Array $field - The field definition to use
} * @return Mixed - The value of the field, or null if we couldn't look it up for some reason
catch (Exception $e) { */
$object = null; protected function _getFieldValue($object, $field)
} {
set_error_handler(create_function('$no, $str', 'throw new Exception("HTML Parse Error: ".$str);'), E_ALL);
restore_error_handler();
return $object; try {
} foreach ($field['lookup_chain'] as $step) {
// Just fail if we've fallen off the end of the chain
/** if (!$object) {
* Given a class, object id, set of stateful ids and a list of changed fields (in a special format), return null;
* return what statefulids need updating in this index }
*
* Internal function used by SearchUpdater. // If we're looking up this step on an array or SS_List, do the step on every item, merge result
* if (is_array($object) || $object instanceof SS_List) {
* @param $class $next = array();
* @param $id
* @param $statefulids foreach ($object as $item) {
* @param $fields if ($step['call'] == 'method') {
* @return array $method = $step['method'];
*/ $item = $item->$method();
public function getDirtyIDs($class, $id, $statefulids, $fields) { } else {
$dirty = array(); $property = $step['property'];
$item = $item->$property;
// First, if this object is directly contained in the index, add it }
foreach ($this->classes as $searchclass => $options) {
if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { if ($item instanceof SS_List) {
$next = array_merge($next, $item->toArray());
$base = ClassInfo::baseDataClass($searchclass); } elseif (is_array($item)) {
$dirty[$base] = array(); $next = array_merge($next, $item);
foreach ($statefulids as $statefulid) { } else {
$key = serialize($statefulid); $next[] = $item;
$dirty[$base][$key] = $statefulid; }
} }
}
} $object = $next;
}
$current = SearchVariant::current_state(); // Otherwise, just call
else {
if ($step['call'] == 'method') {
// Then, for every derived field $method = $step['method'];
foreach ($this->getDerivedFields() as $derivation) { $object = $object->$method();
// If the this object is a subclass of any of the classes we want a field from } elseif ($step['call'] == 'variant') {
if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) continue; $variants = SearchVariant::variants($field['base'], true);
if (!array_intersect_key($fields, $derivation['fields'])) continue; $variant = $variants[$step['variant']];
$method = $step['method'];
foreach (SearchVariant::reindex_states($class, false) as $state) { $object = $variant->$method($object);
SearchVariant::activate_state($state); } else {
$property = $step['property'];
$ids = array($id); $object = $object->$property;
}
foreach ($derivation['chain'] as $step) { }
if ($step['through'] == 'has_one') { }
$sql = new SQLQuery('"ID"', '"'.$step['class'].'"', '"'.$step['foreignkey'].'" IN ('.implode(',', $ids).')'); } catch (Exception $e) {
singleton($step['class'])->extend('augmentSQL', $sql); $object = null;
}
$ids = $sql->execute()->column();
} restore_error_handler();
else if ($step['through'] == 'has_many') { return $object;
$sql = new SQLQuery('"'.$step['class'].'"."ID"', '"'.$step['class'].'"', '"'.$step['otherclass'].'"."ID" IN ('.implode(',', $ids).')'); }
$sql->addInnerJoin($step['otherclass'], '"'.$step['class'].'"."ID" = "'.$step['otherclass'].'"."'.$step['foreignkey'].'"');
singleton($step['class'])->extend('augmentSQL', $sql); /**
* Given a class, object id, set of stateful ids and a list of changed fields (in a special format),
$ids = $sql->execute()->column(); * return what statefulids need updating in this index
} *
} * Internal function used by SearchUpdater.
*
SearchVariant::activate_state($current); * @param $class
* @param $id
if ($ids) { * @param $statefulids
$base = $derivation['base']; * @param $fields
if (!isset($dirty[$base])) $dirty[$base] = array(); * @return array
*/
foreach ($ids as $id) { public function getDirtyIDs($class, $id, $statefulids, $fields)
$statefulid = array('id' => $id, 'state' => $state); {
$key = serialize($statefulid); $dirty = array();
$dirty[$base][$key] = $statefulid;
} // First, if this object is directly contained in the index, add it
} foreach ($this->classes as $searchclass => $options) {
} if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) {
} $base = ClassInfo::baseDataClass($searchclass);
$dirty[$base] = array();
return $dirty; foreach ($statefulids as $statefulid) {
} $key = serialize($statefulid);
$dirty[$base][$key] = $statefulid;
/** !! These should be implemented by the full text search engine */ }
}
abstract public function add($object) ; }
abstract public function delete($base, $id, $state) ;
$current = SearchVariant::current_state();
abstract public function commit();
/** !! These should be implemented by the specific index */ // Then, for every derived field
foreach ($this->getDerivedFields() as $derivation) {
/** // If the this object is a subclass of any of the classes we want a field from
* Called during construction, this is the method that builds the structure. if (!SearchIntrospection::is_subclass_of($class, $derivation['classes'])) {
* Used instead of overriding __construct as we have specific execution order - code that has continue;
* to be run before _and/or_ after this. }
*/ if (!array_intersect_key($fields, $derivation['fields'])) {
abstract public function init(); continue;
}
foreach (SearchVariant::reindex_states($class, false) as $state) {
SearchVariant::activate_state($state);
$ids = array($id);
foreach ($derivation['chain'] as $step) {
if ($step['through'] == 'has_one') {
$sql = new SQLQuery('"ID"', '"'.$step['class'].'"', '"'.$step['foreignkey'].'" IN ('.implode(',', $ids).')');
singleton($step['class'])->extend('augmentSQL', $sql);
$ids = $sql->execute()->column();
} elseif ($step['through'] == 'has_many') {
$sql = new SQLQuery('"'.$step['class'].'"."ID"', '"'.$step['class'].'"', '"'.$step['otherclass'].'"."ID" IN ('.implode(',', $ids).')');
$sql->addInnerJoin($step['otherclass'], '"'.$step['class'].'"."ID" = "'.$step['otherclass'].'"."'.$step['foreignkey'].'"');
singleton($step['class'])->extend('augmentSQL', $sql);
$ids = $sql->execute()->column();
}
}
SearchVariant::activate_state($current);
if ($ids) {
$base = $derivation['base'];
if (!isset($dirty[$base])) {
$dirty[$base] = array();
}
foreach ($ids as $id) {
$statefulid = array('id' => $id, 'state' => $state);
$key = serialize($statefulid);
$dirty[$base][$key] = $statefulid;
}
}
}
}
return $dirty;
}
/** !! These should be implemented by the full text search engine */
abstract public function add($object) ;
abstract public function delete($base, $id, $state) ;
abstract public function commit();
/** !! These should be implemented by the specific index */
/**
* Called during construction, this is the method that builds the structure.
* Used instead of overriding __construct as we have specific execution order - code that has
* to be run before _and/or_ after this.
*/
abstract public function init();
} }
/** /**
* A search index that does nothing. Useful for testing * A search index that does nothing. Useful for testing
*/ */
abstract class SearchIndex_Null extends SearchIndex { abstract class SearchIndex_Null extends SearchIndex
{
public function add($object)
{
}
public function add($object) { } public function delete($base, $id, $state)
{
public function delete($base, $id, $state) { } }
public function commit() { }
public function commit()
{
}
} }
/** /**
* A search index that just records actions. Useful for testing * A search index that just records actions. Useful for testing
*/ */
abstract class SearchIndex_Recording extends SearchIndex { abstract class SearchIndex_Recording extends SearchIndex
{
public $added = array();
public $deleted = array();
public $committed = false;
public $added = array(); public function reset()
public $deleted = array(); {
public $committed = false; $this->added = array();
$this->deleted = array();
$this->committed = false;
}
public function reset() { public function add($object)
$this->added = array(); {
$this->deleted = array(); $res = array();
$this->committed = false;
}
public function add($object) { $res['ID'] = $object->ID;
$res = array();
foreach ($this->getFieldsIterator() as $name => $field) {
$val = $this->_getFieldValue($object, $field);
$res[$name] = $val;
}
$res['ID'] = $object->ID; $this->added[] = $res;
}
foreach ($this->getFieldsIterator() as $name => $field) {
$val = $this->_getFieldValue($object, $field);
$res[$name] = $val;
}
$this->added[] = $res; public function getAdded($fields = array())
} {
$res = array();
public function getAdded($fields = array()) { foreach ($this->added as $added) {
$res = array(); $filtered = array();
foreach ($fields as $field) {
if (isset($added[$field])) {
$filtered[$field] = $added[$field];
}
}
$res[] = $filtered;
}
foreach ($this->added as $added) { return $res;
$filtered = array(); }
foreach ($fields as $field) {
if (isset($added[$field])) $filtered[$field] = $added[$field];
}
$res[] = $filtered;
}
return $res; public function delete($base, $id, $state)
} {
$this->deleted[] = array('base' => $base, 'id' => $id, 'state' => $state);
}
public function delete($base, $id, $state) { public function commit()
$this->deleted[] = array('base' => $base, 'id' => $id, 'state' => $state); {
} $this->committed = true;
}
public function getIndexName()
{
return get_class($this);
}
public function commit() { public function getIsCommitted()
$this->committed = true; {
} return $this->committed;
}
public function getIndexName() {
return get_class($this);
}
public function getIsCommitted() { public function getService()
return $this->committed; {
} // Causes commits to the service to be redirected back to the same object
return $this;
public function getService() { }
// Causes commits to the service to be redirected back to the same object
return $this;
}
} }

137
code/search/SearchIntrospection.php
View File

@ -3,76 +3,91 @@
/** /**
* Some additional introspection tools that are used often by the fulltext search code * Some additional introspection tools that are used often by the fulltext search code
*/ */
class SearchIntrospection { class SearchIntrospection
{
protected static $ancestry = array();
protected static $ancestry = array(); /**
* Check if class is subclass of (a) the class in $of, or (b) any of the classes in the array $of
* @static
* @param $class
* @param $of
* @return bool
*/
public static function is_subclass_of($class, $of)
{
$ancestry = isset(self::$ancestry[$class]) ? self::$ancestry[$class] : (self::$ancestry[$class] = ClassInfo::ancestry($class));
return is_array($of) ? (bool)array_intersect($of, $ancestry) : array_key_exists($of, $ancestry);
}
/** protected static $hierarchy = array();
* Check if class is subclass of (a) the class in $of, or (b) any of the classes in the array $of
* @static
* @param $class
* @param $of
* @return bool
*/
static function is_subclass_of ($class, $of) {
$ancestry = isset(self::$ancestry[$class]) ? self::$ancestry[$class] : (self::$ancestry[$class] = ClassInfo::ancestry($class));
return is_array($of) ? (bool)array_intersect($of, $ancestry) : array_key_exists($of, $ancestry);
}
protected static $hierarchy = array(); /**
* Get all the classes involved in a DataObject hierarchy - both super and optionally subclasses
*
* @static
* @param String $class - The class to query
* @param bool $includeSubclasses - True to return subclasses as well as super classes
* @param bool $dataOnly - True to only return classes that have tables
* @return Array - Integer keys, String values as classes sorted by depth (most super first)
*/
public static function hierarchy($class, $includeSubclasses = true, $dataOnly = false)
{
$key = "$class!" . ($includeSubclasses ? 'sc' : 'an') . '!' . ($dataOnly ? 'do' : 'al');
if (!isset(self::$hierarchy[$key])) {
$classes = array_values(ClassInfo::ancestry($class));
if ($includeSubclasses) {
$classes = array_unique(array_merge($classes, array_values(ClassInfo::subclassesFor($class))));
}
/** $idx = array_search('DataObject', $classes);
* Get all the classes involved in a DataObject hierarchy - both super and optionally subclasses if ($idx !== false) {
* array_splice($classes, 0, $idx+1);
* @static }
* @param String $class - The class to query
* @param bool $includeSubclasses - True to return subclasses as well as super classes
* @param bool $dataOnly - True to only return classes that have tables
* @return Array - Integer keys, String values as classes sorted by depth (most super first)
*/
static function hierarchy ($class, $includeSubclasses = true, $dataOnly = false) {
$key = "$class!" . ($includeSubclasses ? 'sc' : 'an') . '!' . ($dataOnly ? 'do' : 'al');
if (!isset(self::$hierarchy[$key])) {
$classes = array_values(ClassInfo::ancestry($class));
if ($includeSubclasses) $classes = array_unique(array_merge($classes, array_values(ClassInfo::subclassesFor($class))));
$idx = array_search('DataObject', $classes); if ($dataOnly) {
if ($idx !== false) array_splice($classes, 0, $idx+1); foreach ($classes as $i => $class) {
if (!DataObject::has_own_table($class)) {
unset($classes[$i]);
}
}
}
if ($dataOnly) foreach($classes as $i => $class) { self::$hierarchy[$key] = $classes;
if (!DataObject::has_own_table($class)) unset($classes[$i]); }
}
self::$hierarchy[$key] = $classes; return self::$hierarchy[$key];
} }
return self::$hierarchy[$key]; /**
} * Add classes to list, keeping only the parent when parent & child are both in list after add
*/
public static function add_unique_by_ancestor(&$list, $class)
{
// If class already has parent in list, just ignore
if (self::is_subclass_of($class, $list)) {
return;
}
/** // Strip out any subclasses of $class already in the list
* Add classes to list, keeping only the parent when parent & child are both in list after add $children = ClassInfo::subclassesFor($class);
*/ $list = array_diff($list, $children);
static function add_unique_by_ancestor(&$list, $class) {
// If class already has parent in list, just ignore
if (self::is_subclass_of($class, $list)) return;
// Strip out any subclasses of $class already in the list // Then add the class in
$children = ClassInfo::subclassesFor($class); $list[] = $class;
$list = array_diff($list, $children); }
// Then add the class in
$list[] = $class;
}
/**
* Does this class, it's parent (or optionally one of it's children) have the passed extension attached?
*/
static function has_extension($class, $extension, $includeSubclasses = true) {
foreach (self::hierarchy($class, $includeSubclasses) as $relatedclass) {
if ($relatedclass::has_extension($extension)) return true;
}
return false;
}
/**
* Does this class, it's parent (or optionally one of it's children) have the passed extension attached?
*/
public static function has_extension($class, $extension, $includeSubclasses = true)
{
foreach (self::hierarchy($class, $includeSubclasses) as $relatedclass) {
if ($relatedclass::has_extension($extension)) {
return true;
}
}
return false;
}
} }

215
code/search/SearchQuery.php
View File

@ -5,129 +5,148 @@
* *
* API very much still in flux. * API very much still in flux.
*/ */
class SearchQuery extends ViewableData { class SearchQuery extends ViewableData
{
public static $missing = null;
public static $present = null;
static $missing = null; public static $default_page_size = 10;
static $present = null;
static $default_page_size = 10; /** These are public, but only for index & variant access - API users should not manually access these */
/** These are public, but only for index & variant access - API users should not manually access these */ public $search = array();
public $search = array(); public $classes = array();
public $classes = array(); public $require = array();
public $exclude = array();
public $require = array(); protected $start = 0;
public $exclude = array(); protected $limit = -1;
protected $start = 0; /** These are the API functions */
protected $limit = -1;
/** These are the API functions */ public function __construct()
{
if (self::$missing === null) {
self::$missing = new stdClass();
}
if (self::$present === null) {
self::$present = new stdClass();
}
}
function __construct() { /**
if (self::$missing === null) self::$missing = new stdClass(); * @param String $text Search terms. Exact format (grouping, boolean expressions, etc.) depends on the search implementation.
if (self::$present === null) self::$present = new stdClass(); * @param array $fields Limits the search to specific fields (using composite field names)
} * @param array $boost Map of composite field names to float values. The higher the value,
* the more important the field gets for relevancy.
*/
public function search($text, $fields = null, $boost = array())
{
$this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => false);
}
/** /**
* @param String $text Search terms. Exact format (grouping, boolean expressions, etc.) depends on the search implementation. * Similar to {@link search()}, but uses stemming and other similarity algorithms
* @param array $fields Limits the search to specific fields (using composite field names) * to find the searched terms. For example, a term "fishing" would also likely find results
* @param array $boost Map of composite field names to float values. The higher the value, * containing "fish" or "fisher". Depends on search implementation.
* the more important the field gets for relevancy. *
*/ * @param String $text See {@link search()}
function search($text, $fields = null, $boost = array()) { * @param array $fields See {@link search()}
$this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => false); * @param array $boost See {@link search()}
} */
public function fuzzysearch($text, $fields = null, $boost = array())
{
$this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => true);
}
/** public function inClass($class, $includeSubclasses = true)
* Similar to {@link search()}, but uses stemming and other similarity algorithms {
* to find the searched terms. For example, a term "fishing" would also likely find results $this->classes[] = array('class' => $class, 'includeSubclasses' => $includeSubclasses);
* containing "fish" or "fisher". Depends on search implementation. }
*
* @param String $text See {@link search()}
* @param array $fields See {@link search()}
* @param array $boost See {@link search()}
*/
function fuzzysearch($text, $fields = null, $boost = array()) {
$this->search[] = array('text' => $text, 'fields' => $fields ? (array)$fields : null, 'boost' => $boost, 'fuzzy' => true);
}
function inClass($class, $includeSubclasses = true) { /**
$this->classes[] = array('class' => $class, 'includeSubclasses' => $includeSubclasses); * Similar to {@link search()}, but typically used to further narrow down
} * based on other facets which don't influence the field relevancy.
*
* @param String $field Composite name of the field
* @param Mixed $values Scalar value, array of values, or an instance of SearchQuery_Range
*/
public function filter($field, $values)
{
$requires = isset($this->require[$field]) ? $this->require[$field] : array();
$values = is_array($values) ? $values : array($values);
$this->require[$field] = array_merge($requires, $values);
}
/** /**
* Similar to {@link search()}, but typically used to further narrow down * Excludes results which match these criteria, inverse of {@link filter()}.
* based on other facets which don't influence the field relevancy. *
* * @param String $field
* @param String $field Composite name of the field * @param mixed $values
* @param Mixed $values Scalar value, array of values, or an instance of SearchQuery_Range */
*/ public function exclude($field, $values)
function filter($field, $values) { {
$requires = isset($this->require[$field]) ? $this->require[$field] : array(); $excludes = isset($this->exclude[$field]) ? $this->exclude[$field] : array();
$values = is_array($values) ? $values : array($values); $values = is_array($values) ? $values : array($values);
$this->require[$field] = array_merge($requires, $values); $this->exclude[$field] = array_merge($excludes, $values);
} }
/** public function start($start)
* Excludes results which match these criteria, inverse of {@link filter()}. {
* $this->start = $start;
* @param String $field }
* @param mixed $values
*/
function exclude($field, $values) {
$excludes = isset($this->exclude[$field]) ? $this->exclude[$field] : array();
$values = is_array($values) ? $values : array($values);
$this->exclude[$field] = array_merge($excludes, $values);
}
function start($start) { public function limit($limit)
$this->start = $start; {
} $this->limit = $limit;
}
function limit($limit) { public function page($page)
$this->limit = $limit; {
} $this->start = $page * self::$default_page_size;
$this->limit = self::$default_page_size;
}
function page($page) { public function isfiltered()
$this->start = $page * self::$default_page_size; {
$this->limit = self::$default_page_size; return $this->search || $this->classes || $this->require || $this->exclude;
} }
function isfiltered() { public function __toString()
return $this->search || $this->classes || $this->require || $this->exclude; {
} return "Search Query\n";
}
function __toString() {
return "Search Query\n";
}
} }
/** /**
* Create one of these and pass as one of the values in filter or exclude to filter or exclude by a (possibly * Create one of these and pass as one of the values in filter or exclude to filter or exclude by a (possibly
* open ended) range * open ended) range
*/ */
class SearchQuery_Range { class SearchQuery_Range
{
public $start = null;
public $end = null;
public $start = null; public function __construct($start = null, $end = null)
public $end = null; {
$this->start = $start;
$this->end = $end;
}
function __construct($start = null, $end = null) { public function start($start)
$this->start = $start; {
$this->end = $end; $this->start = $start;
} }
function start($start) { public function end($end)
$this->start = $start; {
} $this->end = $end;
}
function end($end) { public function isfiltered()
$this->end = $end; {
} return $this->start !== null || $this->end !== null;
}
function isfiltered() { }
return $this->start !== null || $this->end !== null;
}
}

385
code/search/SearchUpdater.php
View File

@ -12,22 +12,25 @@
* *
* TODO: The way we bind in is awful hacky. * TODO: The way we bind in is awful hacky.
*/ */
class SearchUpdater extends Object { class SearchUpdater extends Object
{
/**
* Replace the database object with a subclass that captures all manipulations and passes them to us
*/
public static function bind_manipulation_capture()
{
global $databaseConfig;
/** $current = DB::getConn();
* Replace the database object with a subclass that captures all manipulations and passes them to us if (!$current || @$current->isManipulationCapture) {
*/ return;
static function bind_manipulation_capture() { } // If not yet set, or its already captured, just return
global $databaseConfig;
$current = DB::getConn(); $type = get_class($current);
if (!$current || @$current->isManipulationCapture) return; // If not yet set, or its already captured, just return $file = TEMP_FOLDER."/.cache.SMC.$type";
$type = get_class($current); if (!is_file($file)) {
$file = TEMP_FOLDER."/.cache.SMC.$type"; file_put_contents($file, "<?php
if (!is_file($file)) {
file_put_contents($file, "<?php
class SearchManipulateCapture_$type extends $type { class SearchManipulateCapture_$type extends $type {
public \$isManipulationCapture = true; public \$isManipulationCapture = true;
@ -38,162 +41,173 @@ class SearchUpdater extends Object {
} }
} }
"); ");
} }
require_once($file); require_once($file);
$dbClass = 'SearchManipulateCapture_'.$type; $dbClass = 'SearchManipulateCapture_'.$type;
/** @var SS_Database $captured */ /** @var SS_Database $captured */
$captured = new $dbClass($databaseConfig); $captured = new $dbClass($databaseConfig);
// Framework 3.2+ ORM needs some dependencies set // Framework 3.2+ ORM needs some dependencies set
if (method_exists($captured, "setConnector")) { if (method_exists($captured, "setConnector")) {
$captured->setConnector($current->getConnector()); $captured->setConnector($current->getConnector());
$captured->setQueryBuilder($current->getQueryBuilder()); $captured->setQueryBuilder($current->getQueryBuilder());
$captured->setSchemaManager($current->getSchemaManager()); $captured->setSchemaManager($current->getSchemaManager());
} }
// The connection might have had it's name changed (like if we're currently in a test) // The connection might have had it's name changed (like if we're currently in a test)
$captured->selectDatabase($current->currentDatabase()); $captured->selectDatabase($current->currentDatabase());
DB::setConn($captured); DB::setConn($captured);
} }
static $registered = false; public static $registered = false;
/** @var SearchUpdateProcessor */ /** @var SearchUpdateProcessor */
static $processor = null; public static $processor = null;
/** /**
* Called by the SearchManiplateCapture database adapter with every manipulation made against the database. * Called by the SearchManiplateCapture database adapter with every manipulation made against the database.
* *
* Check every index to see what objects need re-inserting into what indexes to keep the index fresh, * Check every index to see what objects need re-inserting into what indexes to keep the index fresh,
* but doesn't actually do it yet. * but doesn't actually do it yet.
* *
* TODO: This is pretty sensitive to the format of manipulation that DataObject::write produces. Specifically, * TODO: This is pretty sensitive to the format of manipulation that DataObject::write produces. Specifically,
* it expects the actual class of the object to be present as a table, regardless of if any fields changed in that table * it expects the actual class of the object to be present as a table, regardless of if any fields changed in that table
* (so a class => array( 'fields' => array() ) item), in order to find the actual class for a set of table manipulations * (so a class => array( 'fields' => array() ) item), in order to find the actual class for a set of table manipulations
*/ */
static function handle_manipulation($manipulation) { public static function handle_manipulation($manipulation)
// First, extract any state that is in the manipulation itself {
foreach ($manipulation as $table => $details) { // First, extract any state that is in the manipulation itself
$manipulation[$table]['class'] = $table; foreach ($manipulation as $table => $details) {
$manipulation[$table]['state'] = array(); $manipulation[$table]['class'] = $table;
} $manipulation[$table]['state'] = array();
}
SearchVariant::call('extractManipulationState', $manipulation); SearchVariant::call('extractManipulationState', $manipulation);
// Then combine the manipulation back into object field sets // Then combine the manipulation back into object field sets
$writes = array(); $writes = array();
foreach ($manipulation as $table => $details) { foreach ($manipulation as $table => $details) {
if (!isset($details['id']) || !isset($details['fields'])) continue; if (!isset($details['id']) || !isset($details['fields'])) {
continue;
}
$id = $details['id']; $id = $details['id'];
$state = $details['state']; $state = $details['state'];
$class = $details['class']; $class = $details['class'];
$fields = $details['fields']; $fields = $details['fields'];
$base = ClassInfo::baseDataClass($class); $base = ClassInfo::baseDataClass($class);
$key = "$id:$base:".serialize($state); $key = "$id:$base:".serialize($state);
$statefulids = array(array('id' => $id, 'state' => $state)); $statefulids = array(array('id' => $id, 'state' => $state));
// Is this the first table for this particular object? Then add an item to $writes // Is this the first table for this particular object? Then add an item to $writes
if (!isset($writes[$key])) { if (!isset($writes[$key])) {
$writes[$key] = array( $writes[$key] = array(
'base' => $base, 'base' => $base,
'class' => $class, 'class' => $class,
'id' => $id, 'id' => $id,
'statefulids' => $statefulids, 'statefulids' => $statefulids,
'fields' => array() 'fields' => array()
); );
} }
// Otherwise update the class label if it's more specific than the currently recorded one // Otherwise update the class label if it's more specific than the currently recorded one
else if (is_subclass_of($class, $writes[$key]['class'])) { elseif (is_subclass_of($class, $writes[$key]['class'])) {
$writes[$key]['class'] = $class; $writes[$key]['class'] = $class;
} }
// Update the fields // Update the fields
foreach ($fields as $field => $value) { foreach ($fields as $field => $value) {
$writes[$key]['fields']["$class:$field"] = $value; $writes[$key]['fields']["$class:$field"] = $value;
} }
} }
// Then extract any state that is needed for the writes // Then extract any state that is needed for the writes
SearchVariant::call('extractManipulationWriteState', $writes); SearchVariant::call('extractManipulationWriteState', $writes);
// Submit all of these writes to the search processor // Submit all of these writes to the search processor
static::process_writes($writes); static::process_writes($writes);
} }
/** /**
* Send updates to the current search processor for execution * Send updates to the current search processor for execution
* *
* @param array $writes * @param array $writes
*/ */
public static function process_writes($writes) { public static function process_writes($writes)
foreach ($writes as $write) { {
// For every index foreach ($writes as $write) {
foreach (FullTextSearch::get_indexes() as $index => $instance) { // For every index
// If that index as a field from this class foreach (FullTextSearch::get_indexes() as $index => $instance) {
if (SearchIntrospection::is_subclass_of($write['class'], $instance->dependancyList)) { // If that index as a field from this class
// Get the dirty IDs if (SearchIntrospection::is_subclass_of($write['class'], $instance->dependancyList)) {
$dirtyids = $instance->getDirtyIDs($write['class'], $write['id'], $write['statefulids'], $write['fields']); // Get the dirty IDs
$dirtyids = $instance->getDirtyIDs($write['class'], $write['id'], $write['statefulids'], $write['fields']);
// Then add then then to the global list to deal with later // Then add then then to the global list to deal with later
foreach ($dirtyids as $dirtyclass => $ids) { foreach ($dirtyids as $dirtyclass => $ids) {
if ($ids) { if ($ids) {
if (!self::$processor) { if (!self::$processor) {
self::$processor = Injector::inst()->create('SearchUpdateProcessor'); self::$processor = Injector::inst()->create('SearchUpdateProcessor');
} }
self::$processor->addDirtyIDs($dirtyclass, $ids, $index); self::$processor->addDirtyIDs($dirtyclass, $ids, $index);
} }
} }
} }
} }
} }
// If we do have some work to do register the shutdown function to actually do the work // If we do have some work to do register the shutdown function to actually do the work
// Don't do it if we're testing - there's no database connection outside the test methods, so we'd // Don't do it if we're testing - there's no database connection outside the test methods, so we'd
// just get errors // just get errors
$runningTests = class_exists('SapphireTest', false) && SapphireTest::is_running_test(); $runningTests = class_exists('SapphireTest', false) && SapphireTest::is_running_test();
if (self::$processor && !self::$registered && !$runningTests) { if (self::$processor && !self::$registered && !$runningTests) {
register_shutdown_function(array("SearchUpdater", "flush_dirty_indexes")); register_shutdown_function(array("SearchUpdater", "flush_dirty_indexes"));
self::$registered = true; self::$registered = true;
} }
} }
/** /**
* Throw away the recorded dirty IDs without doing anything with them. * Throw away the recorded dirty IDs without doing anything with them.
*/ */
static function clear_dirty_indexes() { public static function clear_dirty_indexes()
self::$processor = null; {
} self::$processor = null;
}
/** /**
* Do something with the recorded dirty IDs, where that "something" depends on the value of self::$update_method, * Do something with the recorded dirty IDs, where that "something" depends on the value of self::$update_method,
* either immediately update the indexes, queue a messsage to update the indexes at some point in the future, or * either immediately update the indexes, queue a messsage to update the indexes at some point in the future, or
* just throw the dirty IDs away. * just throw the dirty IDs away.
*/ */
static function flush_dirty_indexes() { public static function flush_dirty_indexes()
if (!self::$processor) return; {
self::$processor->triggerProcessing(); if (!self::$processor) {
self::$processor = null; return;
} }
self::$processor->triggerProcessing();
self::$processor = null;
}
} }
class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter { class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter
public function preRequest(SS_HTTPRequest $request, Session $session, DataModel $model) { {
SearchUpdater::bind_manipulation_capture(); public function preRequest(SS_HTTPRequest $request, Session $session, DataModel $model)
} {
SearchUpdater::bind_manipulation_capture();
}
public function postRequest(SS_HTTPRequest $request, SS_HTTPResponse $response, DataModel $model) { public function postRequest(SS_HTTPRequest $request, SS_HTTPResponse $response, DataModel $model)
/* NOP */ {
} /* NOP */
}
} }
/** /**
@ -203,54 +217,57 @@ class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter {
* indexed. This causes the object to be marked for deletion from the index. * indexed. This causes the object to be marked for deletion from the index.
*/ */
class SearchUpdater_ObjectHandler extends DataExtension { class SearchUpdater_ObjectHandler extends DataExtension
{
public function onAfterDelete()
{
// Calling delete() on empty objects does nothing
if (!$this->owner->ID) {
return;
}
public function onAfterDelete() { // Force SearchUpdater to mark this record as dirty
// Calling delete() on empty objects does nothing $manipulation = array(
if (!$this->owner->ID) return; $this->owner->ClassName => array(
'fields' => array(),
'id' => $this->owner->ID,
'command' => 'update'
)
);
$this->owner->extend('augmentWrite', $manipulation);
SearchUpdater::handle_manipulation($manipulation);
}
// Force SearchUpdater to mark this record as dirty /**
$manipulation = array( * Forces this object to trigger a re-index in the current state
$this->owner->ClassName => array( */
'fields' => array(), public function triggerReindex()
'id' => $this->owner->ID, {
'command' => 'update' if (!$this->owner->ID) {
) return;
); }
$this->owner->extend('augmentWrite', $manipulation);
SearchUpdater::handle_manipulation($manipulation);
}
/** $id = $this->owner->ID;
* Forces this object to trigger a re-index in the current state $class = $this->owner->ClassName;
*/ $state = SearchVariant::current_state($class);
public function triggerReindex() { $base = ClassInfo::baseDataClass($class);
if (!$this->owner->ID) { $key = "$id:$base:".serialize($state);
return;
}
$id = $this->owner->ID; $statefulids = array(array(
$class = $this->owner->ClassName; 'id' => $id,
$state = SearchVariant::current_state($class); 'state' => $state
$base = ClassInfo::baseDataClass($class); ));
$key = "$id:$base:".serialize($state);
$statefulids = array(array( $writes = array(
'id' => $id, $key => array(
'state' => $state 'base' => $base,
)); 'class' => $class,
'id' => $id,
$writes = array( 'statefulids' => $statefulids,
$key => array( 'fields' => array()
'base' => $base, )
'class' => $class, );
'id' => $id,
'statefulids' => $statefulids,
'fields' => array()
)
);
SearchUpdater::process_writes($writes);
}
SearchUpdater::process_writes($writes);
}
} }

359
code/search/SearchVariant.php
View File

@ -4,205 +4,226 @@
* A Search Variant handles decorators and other situations where the items to reindex or search through are modified * A Search Variant handles decorators and other situations where the items to reindex or search through are modified
* from the default state - for instance, dealing with Versioned or Subsite * from the default state - for instance, dealing with Versioned or Subsite
*/ */
abstract class SearchVariant { abstract class SearchVariant
{
public function __construct()
{
}
function __construct() {} /*** OVERRIDES start here */
/*** OVERRIDES start here */ /**
* Variants can provide any functions they want, but they _must_ override these functions
* with specific ones
*/
/** /**
* Variants can provide any functions they want, but they _must_ override these functions * Return false if there is something missing from the environment (probably a
* with specific ones * not installed module) that means this variant can't apply to any class
*/ */
abstract public function appliesToEnvironment();
/** /**
* Return false if there is something missing from the environment (probably a * Return true if this variant applies to the passed class & subclass
* not installed module) that means this variant can't apply to any class */
*/ abstract public function appliesTo($class, $includeSubclasses);
abstract function appliesToEnvironment();
/** /**
* Return true if this variant applies to the passed class & subclass * Return the current state
*/ */
abstract function appliesTo($class, $includeSubclasses); abstract public function currentState();
/**
* Return all states to step through to reindex all items
*/
abstract public function reindexStates();
/**
* Activate the passed state
*/
abstract public function activateState($state);
/** /**
* Return the current state * Apply this variant to a search query
*/ *
abstract function currentState(); * @param SearchQuery $query
/** * @param SearchIndex $index
* Return all states to step through to reindex all items */
*/ abstract public function alterQuery($query, $index);
abstract function reindexStates();
/**
* Activate the passed state
*/
abstract function activateState($state);
/** /*** OVERRIDES end here*/
* Apply this variant to a search query
*
* @param SearchQuery $query
* @param SearchIndex $index
*/
abstract public function alterQuery($query, $index);
/*** OVERRIDES end here*/ /** Holds a cache of all variants */
protected static $variants = null;
/** Holds a cache of the variants keyed by "class!" "1"? (1 = include subclasses) */
protected static $class_variants = array();
/** Holds a cache of all variants */ /**
protected static $variants = null; * Returns an array of variants.
/** Holds a cache of the variants keyed by "class!" "1"? (1 = include subclasses) */ *
protected static $class_variants = array(); * With no arguments, returns all variants
*
* With a classname as the first argument, returns the variants that apply to that class
* (optionally including subclasses)
*
* @static
* @param string $class - The class name to get variants for
* @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class
* @return array - An array of (string)$variantClassName => (Object)$variantInstance pairs
*/
public static function variants($class = null, $includeSubclasses = true)
{
if (!$class) {
if (self::$variants === null) {
$classes = ClassInfo::subclassesFor('SearchVariant');
/** $concrete = array();
* Returns an array of variants. foreach ($classes as $variantclass) {
* $ref = new ReflectionClass($variantclass);
* With no arguments, returns all variants if ($ref->isInstantiable()) {
* $variant = singleton($variantclass);
* With a classname as the first argument, returns the variants that apply to that class if ($variant->appliesToEnvironment()) {
* (optionally including subclasses) $concrete[$variantclass] = $variant;
* }
* @static }
* @param string $class - The class name to get variants for }
* @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class
* @return array - An array of (string)$variantClassName => (Object)$variantInstance pairs
*/
public static function variants($class = null, $includeSubclasses = true) {
if (!$class) {
if (self::$variants === null) {
$classes = ClassInfo::subclassesFor('SearchVariant');
$concrete = array(); self::$variants = $concrete;
foreach ($classes as $variantclass) { }
$ref = new ReflectionClass($variantclass);
if ($ref->isInstantiable()) {
$variant = singleton($variantclass);
if ($variant->appliesToEnvironment()) $concrete[$variantclass] = $variant;
}
}
self::$variants = $concrete; return self::$variants;
} } else {
$key = $class . '!' . $includeSubclasses;
return self::$variants; if (!isset(self::$class_variants[$key])) {
} self::$class_variants[$key] = array();
else {
$key = $class . '!' . $includeSubclasses;
if (!isset(self::$class_variants[$key])) { foreach (self::variants() as $variantclass => $instance) {
self::$class_variants[$key] = array(); if ($instance->appliesTo($class, $includeSubclasses)) {
self::$class_variants[$key][$variantclass] = $instance;
}
}
}
foreach (self::variants() as $variantclass => $instance) { return self::$class_variants[$key];
if ($instance->appliesTo($class, $includeSubclasses)) self::$class_variants[$key][$variantclass] = $instance; }
} }
}
return self::$class_variants[$key]; /** Holds a cache of SearchVariant_Caller instances, one for each class/includeSubclasses setting */
} protected static $call_instances = array();
}
/** Holds a cache of SearchVariant_Caller instances, one for each class/includeSubclasses setting */ /**
protected static $call_instances = array(); * Lets you call any function on all variants that support it, in the same manner as "Object#extend" calls
* a method from extensions.
*
* Usage: SearchVariant::with(...)->call($method, $arg1, ...);
*
* @static
*
* @param string $class - (Optional) a classname. If passed, only variants that apply to that class will be checked / called
*
* @param bool $includeSubclasses - (Optional) If false, only variants that apply strictly to the passed class or its super-classes
* will be checked. If true (the default), variants that apply to any sub-class of the passed class with also be checked
*
* @return An object with one method, call()
*/
public static function with($class = null, $includeSubclasses = true)
{
// Make the cache key
$key = $class ? $class . '!' . $includeSubclasses : '!';
// If no SearchVariant_Caller instance yet, create it
if (!isset(self::$call_instances[$key])) {
self::$call_instances[$key] = new SearchVariant_Caller(self::variants($class, $includeSubclasses));
}
// Then return it
return self::$call_instances[$key];
}
/** /**
* Lets you call any function on all variants that support it, in the same manner as "Object#extend" calls * A shortcut to with when calling without passing in a class,
* a method from extensions. *
* * SearchVariant::call(...) ==== SearchVariant::with()->call(...);
* Usage: SearchVariant::with(...)->call($method, $arg1, ...); */
* public static function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null)
* @static {
* return self::with()->call($method, $a1, $a2, $a3, $a4, $a5, $a6, $a7);
* @param string $class - (Optional) a classname. If passed, only variants that apply to that class will be checked / called }
*
* @param bool $includeSubclasses - (Optional) If false, only variants that apply strictly to the passed class or its super-classes
* will be checked. If true (the default), variants that apply to any sub-class of the passed class with also be checked
*
* @return An object with one method, call()
*/
static function with($class = null, $includeSubclasses = true) {
// Make the cache key
$key = $class ? $class . '!' . $includeSubclasses : '!';
// If no SearchVariant_Caller instance yet, create it
if (!isset(self::$call_instances[$key])) self::$call_instances[$key] = new SearchVariant_Caller(self::variants($class, $includeSubclasses));
// Then return it
return self::$call_instances[$key];
}
/** /**
* A shortcut to with when calling without passing in a class, * Get the current state of every variant
* * @static
* SearchVariant::call(...) ==== SearchVariant::with()->call(...); * @return array
*/ */
static function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null) { public static function current_state($class = null, $includeSubclasses = true)
return self::with()->call($method, $a1, $a2, $a3, $a4, $a5, $a6, $a7); {
} $state = array();
foreach (self::variants($class, $includeSubclasses) as $variant => $instance) {
$state[$variant] = $instance->currentState();
}
return $state;
}
/** /**
* Get the current state of every variant * Activate all the states in the passed argument
* @static * @static
* @return array * @param (array) $state. A set of (string)$variantClass => (any)$state pairs , e.g. as returned by
*/ * SearchVariant::current_state()
static function current_state($class = null, $includeSubclasses = true) { * @return void
$state = array(); */
foreach (self::variants($class, $includeSubclasses) as $variant => $instance) { public static function activate_state($state)
$state[$variant] = $instance->currentState(); {
} foreach (self::variants() as $variant => $instance) {
return $state; if (isset($state[$variant])) {
} $instance->activateState($state[$variant]);
}
}
}
/** /**
* Activate all the states in the passed argument * Return an iterator that, when used in a for loop, activates one combination of reindex states per loop, and restores
* @static * back to the original state at the end
* @param (array) $state. A set of (string)$variantClass => (any)$state pairs , e.g. as returned by * @static
* SearchVariant::current_state() * @param string $class - The class name to get variants for
* @return void * @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class
*/ * @return SearchVariant_ReindexStateIteratorRet - The iterator to foreach loop over
static function activate_state($state) { */
foreach (self::variants() as $variant => $instance) { public static function reindex_states($class = null, $includeSubclasses = true)
if (isset($state[$variant])) $instance->activateState($state[$variant]); {
} $allstates = array();
}
/** foreach (self::variants($class, $includeSubclasses) as $variant => $instance) {
* Return an iterator that, when used in a for loop, activates one combination of reindex states per loop, and restores if ($states = $instance->reindexStates()) {
* back to the original state at the end $allstates[$variant] = $states;
* @static }
* @param string $class - The class name to get variants for }
* @param bool $includeSubclasses - True if variants should be included if they apply to at least one subclass of $class
* @return SearchVariant_ReindexStateIteratorRet - The iterator to foreach loop over
*/
static function reindex_states($class = null, $includeSubclasses = true) {
$allstates = array();
foreach (self::variants($class, $includeSubclasses) as $variant => $instance) { return $allstates ? new CombinationsArrayIterator($allstates) : array(array());
if ($states = $instance->reindexStates()) $allstates[$variant] = $states; }
}
return $allstates ? new CombinationsArrayIterator($allstates) : array(array());
}
} }
/** /**
* Internal utility class used to hold the state of the SearchVariant::with call * Internal utility class used to hold the state of the SearchVariant::with call
*/ */
class SearchVariant_Caller { class SearchVariant_Caller
protected $variants = null; {
protected $variants = null;
function __construct($variants) { public function __construct($variants)
$this->variants = $variants; {
} $this->variants = $variants;
}
function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null) { public function call($method, &$a1=null, &$a2=null, &$a3=null, &$a4=null, &$a5=null, &$a6=null, &$a7=null)
$values = array(); {
$values = array();
foreach ($this->variants as $variant) { foreach ($this->variants as $variant) {
if (method_exists($variant, $method)) { if (method_exists($variant, $method)) {
$value = $variant->$method($a1, $a2, $a3, $a4, $a5, $a6, $a7); $value = $variant->$method($a1, $a2, $a3, $a4, $a5, $a6, $a7);
if ($value !== null) $values[] = $value; if ($value !== null) {
} $values[] = $value;
} }
}
}
return $values; return $values;
} }
} }

148
code/search/SearchVariantSiteTreeSubsitesPolyhome.php
View File

@ -1,85 +1,95 @@
<?php <?php
class SearchVariantSiteTreeSubsitesPolyhome extends SearchVariant { class SearchVariantSiteTreeSubsitesPolyhome extends SearchVariant
{
public function appliesToEnvironment()
{
return class_exists('Subsite') && class_exists('SubsitePolyhome');
}
function appliesToEnvironment() { public function appliesTo($class, $includeSubclasses)
return class_exists('Subsite') && class_exists('SubsitePolyhome'); {
} return SearchIntrospection::has_extension($class, 'SiteTreeSubsitesPolyhome', $includeSubclasses);
}
function appliesTo($class, $includeSubclasses) { public function currentState()
return SearchIntrospection::has_extension($class, 'SiteTreeSubsitesPolyhome', $includeSubclasses); {
} return Subsite::currentSubsiteID();
}
public function reindexStates()
{
static $ids = null;
function currentState() { if ($ids === null) {
return Subsite::currentSubsiteID(); $ids = array(0);
} foreach (DataObject::get('Subsite') as $subsite) {
function reindexStates() { $ids[] = $subsite->ID;
static $ids = null; }
}
if ($ids === null) { return $ids;
$ids = array(0); }
foreach (DataObject::get('Subsite') as $subsite) $ids[] = $subsite->ID; public function activateState($state)
} {
if (Controller::has_curr()) {
Subsite::changeSubsite($state);
} else {
// TODO: This is a nasty hack - calling Subsite::changeSubsite after request ends
// throws error because no current controller to access session on
$_REQUEST['SubsiteID'] = $state;
}
}
return $ids; public function alterDefinition($base, $index)
} {
function activateState($state) { $self = get_class($this);
if (Controller::has_curr()) {
Subsite::changeSubsite($state); $index->filterFields['_subsite'] = array(
} 'name' => '_subsite',
else { 'field' => '_subsite',
// TODO: This is a nasty hack - calling Subsite::changeSubsite after request ends 'fullfield' => '_subsite',
// throws error because no current controller to access session on 'base' => $base,
$_REQUEST['SubsiteID'] = $state; 'origin' => $base,
} 'type' => 'Int',
} 'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
function alterDefinition($base, $index) { public function alterQuery($query, $index)
$self = get_class($this); {
$subsite = Subsite::currentSubsiteID();
$index->filterFields['_subsite'] = array( $query->filter('_subsite', array($subsite, SearchQuery::$missing));
'name' => '_subsite', }
'field' => '_subsite',
'fullfield' => '_subsite',
'base' => $base,
'origin' => $base,
'type' => 'Int',
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
public function alterQuery($query, $index) { public static $subsites = null;
$subsite = Subsite::currentSubsiteID();
$query->filter('_subsite', array($subsite, SearchQuery::$missing));
}
static $subsites = null; /**
* We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit
* deletes, just new versions with different members) so just always use all of them
*/
public function extractManipulationWriteState(&$writes)
{
$self = get_class($this);
/** foreach ($writes as $key => $write) {
* We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit if (!$this->appliesTo($write['class'], true)) {
* deletes, just new versions with different members) so just always use all of them continue;
*/ }
function extractManipulationWriteState(&$writes) {
$self = get_class($this);
foreach ($writes as $key => $write) { if (self::$subsites === null) {
if (!$this->appliesTo($write['class'], true)) continue; $query = new SQLQuery('ID', 'Subsite');
self::$subsites = array_merge(array('0'), $query->execute()->column());
}
if (self::$subsites === null) { $next = array();
$query = new SQLQuery('ID', 'Subsite');
self::$subsites = array_merge(array('0'), $query->execute()->column());
}
$next = array(); foreach ($write['statefulids'] as $i => $statefulid) {
foreach (self::$subsites as $subsiteID) {
foreach ($write['statefulids'] as $i => $statefulid) { $next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => $subsiteID)));
foreach (self::$subsites as $subsiteID) { }
$next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => $subsiteID))); }
}
}
$writes[$key]['statefulids'] = $next;
}
}
$writes[$key]['statefulids'] = $next;
}
}
} }

151
code/search/SearchVariantSubsites.php
View File

@ -1,88 +1,99 @@
<?php <?php
class SearchVariantSubsites extends SearchVariant { class SearchVariantSubsites extends SearchVariant
{
public function appliesToEnvironment()
{
return class_exists('Subsite');
}
function appliesToEnvironment() { public function appliesTo($class, $includeSubclasses)
return class_exists('Subsite'); {
} // Include all DataExtensions that contain a SubsiteID.
// TODO: refactor subsites to inherit a common interface, so we can run introspection once only.
return SearchIntrospection::has_extension($class, 'SiteTreeSubsites', $includeSubclasses) ||
SearchIntrospection::has_extension($class, 'GroupSubsites', $includeSubclasses) ||
SearchIntrospection::has_extension($class, 'FileSubsites', $includeSubclasses) ||
SearchIntrospection::has_extension($class, 'SiteConfigSubsites', $includeSubclasses);
}
function appliesTo($class, $includeSubclasses) { public function currentState()
// Include all DataExtensions that contain a SubsiteID. {
// TODO: refactor subsites to inherit a common interface, so we can run introspection once only. return (string)Subsite::currentSubsiteID();
return SearchIntrospection::has_extension($class, 'SiteTreeSubsites', $includeSubclasses) || }
SearchIntrospection::has_extension($class, 'GroupSubsites', $includeSubclasses) ||
SearchIntrospection::has_extension($class, 'FileSubsites', $includeSubclasses) ||
SearchIntrospection::has_extension($class, 'SiteConfigSubsites', $includeSubclasses);
}
function currentState() { public function reindexStates()
return (string)Subsite::currentSubsiteID(); {
} static $ids = null;
function reindexStates() { if ($ids === null) {
static $ids = null; $ids = array('0');
foreach (DataObject::get('Subsite') as $subsite) {
$ids[] = (string)$subsite->ID;
}
}
if ($ids === null) { return $ids;
$ids = array('0'); }
foreach (DataObject::get('Subsite') as $subsite) $ids[] = (string)$subsite->ID;
}
return $ids; public function activateState($state)
} {
// We always just set the $_GET variable rather than store in Session - this always works, has highest priority
// in Subsite::currentSubsiteID() and doesn't persist unlike Subsite::changeSubsite
$_GET['SubsiteID'] = $state;
Permission::flush_permission_cache();
}
function activateState($state) { public function alterDefinition($base, $index)
// We always just set the $_GET variable rather than store in Session - this always works, has highest priority {
// in Subsite::currentSubsiteID() and doesn't persist unlike Subsite::changeSubsite $self = get_class($this);
$_GET['SubsiteID'] = $state;
Permission::flush_permission_cache(); $index->filterFields['_subsite'] = array(
} 'name' => '_subsite',
'field' => '_subsite',
'fullfield' => '_subsite',
'base' => $base,
'origin' => $base,
'type' => 'Int',
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
function alterDefinition($base, $index) { public function alterQuery($query, $index)
$self = get_class($this); {
$subsite = Subsite::currentSubsiteID();
$index->filterFields['_subsite'] = array( $query->filter('_subsite', array($subsite, SearchQuery::$missing));
'name' => '_subsite', }
'field' => '_subsite',
'fullfield' => '_subsite',
'base' => $base,
'origin' => $base,
'type' => 'Int',
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
function alterQuery($query, $index) { public static $subsites = null;
$subsite = Subsite::currentSubsiteID();
$query->filter('_subsite', array($subsite, SearchQuery::$missing));
}
static $subsites = null; /**
* We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit
* deletes, just new versions with different members) so just always use all of them
*/
public function extractManipulationWriteState(&$writes)
{
$self = get_class($this);
/** foreach ($writes as $key => $write) {
* We need _really_ complicated logic to find just the changed subsites (because we use versions there's no explicit if (!$this->appliesTo($write['class'], true)) {
* deletes, just new versions with different members) so just always use all of them continue;
*/ }
function extractManipulationWriteState(&$writes) {
$self = get_class($this);
foreach ($writes as $key => $write) { if (self::$subsites === null) {
if (!$this->appliesTo($write['class'], true)) continue; $query = new SQLQuery('"ID"', '"Subsite"');
self::$subsites = array_merge(array('0'), $query->execute()->column());
}
if (self::$subsites === null) { $next = array();
$query = new SQLQuery('"ID"', '"Subsite"');
self::$subsites = array_merge(array('0'), $query->execute()->column());
}
$next = array(); foreach ($write['statefulids'] as $i => $statefulid) {
foreach (self::$subsites as $subsiteID) {
foreach ($write['statefulids'] as $i => $statefulid) { $next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => (string)$subsiteID)));
foreach (self::$subsites as $subsiteID) { }
$next[] = array('id' => $statefulid['id'], 'state' => array_merge($statefulid['state'], array($self => (string)$subsiteID))); }
}
}
$writes[$key]['statefulids'] = $next;
}
}
$writes[$key]['statefulids'] = $next;
}
}
} }

126
code/search/SearchVariantVersioned.php
View File

@ -1,70 +1,84 @@
<?php <?php
class SearchVariantVersioned extends SearchVariant { class SearchVariantVersioned extends SearchVariant
{
public function appliesToEnvironment()
{
return class_exists('Versioned');
}
function appliesToEnvironment() { public function appliesTo($class, $includeSubclasses)
return class_exists('Versioned'); {
} return SearchIntrospection::has_extension($class, 'Versioned', $includeSubclasses);
}
function appliesTo($class, $includeSubclasses) { public function currentState()
return SearchIntrospection::has_extension($class, 'Versioned', $includeSubclasses); {
} return Versioned::current_stage();
}
public function reindexStates()
{
return array('Stage', 'Live');
}
public function activateState($state)
{
Versioned::reading_stage($state);
}
function currentState() { return Versioned::current_stage(); } public function alterDefinition($base, $index)
function reindexStates() { return array('Stage', 'Live'); } {
function activateState($state) { Versioned::reading_stage($state); } $self = get_class($this);
function alterDefinition($base, $index) { $index->filterFields['_versionedstage'] = array(
$self = get_class($this); 'name' => '_versionedstage',
'field' => '_versionedstage',
'fullfield' => '_versionedstage',
'base' => $base,
'origin' => $base,
'type' => 'String',
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
$index->filterFields['_versionedstage'] = array( public function alterQuery($query, $index)
'name' => '_versionedstage', {
'field' => '_versionedstage', $stage = Versioned::current_stage();
'fullfield' => '_versionedstage', $query->filter('_versionedstage', array($stage, SearchQuery::$missing));
'base' => $base, }
'origin' => $base,
'type' => 'String', public function extractManipulationState(&$manipulation)
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState')) {
); $self = get_class($this);
}
foreach ($manipulation as $table => $details) {
$class = $details['class'];
$stage = 'Stage';
public function alterQuery($query, $index) { if (preg_match('/^(.*)_Live$/', $table, $matches)) {
$stage = Versioned::current_stage(); $class = $matches[1];
$query->filter('_versionedstage', array($stage, SearchQuery::$missing)); $stage = 'Live';
} }
function extractManipulationState(&$manipulation) {
$self = get_class($this);
foreach ($manipulation as $table => $details) {
$class = $details['class'];
$stage = 'Stage';
if (preg_match('/^(.*)_Live$/', $table, $matches)) { if (ClassInfo::exists($class) && $this->appliesTo($class, false)) {
$class = $matches[1]; $manipulation[$table]['class'] = $class;
$stage = 'Live'; $manipulation[$table]['state'][$self] = $stage;
} }
}
}
if (ClassInfo::exists($class) && $this->appliesTo($class, false)) { public function extractStates(&$table, &$ids, &$fields)
$manipulation[$table]['class'] = $class; {
$manipulation[$table]['state'][$self] = $stage; $class = $table;
} $suffix = null;
}
}
function extractStates(&$table, &$ids, &$fields) {
$class = $table;
$suffix = null;
if (ClassInfo::exists($class) && $this->appliesTo($class, false)) { if (ClassInfo::exists($class) && $this->appliesTo($class, false)) {
$table = $class; $table = $class;
$self = get_class($this); $self = get_class($this);
foreach ($ids as $i => $statefulid) {
$ids[$i]['state'][$self] = $suffix ? $suffix : 'Stage';
}
}
}
foreach ($ids as $i => $statefulid) {
$ids[$i]['state'][$self] = $suffix ? $suffix : 'Stage';
}
}
}
} }

319
code/search/processors/SearchUpdateBatchedProcessor.php
View File

@ -3,155 +3,176 @@
/** /**
* Provides batching of search updates * Provides batching of search updates
*/ */
abstract class SearchUpdateBatchedProcessor extends SearchUpdateProcessor { abstract class SearchUpdateBatchedProcessor extends SearchUpdateProcessor
{
/** /**
* List of batches to be processed * List of batches to be processed
* *
* @var array * @var array
*/ */
protected $batches; protected $batches;
/** /**
* Pointer to index of $batches assigned to $current. * Pointer to index of $batches assigned to $current.
* Set to 0 (first index) if not started, or count + 1 if completed. * Set to 0 (first index) if not started, or count + 1 if completed.
* *
* @var int * @var int
*/ */
protected $currentBatch; protected $currentBatch;
/** /**
* List of indexes successfully comitted in the current batch * List of indexes successfully comitted in the current batch
* *
* @var array * @var array
*/ */
protected $completedIndexes; protected $completedIndexes;
/** /**
* Maximum number of record-states to process in one batch. * Maximum number of record-states to process in one batch.
* Set to zero to process all records in a single batch * Set to zero to process all records in a single batch
* *
* @config * @config
* @var int * @var int
*/ */
private static $batch_size = 100; private static $batch_size = 100;
/** /**
* Up to this number of additional ids can be added to any batch in order to reduce the number * Up to this number of additional ids can be added to any batch in order to reduce the number
* of batches * of batches
* *
* @config * @config
* @var int * @var int
*/ */
private static $batch_soft_cap = 10; private static $batch_soft_cap = 10;
public function __construct() { public function __construct()
parent::__construct(); {
parent::__construct();
$this->batches = array();
$this->setBatch(0); $this->batches = array();
} $this->setBatch(0);
}
/**
* Set the current batch index /**
* * Set the current batch index
* @param int $batch Index of the batch *
*/ * @param int $batch Index of the batch
protected function setBatch($batch) { */
$this->currentBatch = $batch; protected function setBatch($batch)
} {
$this->currentBatch = $batch;
protected function getSource() { }
if(isset($this->batches[$this->currentBatch])) {
return $this->batches[$this->currentBatch]; protected function getSource()
} {
} if (isset($this->batches[$this->currentBatch])) {
return $this->batches[$this->currentBatch];
/** }
* Process the current queue }
*
* @return boolean /**
*/ * Process the current queue
public function process() { *
// Skip blank queues * @return boolean
if(empty($this->batches)) return true; */
public function process()
// Don't re-process completed queue {
if($this->currentBatch >= count($this->batches)) return true; // Skip blank queues
if (empty($this->batches)) {
// Send current patch to indexes return true;
$this->prepareIndexes(); }
// Advance to next batch if successful // Don't re-process completed queue
$this->setBatch($this->currentBatch + 1); if ($this->currentBatch >= count($this->batches)) {
return true; return true;
} }
/** // Send current patch to indexes
* Segments batches acording to the specified rules $this->prepareIndexes();
*
* @param array $source Source input // Advance to next batch if successful
* @return array Batches $this->setBatch($this->currentBatch + 1);
*/ return true;
protected function segmentBatches($source) { }
// Measure batch_size
$batchSize = Config::inst()->get(get_class(), 'batch_size'); /**
if($batchSize === 0) return array($source); * Segments batches acording to the specified rules
$softCap = Config::inst()->get(get_class(), 'batch_soft_cap'); *
* @param array $source Source input
// Clear batches * @return array Batches
$batches = array(); */
$current = array(); protected function segmentBatches($source)
$currentSize = 0; {
// Measure batch_size
$batchSize = Config::inst()->get(get_class(), 'batch_size');
if ($batchSize === 0) {
return array($source);
}
$softCap = Config::inst()->get(get_class(), 'batch_soft_cap');
// Clear batches
$batches = array();
$current = array();
$currentSize = 0;
// Build batches from data // Build batches from data
foreach ($source as $base => $statefulids) { foreach ($source as $base => $statefulids) {
if (!$statefulids) continue; if (!$statefulids) {
continue;
}
foreach ($statefulids as $stateKey => $statefulid) { foreach ($statefulids as $stateKey => $statefulid) {
$state = $statefulid['state']; $state = $statefulid['state'];
$ids = $statefulid['ids']; $ids = $statefulid['ids'];
if(!$ids) continue; if (!$ids) {
continue;
// Extract items from $ids until empty }
while($ids) {
// Estimate maximum number of items to take for this iteration, allowing for the soft cap // Extract items from $ids until empty
$take = $batchSize - $currentSize; while ($ids) {
if(count($ids) <= $take + $softCap) $take += $softCap; // Estimate maximum number of items to take for this iteration, allowing for the soft cap
$items = array_slice($ids, 0, $take, true); $take = $batchSize - $currentSize;
$ids = array_slice($ids, count($items), null, true); if (count($ids) <= $take + $softCap) {
$take += $softCap;
// Update batch }
$currentSize += count($items); $items = array_slice($ids, 0, $take, true);
$merge = array( $ids = array_slice($ids, count($items), null, true);
$base => array(
$stateKey => array( // Update batch
'state' => $state, $currentSize += count($items);
'ids' => $items $merge = array(
) $base => array(
) $stateKey => array(
); 'state' => $state,
$current = $current ? array_merge_recursive($current, $merge) : $merge; 'ids' => $items
if($currentSize >= $batchSize) { )
$batches[] = $current; )
$current = array(); );
$currentSize = 0; $current = $current ? array_merge_recursive($current, $merge) : $merge;
} if ($currentSize >= $batchSize) {
} $batches[] = $current;
} $current = array();
} $currentSize = 0;
// Add incomplete batch }
if($currentSize) $batches[] = $current; }
}
return $batches; }
} // Add incomplete batch
if ($currentSize) {
public function batchData() { $batches[] = $current;
$this->batches = $this->segmentBatches($this->dirty); }
$this->setBatch(0);
} return $batches;
}
public function triggerProcessing() {
$this->batchData(); public function batchData()
} {
$this->batches = $this->segmentBatches($this->dirty);
$this->setBatch(0);
}
public function triggerProcessing()
{
$this->batchData();
}
} }

509
code/search/processors/SearchUpdateCommitJobProcessor.php
View File

@ -1,249 +1,266 @@
<?php <?php
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
class SearchUpdateCommitJobProcessor implements QueuedJob { }
/** class SearchUpdateCommitJobProcessor implements QueuedJob
* The QueuedJob queue to use when processing commits {
* /**
* @config * The QueuedJob queue to use when processing commits
* @var int *
*/ * @config
private static $commit_queue = 2; // QueuedJob::QUEUED; * @var int
*/
/** private static $commit_queue = 2; // QueuedJob::QUEUED;
* List of indexes to commit
* /**
* @var array * List of indexes to commit
*/ *
protected $indexes = array(); * @var array
*/
/** protected $indexes = array();
* True if this job is skipped to be be re-scheduled in the future
* /**
* @var boolean * True if this job is skipped to be be re-scheduled in the future
*/ *
protected $skipped = false; * @var boolean
*/
/** protected $skipped = false;
* List of completed indexes
* /**
* @var array * List of completed indexes
*/ *
protected $completed = array(); * @var array
*/
/** protected $completed = array();
* List of messages
* /**
* @var array * List of messages
*/ *
protected $messages = array(); * @var array
*/
/** protected $messages = array();
* List of dirty indexes to be committed
* /**
* @var array * List of dirty indexes to be committed
*/ *
public static $dirty_indexes = true; * @var array
*/
/** public static $dirty_indexes = true;
* If solrindex::commit has already been performed, but additional commits are necessary,
* how long do we wait before attempting to touch the index again? /**
* * If solrindex::commit has already been performed, but additional commits are necessary,
* {@see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers} * how long do we wait before attempting to touch the index again?
* *
* @var int * {@see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers}
* @config *
*/ * @var int
private static $cooldown = 300; * @config
*/
/** private static $cooldown = 300;
* True if any commits have been executed this request. If so, any attempts to run subsequent commits
* should be delayed until next queuedjob to prevent solr reaching maxWarmingSearchers /**
* * True if any commits have been executed this request. If so, any attempts to run subsequent commits
* {@see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers} * should be delayed until next queuedjob to prevent solr reaching maxWarmingSearchers
* *
* @var boolean * {@see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers}
*/ *
public static $has_run = false; * @var boolean
*/
/** public static $has_run = false;
* This method is invoked once indexes with dirty ids have been updapted and a commit is necessary
* /**
* @param boolean $dirty Marks all indexes as dirty by default. Set to false if there are known comitted and * This method is invoked once indexes with dirty ids have been updapted and a commit is necessary
* clean indexes *
* @param string $startAfter Start date * @param boolean $dirty Marks all indexes as dirty by default. Set to false if there are known comitted and
* @return int The ID of the next queuedjob to run. This could be a new one or an existing one. * clean indexes
*/ * @param string $startAfter Start date
public static function queue($dirty = true, $startAfter = null) { * @return int The ID of the next queuedjob to run. This could be a new one or an existing one.
$commit = Injector::inst()->create(__CLASS__); */
$id = singleton('QueuedJobService')->queueJob($commit, $startAfter); public static function queue($dirty = true, $startAfter = null)
{
if($dirty) { $commit = Injector::inst()->create(__CLASS__);
$indexes = FullTextSearch::get_indexes(); $id = singleton('QueuedJobService')->queueJob($commit, $startAfter);
static::$dirty_indexes = array_keys($indexes);
} if ($dirty) {
return $id; $indexes = FullTextSearch::get_indexes();
} static::$dirty_indexes = array_keys($indexes);
}
public function getJobType() { return $id;
return Config::inst()->get(__CLASS__, 'commit_queue'); }
}
public function getJobType()
public function getSignature() { {
// There is only ever one commit job on the queue so the signature is consistent return Config::inst()->get(__CLASS__, 'commit_queue');
// See QueuedJobService::queueJob() for the code that prevents duplication }
return __CLASS__;
} public function getSignature()
{
public function getTitle() { // There is only ever one commit job on the queue so the signature is consistent
return "FullTextSearch Commit Job"; // See QueuedJobService::queueJob() for the code that prevents duplication
} return __CLASS__;
}
/**
* Get the list of index names we should process public function getTitle()
* {
* @return array return "FullTextSearch Commit Job";
*/ }
public function getAllIndexes() {
if(empty($this->indexes)) { /**
$indexes = FullTextSearch::get_indexes(); * Get the list of index names we should process
$this->indexes = array_keys($indexes); *
} * @return array
return $this->indexes; */
} public function getAllIndexes()
{
public function jobFinished() { if (empty($this->indexes)) {
// If we've indexed exactly as many as we would like, we are done $indexes = FullTextSearch::get_indexes();
return $this->skipped $this->indexes = array_keys($indexes);
|| (count($this->getAllIndexes()) <= count($this->completed)); }
} return $this->indexes;
}
public function prepareForRestart() {
// NOOP public function jobFinished()
} {
// If we've indexed exactly as many as we would like, we are done
public function afterComplete() { return $this->skipped
// NOOP || (count($this->getAllIndexes()) <= count($this->completed));
} }
/** public function prepareForRestart()
* Abort this job, potentially rescheduling a replacement if there is still work to do {
*/ // NOOP
protected function discardJob() { }
$this->skipped = true;
public function afterComplete()
// If we do not have dirty records, then assume that these dirty records were committed {
// already this request (but probably another job), so we don't need to commit anything else. // NOOP
// This could occur if we completed multiple searchupdate jobs in a prior request, and }
// we only need one commit job to commit all of them in the current request.
if(empty(static::$dirty_indexes)) { /**
$this->addMessage("Indexing already completed this request: Discarding this job"); * Abort this job, potentially rescheduling a replacement if there is still work to do
return; */
} protected function discardJob()
{
$this->skipped = true;
// If any commit has run, but some (or all) indexes are un-comitted, we must re-schedule this task.
// This could occur if we completed a searchupdate job in a prior request, as well as in // If we do not have dirty records, then assume that these dirty records were committed
// the current request // already this request (but probably another job), so we don't need to commit anything else.
$cooldown = Config::inst()->get(__CLASS__, 'cooldown'); // This could occur if we completed multiple searchupdate jobs in a prior request, and
$now = new DateTime(SS_Datetime::now()->getValue()); // we only need one commit job to commit all of them in the current request.
$now->add(new DateInterval('PT'.$cooldown.'S')); if (empty(static::$dirty_indexes)) {
$runat = $now->Format('Y-m-d H:i:s'); $this->addMessage("Indexing already completed this request: Discarding this job");
return;
$this->addMessage("Indexing already run this request, but incomplete. Re-scheduling for {$runat}"); }
// Queue after the given cooldown
static::queue(false, $runat); // If any commit has run, but some (or all) indexes are un-comitted, we must re-schedule this task.
} // This could occur if we completed a searchupdate job in a prior request, as well as in
// the current request
public function process() { $cooldown = Config::inst()->get(__CLASS__, 'cooldown');
// If we have already run an instance of SearchUpdateCommitJobProcessor this request, immediately $now = new DateTime(SS_Datetime::now()->getValue());
// quit this job to prevent hitting warming search limits in Solr $now->add(new DateInterval('PT'.$cooldown.'S'));
if(static::$has_run) { $runat = $now->Format('Y-m-d H:i:s');
$this->discardJob();
return true; $this->addMessage("Indexing already run this request, but incomplete. Re-scheduling for {$runat}");
}
// Queue after the given cooldown
// To prevent other commit jobs from running this request static::queue(false, $runat);
static::$has_run = true; }
// Run all incompleted indexes public function process()
$indexNames = $this->getAllIndexes(); {
foreach ($indexNames as $name) { // If we have already run an instance of SearchUpdateCommitJobProcessor this request, immediately
$index = singleton($name); // quit this job to prevent hitting warming search limits in Solr
$this->commitIndex($index); if (static::$has_run) {
} $this->discardJob();
return true;
$this->addMessage("All indexes committed"); }
return true; // To prevent other commit jobs from running this request
} static::$has_run = true;
/** // Run all incompleted indexes
* Commits a specific index $indexNames = $this->getAllIndexes();
* foreach ($indexNames as $name) {
* @param SolrIndex $index $index = singleton($name);
* @throws Exception $this->commitIndex($index);
*/ }
protected function commitIndex($index) {
// Skip index if this is already complete $this->addMessage("All indexes committed");
$name = get_class($index);
if(in_array($name, $this->completed)) { return true;
$this->addMessage("Skipping already comitted index {$name}"); }
return;
} /**
* Commits a specific index
// Bypass SolrIndex::commit exception handling so that queuedjobs can handle the error *
$this->addMessage("Committing index {$name}"); * @param SolrIndex $index
$index->getService()->commit(false, false, false); * @throws Exception
$this->addMessage("Committing index {$name} was successful"); */
protected function commitIndex($index)
// If this index is currently marked as dirty, it's now clean {
if(in_array($name, static::$dirty_indexes)) { // Skip index if this is already complete
static::$dirty_indexes = array_diff(static::$dirty_indexes, array($name)); $name = get_class($index);
} if (in_array($name, $this->completed)) {
$this->addMessage("Skipping already comitted index {$name}");
// Mark complete return;
$this->completed[] = $name; }
}
// Bypass SolrIndex::commit exception handling so that queuedjobs can handle the error
public function setup() { $this->addMessage("Committing index {$name}");
// NOOP $index->getService()->commit(false, false, false);
} $this->addMessage("Committing index {$name} was successful");
public function getJobData() { // If this index is currently marked as dirty, it's now clean
$data = new stdClass(); if (in_array($name, static::$dirty_indexes)) {
$data->totalSteps = count($this->getAllIndexes()); static::$dirty_indexes = array_diff(static::$dirty_indexes, array($name));
$data->currentStep = count($this->completed); }
$data->isComplete = $this->jobFinished();
$data->messages = $this->messages; // Mark complete
$this->completed[] = $name;
$data->jobData = new stdClass(); }
$data->jobData->skipped = $this->skipped;
$data->jobData->completed = $this->completed; public function setup()
$data->jobData->indexes = $this->getAllIndexes(); {
// NOOP
return $data; }
}
public function getJobData()
public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { {
$this->isComplete = $isComplete; $data = new stdClass();
$this->messages = $messages; $data->totalSteps = count($this->getAllIndexes());
$data->currentStep = count($this->completed);
$this->skipped = $jobData->skipped; $data->isComplete = $this->jobFinished();
$this->completed = $jobData->completed; $data->messages = $this->messages;
$this->indexes = $jobData->indexes;
} $data->jobData = new stdClass();
$data->jobData->skipped = $this->skipped;
public function addMessage($message, $severity='INFO') { $data->jobData->completed = $this->completed;
$severity = strtoupper($severity); $data->jobData->indexes = $this->getAllIndexes();
$this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message";
} return $data;
}
public function getMessages() {
return $this->messages; public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages)
} {
$this->isComplete = $isComplete;
$this->messages = $messages;
$this->skipped = $jobData->skipped;
$this->completed = $jobData->completed;
$this->indexes = $jobData->indexes;
}
public function addMessage($message, $severity='INFO')
{
$severity = strtoupper($severity);
$this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message";
}
public function getMessages()
{
return $this->messages;
}
} }

10
code/search/processors/SearchUpdateImmediateProcessor.php
View File

@ -1,7 +1,9 @@
<?php <?php
class SearchUpdateImmediateProcessor extends SearchUpdateProcessor { class SearchUpdateImmediateProcessor extends SearchUpdateProcessor
public function triggerProcessing() { {
$this->process(); public function triggerProcessing()
} {
$this->process();
}
} }

28
code/search/processors/SearchUpdateMessageQueueProcessor.php
View File

@ -1,17 +1,19 @@
<?php <?php
class SearchUpdateMessageQueueProcessor extends SearchUpdateProcessor { class SearchUpdateMessageQueueProcessor extends SearchUpdateProcessor
/** {
* The MessageQueue to use when processing updates /**
* @config * The MessageQueue to use when processing updates
* @var string * @config
*/ * @var string
private static $reindex_queue = "search_indexing"; */
private static $reindex_queue = "search_indexing";
public function triggerProcessing() { public function triggerProcessing()
MessageQueue::send( {
Config::inst()->get('SearchMessageQueueUpdater', 'reindex_queue'), MessageQueue::send(
new MethodInvocationMessage($this, "process") Config::inst()->get('SearchMessageQueueUpdater', 'reindex_queue'),
); new MethodInvocationMessage($this, "process")
} );
}
} }

257
code/search/processors/SearchUpdateProcessor.php
View File

@ -1,137 +1,146 @@
<?php <?php
abstract class SearchUpdateProcessor { abstract class SearchUpdateProcessor
{
/** /**
* List of dirty records to process in format * List of dirty records to process in format
* *
* array( * array(
* '$BaseClass' => array( * '$BaseClass' => array(
* '$State Key' => array( * '$State Key' => array(
* 'state' => array( * 'state' => array(
* 'key1' => 'value', * 'key1' => 'value',
* 'key2' => 'value' * 'key2' => 'value'
* ), * ),
* 'ids' => array( * 'ids' => array(
* '*id*' => array( * '*id*' => array(
* '*Index Name 1*', * '*Index Name 1*',
* '*Index Name 2*' * '*Index Name 2*'
* ) * )
* ) * )
* ) * )
* ) * )
* ) * )
* *
* @var array * @var array
*/ */
protected $dirty; protected $dirty;
public function __construct() { public function __construct()
$this->dirty = array(); {
} $this->dirty = array();
}
public function addDirtyIDs($class, $statefulids, $index) { public function addDirtyIDs($class, $statefulids, $index)
$base = ClassInfo::baseDataClass($class); {
$forclass = isset($this->dirty[$base]) ? $this->dirty[$base] : array(); $base = ClassInfo::baseDataClass($class);
$forclass = isset($this->dirty[$base]) ? $this->dirty[$base] : array();
foreach ($statefulids as $statefulid) { foreach ($statefulids as $statefulid) {
$id = $statefulid['id']; $id = $statefulid['id'];
$state = $statefulid['state']; $statekey = serialize($state); $state = $statefulid['state'];
$statekey = serialize($state);
if (!isset($forclass[$statekey])) { if (!isset($forclass[$statekey])) {
$forclass[$statekey] = array('state' => $state, 'ids' => array($id => array($index))); $forclass[$statekey] = array('state' => $state, 'ids' => array($id => array($index)));
} } elseif (!isset($forclass[$statekey]['ids'][$id])) {
else if (!isset($forclass[$statekey]['ids'][$id])) { $forclass[$statekey]['ids'][$id] = array($index);
$forclass[$statekey]['ids'][$id] = array($index); } elseif (array_search($index, $forclass[$statekey]['ids'][$id]) === false) {
} $forclass[$statekey]['ids'][$id][] = $index;
else if (array_search($index, $forclass[$statekey]['ids'][$id]) === false) { // dirty count stays the same
$forclass[$statekey]['ids'][$id][] = $index; }
// dirty count stays the same }
}
}
$this->dirty[$base] = $forclass; $this->dirty[$base] = $forclass;
} }
/** /**
* Generates the list of indexes to process for the dirty items * Generates the list of indexes to process for the dirty items
* *
* @return array * @return array
*/ */
protected function prepareIndexes() { protected function prepareIndexes()
$originalState = SearchVariant::current_state(); {
$dirtyIndexes = array(); $originalState = SearchVariant::current_state();
$dirty = $this->getSource(); $dirtyIndexes = array();
$indexes = FullTextSearch::get_indexes(); $dirty = $this->getSource();
foreach ($dirty as $base => $statefulids) { $indexes = FullTextSearch::get_indexes();
if (!$statefulids) continue; foreach ($dirty as $base => $statefulids) {
if (!$statefulids) {
continue;
}
foreach ($statefulids as $statefulid) { foreach ($statefulids as $statefulid) {
$state = $statefulid['state']; $state = $statefulid['state'];
$ids = $statefulid['ids']; $ids = $statefulid['ids'];
SearchVariant::activate_state($state); SearchVariant::activate_state($state);
// Ensure that indexes for all new / updated objects are included // Ensure that indexes for all new / updated objects are included
$objs = DataObject::get($base)->byIDs(array_keys($ids)); $objs = DataObject::get($base)->byIDs(array_keys($ids));
foreach ($objs as $obj) { foreach ($objs as $obj) {
foreach ($ids[$obj->ID] as $index) { foreach ($ids[$obj->ID] as $index) {
if (!$indexes[$index]->variantStateExcluded($state)) { if (!$indexes[$index]->variantStateExcluded($state)) {
$indexes[$index]->add($obj); $indexes[$index]->add($obj);
$dirtyIndexes[$index] = $indexes[$index]; $dirtyIndexes[$index] = $indexes[$index];
} }
} }
unset($ids[$obj->ID]); unset($ids[$obj->ID]);
} }
// Generate list of records that do not exist and should be removed // Generate list of records that do not exist and should be removed
foreach ($ids as $id => $fromindexes) { foreach ($ids as $id => $fromindexes) {
foreach ($fromindexes as $index) { foreach ($fromindexes as $index) {
if (!$indexes[$index]->variantStateExcluded($state)) { if (!$indexes[$index]->variantStateExcluded($state)) {
$indexes[$index]->delete($base, $id, $state); $indexes[$index]->delete($base, $id, $state);
$dirtyIndexes[$index] = $indexes[$index]; $dirtyIndexes[$index] = $indexes[$index];
} }
} }
} }
} }
} }
SearchVariant::activate_state($originalState); SearchVariant::activate_state($originalState);
return $dirtyIndexes; return $dirtyIndexes;
} }
/** /**
* Commits the specified index to the Solr service * Commits the specified index to the Solr service
* *
* @param SolrIndex $index Index object * @param SolrIndex $index Index object
* @return bool Flag indicating success * @return bool Flag indicating success
*/ */
protected function commitIndex($index) { protected function commitIndex($index)
return $index->commit() !== false; {
} return $index->commit() !== false;
}
/**
* Gets the record data source to process /**
* * Gets the record data source to process
* @return array *
*/ * @return array
protected function getSource() { */
return $this->dirty; protected function getSource()
} {
return $this->dirty;
}
/** /**
* Process all indexes, returning true if successful * Process all indexes, returning true if successful
* *
* @return bool Flag indicating success * @return bool Flag indicating success
*/ */
public function process() { public function process()
// Generate and commit all instances {
$indexes = $this->prepareIndexes(); // Generate and commit all instances
foreach ($indexes as $index) { $indexes = $this->prepareIndexes();
if(!$this->commitIndex($index)) return false; foreach ($indexes as $index) {
} if (!$this->commitIndex($index)) {
return true; return false;
} }
}
return true;
}
abstract public function triggerProcessing(); abstract public function triggerProcessing();
} }

182
code/search/processors/SearchUpdateQueuedJobProcessor.php
View File

@ -1,87 +1,101 @@
<?php <?php
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
class SearchUpdateQueuedJobProcessor extends SearchUpdateBatchedProcessor implements QueuedJob { }
/** class SearchUpdateQueuedJobProcessor extends SearchUpdateBatchedProcessor implements QueuedJob
* The QueuedJob queue to use when processing updates {
* @config /**
* @var int * The QueuedJob queue to use when processing updates
*/ * @config
private static $reindex_queue = 2; // QueuedJob::QUEUED; * @var int
*/
protected $messages = array(); private static $reindex_queue = 2; // QueuedJob::QUEUED;
public function triggerProcessing() { protected $messages = array();
parent::triggerProcessing();
singleton('QueuedJobService')->queueJob($this); public function triggerProcessing()
} {
parent::triggerProcessing();
public function getTitle() { singleton('QueuedJobService')->queueJob($this);
return "FullTextSearch Update Job"; }
}
public function getTitle()
public function getSignature() { {
return md5(get_class($this) . time() . mt_rand(0, 100000)); return "FullTextSearch Update Job";
} }
public function getJobType() { public function getSignature()
return Config::inst()->get('SearchUpdateQueuedJobProcessor', 'reindex_queue'); {
} return md5(get_class($this) . time() . mt_rand(0, 100000));
}
public function jobFinished() {
return $this->currentBatch >= count($this->batches); public function getJobType()
} {
return Config::inst()->get('SearchUpdateQueuedJobProcessor', 'reindex_queue');
public function setup() { }
// NOP
} public function jobFinished()
{
public function prepareForRestart() { return $this->currentBatch >= count($this->batches);
// NOP }
}
public function setup()
public function afterComplete() { {
// Once indexing is complete, commit later in order to avoid solr limits // NOP
// see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers }
SearchUpdateCommitJobProcessor::queue();
} public function prepareForRestart()
{
public function getJobData() { // NOP
$data = new stdClass(); }
$data->totalSteps = count($this->batches);
$data->currentStep = $this->currentBatch; public function afterComplete()
$data->isComplete = $this->jobFinished(); {
$data->messages = $this->messages; // Once indexing is complete, commit later in order to avoid solr limits
// see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers
$data->jobData = new stdClass(); SearchUpdateCommitJobProcessor::queue();
$data->jobData->batches = $this->batches; }
$data->jobData->currentBatch = $this->currentBatch;
public function getJobData()
return $data; {
} $data = new stdClass();
$data->totalSteps = count($this->batches);
public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { $data->currentStep = $this->currentBatch;
$this->isComplete = $isComplete; $data->isComplete = $this->jobFinished();
$this->messages = $messages; $data->messages = $this->messages;
$this->batches = $jobData->batches; $data->jobData = new stdClass();
$this->currentBatch = $jobData->currentBatch; $data->jobData->batches = $this->batches;
} $data->jobData->currentBatch = $this->currentBatch;
public function addMessage($message, $severity='INFO') { return $data;
$severity = strtoupper($severity); }
$this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message";
} public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages)
{
public function process() { $this->isComplete = $isComplete;
$result = parent::process(); $this->messages = $messages;
if($this->jobFinished()) { $this->batches = $jobData->batches;
$this->addMessage("All batched updates complete. Queuing commit job"); $this->currentBatch = $jobData->currentBatch;
} }
return $result; public function addMessage($message, $severity='INFO')
} {
$severity = strtoupper($severity);
$this->messages[] = '[' . date('Y-m-d H:i:s') . "][$severity] $message";
}
public function process()
{
$result = parent::process();
if ($this->jobFinished()) {
$this->addMessage("All batched updates complete. Queuing commit job");
}
return $result;
}
} }

676
code/solr/Solr.php
View File

@ -5,284 +5,296 @@ use Monolog\Handler\StreamHandler;
use Monolog\Logger; use Monolog\Logger;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
class Solr { class Solr
{
/**
* Configuration on where to find the solr server and how to get new index configurations into it.
*
* Required fields:
* host (default: localhost) - The host or IP Solr is listening on
* port (default: 8983) - The port Solr is listening on
* path (default: /solr) - The suburl the solr service is available on
*
* Optional fields:
* version (default: 4) - The Solr server version. Currently supports 3 and 4 (you can add a sub-version like 4.5 if
* you like, but currently it has no effect)
* service (default: depends on version, Solr3Service for 3, Solr4Service for 4)
* the class that provides actual communcation to the Solr server
* extraspath (default: <basefolder>/fulltextsearch/conf/solr/{version}/extras/) - Absolute path to
* the folder containing templates which are used for generating the schema and field definitions.
* templates (default: <basefolder>/fulltextsearch/conf/solr/{version}/templates/) - Absolute path to
* the configuration default files, e.g. solrconfig.xml.
*
* indexstore => an array with
*
* mode - a classname which implements SolrConfigStore, or 'file' or 'webdav'
*
* When mode == SolrConfigStore_File or file (indexes should be written on a local filesystem)
* path - The (locally accessible) path to write the index configurations to.
* remotepath (default: the same as indexpath) - The path that the Solr server will read the index configurations from
*
* When mode == SolrConfigStore_WebDAV or webdav (indexes should stored on a remote Solr server via webdav)
* auth (default: none) - A username:password pair string to use to auth against the webdav server
* path (default: /solrindex) - The suburl on the solr host that is set up to accept index configurations via webdav
* remotepath - The path that the Solr server will read the index configurations from
*/
protected static $solr_options = array();
/** /** A cache of solr_options with the defaults all merged in */
* Configuration on where to find the solr server and how to get new index configurations into it. protected static $merged_solr_options = null;
*
* Required fields:
* host (default: localhost) - The host or IP Solr is listening on
* port (default: 8983) - The port Solr is listening on
* path (default: /solr) - The suburl the solr service is available on
*
* Optional fields:
* version (default: 4) - The Solr server version. Currently supports 3 and 4 (you can add a sub-version like 4.5 if
* you like, but currently it has no effect)
* service (default: depends on version, Solr3Service for 3, Solr4Service for 4)
* the class that provides actual communcation to the Solr server
* extraspath (default: <basefolder>/fulltextsearch/conf/solr/{version}/extras/) - Absolute path to
* the folder containing templates which are used for generating the schema and field definitions.
* templates (default: <basefolder>/fulltextsearch/conf/solr/{version}/templates/) - Absolute path to
* the configuration default files, e.g. solrconfig.xml.
*
* indexstore => an array with
*
* mode - a classname which implements SolrConfigStore, or 'file' or 'webdav'
*
* When mode == SolrConfigStore_File or file (indexes should be written on a local filesystem)
* path - The (locally accessible) path to write the index configurations to.
* remotepath (default: the same as indexpath) - The path that the Solr server will read the index configurations from
*
* When mode == SolrConfigStore_WebDAV or webdav (indexes should stored on a remote Solr server via webdav)
* auth (default: none) - A username:password pair string to use to auth against the webdav server
* path (default: /solrindex) - The suburl on the solr host that is set up to accept index configurations via webdav
* remotepath - The path that the Solr server will read the index configurations from
*/
protected static $solr_options = array();
/** A cache of solr_options with the defaults all merged in */ /**
protected static $merged_solr_options = null; * Update the configuration for Solr. See $solr_options for a discussion of the accepted array keys
* @param array $options - The options to update
*/
public static function configure_server($options = array())
{
self::$solr_options = array_merge(self::$solr_options, $options);
self::$merged_solr_options = null;
/** self::$service_singleton = null;
* Update the configuration for Solr. See $solr_options for a discussion of the accepted array keys self::$service_core_singletons = array();
* @param array $options - The options to update }
*/
static function configure_server($options = array()) {
self::$solr_options = array_merge(self::$solr_options, $options);
self::$merged_solr_options = null;
self::$service_singleton = null; /**
self::$service_core_singletons = array(); * Get the configured Solr options with the defaults all merged in
} * @return array - The merged options
*/
public static function solr_options()
{
if (self::$merged_solr_options) {
return self::$merged_solr_options;
}
/** $defaults = array(
* Get the configured Solr options with the defaults all merged in 'host' => 'localhost',
* @return array - The merged options 'port' => 8983,
*/ 'path' => '/solr',
static function solr_options() { 'version' => '4'
if (self::$merged_solr_options) return self::$merged_solr_options; );
$defaults = array( // Build some by-version defaults
'host' => 'localhost', $version = isset(self::$solr_options['version']) ? self::$solr_options['version'] : $defaults['version'];
'port' => 8983,
'path' => '/solr',
'version' => '4'
);
// Build some by-version defaults if (version_compare($version, '4', '>=')) {
$version = isset(self::$solr_options['version']) ? self::$solr_options['version'] : $defaults['version']; $versionDefaults = array(
'service' => 'Solr4Service',
'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/extras/',
'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/templates/',
);
} else {
$versionDefaults = array(
'service' => 'Solr3Service',
'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/extras/',
'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/templates/',
);
}
if (version_compare($version, '4', '>=')){ return (self::$merged_solr_options = array_merge($defaults, $versionDefaults, self::$solr_options));
$versionDefaults = array( }
'service' => 'Solr4Service',
'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/extras/',
'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/4/templates/',
);
}
else {
$versionDefaults = array(
'service' => 'Solr3Service',
'extraspath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/extras/',
'templatespath' => Director::baseFolder().'/fulltextsearch/conf/solr/3/templates/',
);
}
return (self::$merged_solr_options = array_merge($defaults, $versionDefaults, self::$solr_options));
}
static function set_service_class($class) { public static function set_service_class($class)
user_error('set_service_class is deprecated - pass as part of $options to configure_server', E_USER_WARNING); {
self::configure_server(array('service' => $class)); user_error('set_service_class is deprecated - pass as part of $options to configure_server', E_USER_WARNING);
} self::configure_server(array('service' => $class));
}
/** @var SolrService | null - The instance of SolrService for core management */ /** @var SolrService | null - The instance of SolrService for core management */
static protected $service_singleton = null; protected static $service_singleton = null;
/** @var [SolrService_Core] - The instances of SolrService_Core for each core */ /** @var [SolrService_Core] - The instances of SolrService_Core for each core */
static protected $service_core_singletons = array(); protected static $service_core_singletons = array();
/** /**
* Get a SolrService * Get a SolrService
* *
* @param string $core Optional core name * @param string $core Optional core name
* @return SolrService_Core * @return SolrService_Core
*/ */
static function service($core = null) { public static function service($core = null)
$options = self::solr_options(); {
$options = self::solr_options();
if (!self::$service_singleton) { if (!self::$service_singleton) {
self::$service_singleton = Object::create( self::$service_singleton = Object::create(
$options['service'], $options['host'], $options['port'], $options['path'] $options['service'], $options['host'], $options['port'], $options['path']
); );
} }
if ($core) { if ($core) {
if (!isset(self::$service_core_singletons[$core])) { if (!isset(self::$service_core_singletons[$core])) {
self::$service_core_singletons[$core] = self::$service_singleton->serviceForCore( self::$service_core_singletons[$core] = self::$service_singleton->serviceForCore(
singleton($core)->getIndexName() singleton($core)->getIndexName()
); );
} }
return self::$service_core_singletons[$core]; return self::$service_core_singletons[$core];
} else { } else {
return self::$service_singleton; return self::$service_singleton;
} }
} }
static function get_indexes() { public static function get_indexes()
return FullTextSearch::get_indexes('SolrIndex'); {
} return FullTextSearch::get_indexes('SolrIndex');
}
/** /**
* Include the thirdparty Solr client api library. Done this way to avoid issues where code is called in * Include the thirdparty Solr client api library. Done this way to avoid issues where code is called in
* mysite/_config before fulltextsearch/_config has a change to update the include path. * mysite/_config before fulltextsearch/_config has a change to update the include path.
*/ */
static function include_client_api() { public static function include_client_api()
static $included = false; {
static $included = false;
if (!$included) { if (!$included) {
set_include_path(get_include_path() . PATH_SEPARATOR . Director::baseFolder() . '/fulltextsearch/thirdparty/solr-php-client'); set_include_path(get_include_path() . PATH_SEPARATOR . Director::baseFolder() . '/fulltextsearch/thirdparty/solr-php-client');
require_once('Apache/Solr/Service.php'); require_once('Apache/Solr/Service.php');
require_once('Apache/Solr/Document.php'); require_once('Apache/Solr/Document.php');
$included = true; $included = true;
} }
} }
} }
/** /**
* Abstract class for build tasks * Abstract class for build tasks
*/ */
class Solr_BuildTask extends BuildTask { class Solr_BuildTask extends BuildTask
{
protected $enabled = false;
protected $enabled = false; /**
* Logger
*
* @var LoggerInterface
*/
protected $logger = null;
/** /**
* Logger * Get the current logger
* *
* @var LoggerInterface * @return LoggerInterface
*/ */
protected $logger = null; public function getLogger()
{
return $this->logger;
}
/** /**
* Get the current logger * Assign a new logger
* *
* @return LoggerInterface * @param LoggerInterface $logger
*/ */
public function getLogger() { public function setLogger(LoggerInterface $logger)
return $this->logger; {
} $this->logger = $logger;
}
/** /**
* Assign a new logger * @return SearchLogFactory
* */
* @param LoggerInterface $logger protected function getLoggerFactory()
*/ {
public function setLogger(LoggerInterface $logger) { return Injector::inst()->get('SearchLogFactory');
$this->logger = $logger; }
}
/** /**
* @return SearchLogFactory * Setup task
*/ *
protected function getLoggerFactory() { * @param SS_HTTPReqest $request
return Injector::inst()->get('SearchLogFactory'); */
} public function run($request)
{
$name = get_class($this);
$verbose = $request->getVar('verbose');
/** // Set new logger
* Setup task $logger = $this
* ->getLoggerFactory()
* @param SS_HTTPReqest $request ->getOutputLogger($name, $verbose);
*/ $this->setLogger($logger);
public function run($request) { }
$name = get_class($this);
$verbose = $request->getVar('verbose');
// Set new logger
$logger = $this
->getLoggerFactory()
->getOutputLogger($name, $verbose);
$this->setLogger($logger);
}
} }
class Solr_Configure extends Solr_BuildTask { class Solr_Configure extends Solr_BuildTask
{
protected $enabled = true;
protected $enabled = true; public function run($request)
{
parent::run($request);
// Find the IndexStore handler, which will handle uploading config files to Solr
$store = $this->getSolrConfigStore();
$indexes = Solr::get_indexes();
foreach ($indexes as $instance) {
try {
$this->updateIndex($instance, $store);
} catch (Exception $e) {
// We got an exception. Warn, but continue to next index.
$this
->getLogger()
->error("Failure: " . $e->getMessage());
}
}
}
/**
* Update the index on the given store
*
* @param SolrIndex $instance Instance
* @param SolrConfigStore $store
*/
protected function updateIndex($instance, $store)
{
$index = $instance->getIndexName();
$this->getLogger()->info("Configuring $index.");
// Upload the config files for this index
$this->getLogger()->info("Uploading configuration ...");
$instance->uploadConfig($store);
// Then tell Solr to use those config files
$service = Solr::service();
if ($service->coreIsActive($index)) {
$this->getLogger()->info("Reloading core ...");
$service->coreReload($index);
} else {
$this->getLogger()->info("Creating core ...");
$service->coreCreate($index, $store->instanceDir($index));
}
$this->getLogger()->info("Done");
}
public function run($request) { /**
parent::run($request); * Get config store
*
// Find the IndexStore handler, which will handle uploading config files to Solr * @return SolrConfigStore
$store = $this->getSolrConfigStore(); */
$indexes = Solr::get_indexes(); protected function getSolrConfigStore()
foreach ($indexes as $instance) { {
$options = Solr::solr_options();
if (!isset($options['indexstore']) || !($indexstore = $options['indexstore'])) {
user_error('No index configuration for Solr provided', E_USER_ERROR);
}
// Find the IndexStore handler, which will handle uploading config files to Solr
$mode = $indexstore['mode'];
try { if ($mode == 'file') {
$this->updateIndex($instance, $store); return new SolrConfigStore_File($indexstore);
} catch(Exception $e) { } elseif ($mode == 'webdav') {
// We got an exception. Warn, but continue to next index. return new SolrConfigStore_WebDAV($indexstore);
$this } elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, 'SolrConfigStore')) {
->getLogger() return new $mode($indexstore);
->error("Failure: " . $e->getMessage()); } else {
} user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR);
} }
} }
/**
* Update the index on the given store
*
* @param SolrIndex $instance Instance
* @param SolrConfigStore $store
*/
protected function updateIndex($instance, $store) {
$index = $instance->getIndexName();
$this->getLogger()->info("Configuring $index.");
// Upload the config files for this index
$this->getLogger()->info("Uploading configuration ...");
$instance->uploadConfig($store);
// Then tell Solr to use those config files
$service = Solr::service();
if ($service->coreIsActive($index)) {
$this->getLogger()->info("Reloading core ...");
$service->coreReload($index);
} else {
$this->getLogger()->info("Creating core ...");
$service->coreCreate($index, $store->instanceDir($index));
}
$this->getLogger()->info("Done");
}
/**
* Get config store
*
* @return SolrConfigStore
*/
protected function getSolrConfigStore() {
$options = Solr::solr_options();
if (!isset($options['indexstore']) || !($indexstore = $options['indexstore'])) {
user_error('No index configuration for Solr provided', E_USER_ERROR);
}
// Find the IndexStore handler, which will handle uploading config files to Solr
$mode = $indexstore['mode'];
if ($mode == 'file') {
return new SolrConfigStore_File($indexstore);
} elseif ($mode == 'webdav') {
return new SolrConfigStore_WebDAV($indexstore);
} elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, 'SolrConfigStore')) {
return new $mode($indexstore);
} else {
user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR);
}
}
} }
/** /**
@ -300,106 +312,110 @@ class Solr_Configure extends Solr_BuildTask {
* - variantstate * - variantstate
* - verbose (optional) * - verbose (optional)
*/ */
class Solr_Reindex extends Solr_BuildTask { class Solr_Reindex extends Solr_BuildTask
{
protected $enabled = true;
protected $enabled = true; /**
* Number of records to load and index per request
*
* @var int
* @config
*/
private static $recordsPerRequest = 200;
/** /**
* Number of records to load and index per request * Get the reindex handler
* *
* @var int * @return SolrReindexHandler
* @config */
*/ protected function getHandler()
private static $recordsPerRequest = 200; {
return Injector::inst()->get('SolrReindexHandler');
}
/** /**
* Get the reindex handler * @param SS_HTTPRequest $request
* */
* @return SolrReindexHandler public function run($request)
*/ {
protected function getHandler() { parent::run($request);
return Injector::inst()->get('SolrReindexHandler');
} // Reset state
$originalState = SearchVariant::current_state();
$this->doReindex($request);
SearchVariant::activate_state($originalState);
}
/** /**
* @param SS_HTTPRequest $request * @param SS_HTTPRequest $request
*/ */
public function run($request) { protected function doReindex($request)
parent::run($request); {
$class = $request->getVar('class');
// Reset state
$originalState = SearchVariant::current_state();
$this->doReindex($request);
SearchVariant::activate_state($originalState);
}
/** // Deprecated reindex mechanism
* @param SS_HTTPRequest $request $start = $request->getVar('start');
*/ if ($start !== null) {
protected function doReindex($request) { // Run single batch directly
$class = $request->getVar('class'); $indexInstance = singleton($request->getVar('index'));
$state = json_decode($request->getVar('variantstate'), true);
$this->runFrom($indexInstance, $class, $start, $state);
return;
}
// Deprecated reindex mechanism // Check if we are re-indexing a single group
$start = $request->getVar('start'); // If not using queuedjobs, we need to invoke Solr_Reindex as a separate process
if ($start !== null) { // Otherwise each group is processed via a SolrReindexGroupJob
// Run single batch directly $groups = $request->getVar('groups');
$indexInstance = singleton($request->getVar('index')); $handler = $this->getHandler();
$state = json_decode($request->getVar('variantstate'), true); if ($groups) {
$this->runFrom($indexInstance, $class, $start, $state); // Run grouped batches (id % groups = group)
return; $group = $request->getVar('group');
} $indexInstance = singleton($request->getVar('index'));
$state = json_decode($request->getVar('variantstate'), true);
// Check if we are re-indexing a single group $handler->runGroup($this->getLogger(), $indexInstance, $state, $class, $groups, $group);
// If not using queuedjobs, we need to invoke Solr_Reindex as a separate process return;
// Otherwise each group is processed via a SolrReindexGroupJob }
$groups = $request->getVar('groups');
$handler = $this->getHandler();
if($groups) {
// Run grouped batches (id % groups = group)
$group = $request->getVar('group');
$indexInstance = singleton($request->getVar('index'));
$state = json_decode($request->getVar('variantstate'), true);
$handler->runGroup($this->getLogger(), $indexInstance, $state, $class, $groups, $group); // If run at the top level, delegate to appropriate handler
return; $self = get_class($this);
} $handler->triggerReindex($this->getLogger(), $this->config()->recordsPerRequest, $self, $class);
}
// If run at the top level, delegate to appropriate handler /**
$self = get_class($this); * @deprecated since version 2.0.0
$handler->triggerReindex($this->getLogger(), $this->config()->recordsPerRequest, $self, $class); */
} protected function runFrom($index, $class, $start, $variantstate)
{
DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism');
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($variantstate);
/** // Generate filtered list
* @deprecated since version 2.0.0 $items = DataList::create($class)
*/ ->limit($this->config()->recordsPerRequest, $start);
protected function runFrom($index, $class, $start, $variantstate) {
DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism');
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($variantstate);
// Generate filtered list // Add child filter
$items = DataList::create($class) $classes = $index->getClasses();
->limit($this->config()->recordsPerRequest, $start); $options = $classes[$class];
if (!$options['include_children']) {
$items = $items->filter('ClassName', $class);
}
// Add child filter // Process selected records in this class
$classes = $index->getClasses(); $this->getLogger()->info("Adding $class");
$options = $classes[$class]; foreach ($items->sort("ID") as $item) {
if(!$options['include_children']) { $this->getLogger()->debug($item->ID);
$items = $items->filter('ClassName', $class);
}
// Process selected records in this class // See SearchUpdater_ObjectHandler::triggerReindex
$this->getLogger()->info("Adding $class"); $item->triggerReindex();
foreach ($items->sort("ID") as $item) { $item->destroy();
$this->getLogger()->debug($item->ID); }
// See SearchUpdater_ObjectHandler::triggerReindex $this->getLogger()->info("Done");
$item->triggerReindex(); }
$item->destroy();
}
$this->getLogger()->info("Done");
}
} }

8
code/solr/Solr3Service.php
View File

@ -1,8 +1,10 @@
<?php <?php
class Solr3Service_Core extends SolrService_Core { class Solr3Service_Core extends SolrService_Core
{
} }
class Solr3Service extends SolrService { class Solr3Service extends SolrService
private static $core_class = 'Solr3Service_Core'; {
private static $core_class = 'Solr3Service_Core';
} }

95
code/solr/Solr4Service.php
View File

@ -1,57 +1,58 @@
<?php <?php
class Solr4Service_Core extends SolrService_Core { class Solr4Service_Core extends SolrService_Core
{
/**
* Replace underlying commit function to remove waitFlush in 4.0+, since it's been deprecated and 4.4 throws errors
* if you pass it
*/
public function commit($expungeDeletes = false, $waitFlush = null, $waitSearcher = true, $timeout = 3600)
{
if ($waitFlush) {
user_error('waitFlush must be false when using Solr 4.0+' . E_USER_ERROR);
}
/** $expungeValue = $expungeDeletes ? 'true' : 'false';
* Replace underlying commit function to remove waitFlush in 4.0+, since it's been deprecated and 4.4 throws errors $searcherValue = $waitSearcher ? 'true' : 'false';
* if you pass it
*/
public function commit($expungeDeletes = false, $waitFlush = null, $waitSearcher = true, $timeout = 3600) {
if ($waitFlush) {
user_error('waitFlush must be false when using Solr 4.0+' . E_USER_ERROR);
}
$expungeValue = $expungeDeletes ? 'true' : 'false'; $rawPost = '<commit expungeDeletes="' . $expungeValue . '" waitSearcher="' . $searcherValue . '" />';
$searcherValue = $waitSearcher ? 'true' : 'false'; return $this->_sendRawPost($this->_updateUrl, $rawPost, $timeout);
}
/**
* @inheritdoc
* @see Solr4Service_Core::addDocuments
*/
public function addDocument(Apache_Solr_Document $document, $allowDups = false,
$overwritePending = true, $overwriteCommitted = true, $commitWithin = 0
) {
return $this->addDocuments(array($document), $allowDups, $overwritePending, $overwriteCommitted, $commitWithin);
}
$rawPost = '<commit expungeDeletes="' . $expungeValue . '" waitSearcher="' . $searcherValue . '" />'; /**
return $this->_sendRawPost($this->_updateUrl, $rawPost, $timeout); * Solr 4.0 compat http://wiki.apache.org/solr/UpdateXmlMessages#Optional_attributes_for_.22add.22
} * Remove allowDups, overwritePending and overwriteComitted
*/
/** public function addDocuments($documents, $allowDups = false, $overwritePending = true,
* @inheritdoc $overwriteCommitted = true, $commitWithin = 0
* @see Solr4Service_Core::addDocuments ) {
*/ $overwriteVal = $allowDups ? 'false' : 'true';
public function addDocument(Apache_Solr_Document $document, $allowDups = false, $commitWithin = (int) $commitWithin;
$overwritePending = true, $overwriteCommitted = true, $commitWithin = 0 $commitWithinString = $commitWithin > 0 ? " commitWithin=\"{$commitWithin}\"" : '';
) {
return $this->addDocuments(array($document), $allowDups, $overwritePending, $overwriteCommitted, $commitWithin);
}
/** $rawPost = "<add overwrite=\"{$overwriteVal}\"{$commitWithinString}>";
* Solr 4.0 compat http://wiki.apache.org/solr/UpdateXmlMessages#Optional_attributes_for_.22add.22 foreach ($documents as $document) {
* Remove allowDups, overwritePending and overwriteComitted if ($document instanceof Apache_Solr_Document) {
*/ $rawPost .= $this->_documentToXmlFragment($document);
public function addDocuments($documents, $allowDups = false, $overwritePending = true, }
$overwriteCommitted = true, $commitWithin = 0 }
) { $rawPost .= '</add>';
$overwriteVal = $allowDups ? 'false' : 'true';
$commitWithin = (int) $commitWithin;
$commitWithinString = $commitWithin > 0 ? " commitWithin=\"{$commitWithin}\"" : '';
$rawPost = "<add overwrite=\"{$overwriteVal}\"{$commitWithinString}>"; return $this->add($rawPost);
foreach ($documents as $document) { }
if ($document instanceof Apache_Solr_Document) {
$rawPost .= $this->_documentToXmlFragment($document);
}
}
$rawPost .= '</add>';
return $this->add($rawPost);
}
} }
class Solr4Service extends SolrService { class Solr4Service extends SolrService
private static $core_class = 'Solr4Service_Core'; {
private static $core_class = 'Solr4Service_Core';
} }

173
code/solr/SolrConfigStore.php
View File

@ -5,29 +5,30 @@
* *
* The interface Solr_Configure uses to upload configuration files to Solr * The interface Solr_Configure uses to upload configuration files to Solr
*/ */
interface SolrConfigStore { interface SolrConfigStore
/** {
* Upload a file to Solr for index $index /**
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index) * Upload a file to Solr for index $index
* @param $file string - A path to a file to upload. The base name of the file will be used on the remote side * @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
* @return null * @param $file string - A path to a file to upload. The base name of the file will be used on the remote side
*/ * @return null
function uploadFile($index, $file); */
public function uploadFile($index, $file);
/** /**
* Upload a file to Solr from a string for index $index * Upload a file to Solr from a string for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index) * @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
* @param $filename string - The base name of the file to use on the remote side * @param $filename string - The base name of the file to use on the remote side
* @param $strong string - The contents of the file * @param $strong string - The contents of the file
* @return null * @return null
*/ */
function uploadString($index, $filename, $string); public function uploadString($index, $filename, $string);
/** /**
* Get the instanceDir to tell Solr to use for index $index * Get the instanceDir to tell Solr to use for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index) * @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
*/ */
function instanceDir($index); public function instanceDir($index);
} }
/** /**
@ -36,41 +37,47 @@ interface SolrConfigStore {
* A ConfigStore that uploads files to a Solr instance on a locally accessible filesystem * A ConfigStore that uploads files to a Solr instance on a locally accessible filesystem
* by just using file copies * by just using file copies
*/ */
class SolrConfigStore_File implements SolrConfigStore { class SolrConfigStore_File implements SolrConfigStore
function __construct($config) { {
$this->local = $config['path']; public function __construct($config)
$this->remote = isset($config['remotepath']) ? $config['remotepath'] : $config['path']; {
} $this->local = $config['path'];
$this->remote = isset($config['remotepath']) ? $config['remotepath'] : $config['path'];
}
function getTargetDir($index) { public function getTargetDir($index)
$targetDir = "{$this->local}/{$index}/conf"; {
$targetDir = "{$this->local}/{$index}/conf";
if (!is_dir($targetDir)) { if (!is_dir($targetDir)) {
$worked = @mkdir($targetDir, 0770, true); $worked = @mkdir($targetDir, 0770, true);
if(!$worked) { if (!$worked) {
throw new RuntimeException( throw new RuntimeException(
sprintf('Failed creating target directory %s, please check permissions', $targetDir) sprintf('Failed creating target directory %s, please check permissions', $targetDir)
); );
} }
} }
return $targetDir; return $targetDir;
} }
function uploadFile($index, $file) { public function uploadFile($index, $file)
$targetDir = $this->getTargetDir($index); {
copy($file, $targetDir.'/'.basename($file)); $targetDir = $this->getTargetDir($index);
} copy($file, $targetDir.'/'.basename($file));
}
function uploadString($index, $filename, $string) { public function uploadString($index, $filename, $string)
$targetDir = $this->getTargetDir($index); {
file_put_contents("$targetDir/$filename", $string); $targetDir = $this->getTargetDir($index);
} file_put_contents("$targetDir/$filename", $string);
}
function instanceDir($index) { public function instanceDir($index)
return $this->remote.'/'.$index; {
} return $this->remote.'/'.$index;
}
} }
/** /**
@ -78,40 +85,50 @@ class SolrConfigStore_File implements SolrConfigStore {
* *
* A ConfigStore that uploads files to a Solr instance via a WebDAV server * A ConfigStore that uploads files to a Solr instance via a WebDAV server
*/ */
class SolrConfigStore_WebDAV implements SolrConfigStore { class SolrConfigStore_WebDAV implements SolrConfigStore
function __construct($config) { {
$options = Solr::solr_options(); public function __construct($config)
{
$options = Solr::solr_options();
$this->url = implode('', array( $this->url = implode('', array(
'http://', 'http://',
isset($config['auth']) ? $config['auth'].'@' : '', isset($config['auth']) ? $config['auth'].'@' : '',
$options['host'].':'.$options['port'], $options['host'].':'.$options['port'],
$config['path'] $config['path']
)); ));
$this->remote = $config['remotepath']; $this->remote = $config['remotepath'];
} }
function getTargetDir($index) { public function getTargetDir($index)
$indexdir = "{$this->url}/$index"; {
if (!WebDAV::exists($indexdir)) WebDAV::mkdir($indexdir); $indexdir = "{$this->url}/$index";
if (!WebDAV::exists($indexdir)) {
WebDAV::mkdir($indexdir);
}
$targetDir = "{$this->url}/$index/conf"; $targetDir = "{$this->url}/$index/conf";
if (!WebDAV::exists($targetDir)) WebDAV::mkdir($targetDir); if (!WebDAV::exists($targetDir)) {
WebDAV::mkdir($targetDir);
}
return $targetDir; return $targetDir;
} }
function uploadFile($index, $file) { public function uploadFile($index, $file)
$targetDir = $this->getTargetDir($index); {
WebDAV::upload_from_file($file, $targetDir.'/'.basename($file)); $targetDir = $this->getTargetDir($index);
} WebDAV::upload_from_file($file, $targetDir.'/'.basename($file));
}
function uploadString($index, $filename, $string) { public function uploadString($index, $filename, $string)
$targetDir = $this->getTargetDir($index); {
WebDAV::upload_from_string($string, "$targetDir/$filename"); $targetDir = $this->getTargetDir($index);
} WebDAV::upload_from_string($string, "$targetDir/$filename");
}
function instanceDir($index) { public function instanceDir($index)
return $this->remote ? "{$this->remote}/$index" : $index; {
} return $this->remote ? "{$this->remote}/$index" : $index;
}
} }

1806
code/solr/SolrIndex.php
View File

@ -2,868 +2,946 @@
Solr::include_client_api(); Solr::include_client_api();
abstract class SolrIndex extends SearchIndex { abstract class SolrIndex extends SearchIndex
{
static $fulltextTypeMap = array( public static $fulltextTypeMap = array(
'*' => 'text', '*' => 'text',
'HTMLVarchar' => 'htmltext', 'HTMLVarchar' => 'htmltext',
'HTMLText' => 'htmltext' 'HTMLText' => 'htmltext'
); );
static $filterTypeMap = array( public static $filterTypeMap = array(
'*' => 'string', '*' => 'string',
'Boolean' => 'boolean', 'Boolean' => 'boolean',
'Date' => 'tdate', 'Date' => 'tdate',
'SSDatetime' => 'tdate', 'SSDatetime' => 'tdate',
'SS_Datetime' => 'tdate', 'SS_Datetime' => 'tdate',
'ForeignKey' => 'tint', 'ForeignKey' => 'tint',
'Int' => 'tint', 'Int' => 'tint',
'Float' => 'tfloat', 'Float' => 'tfloat',
'Double' => 'tdouble' 'Double' => 'tdouble'
); );
static $sortTypeMap = array(); public static $sortTypeMap = array();
protected $analyzerFields = array(); protected $analyzerFields = array();
protected $copyFields = array(); protected $copyFields = array();
protected $extrasPath = null; protected $extrasPath = null;
protected $templatesPath = null; protected $templatesPath = null;
/** /**
* List of boosted fields * List of boosted fields
* *
* @var array * @var array
*/ */
protected $boostedFields = array(); protected $boostedFields = array();
/** /**
* Name of default field * Name of default field
* *
* @var string * @var string
* @config * @config
*/ */
private static $default_field = '_text'; private static $default_field = '_text';
/** /**
* List of copy fields all fulltext fields should be copied into. * List of copy fields all fulltext fields should be copied into.
* This will fallback to default_field if not specified * This will fallback to default_field if not specified
* *
* @var array * @var array
*/ */
private static $copy_fields = array(); private static $copy_fields = array();
/** /**
* @return String Absolute path to the folder containing * @return String Absolute path to the folder containing
* templates which are used for generating the schema and field definitions. * templates which are used for generating the schema and field definitions.
*/ */
function getTemplatesPath() { public function getTemplatesPath()
$globalOptions = Solr::solr_options(); {
return $this->templatesPath ? $this->templatesPath : $globalOptions['templatespath']; $globalOptions = Solr::solr_options();
} return $this->templatesPath ? $this->templatesPath : $globalOptions['templatespath'];
}
/**
* @return String Absolute path to the configuration default files, /**
* e.g. solrconfig.xml. * @return String Absolute path to the configuration default files,
*/ * e.g. solrconfig.xml.
function getExtrasPath() { */
$globalOptions = Solr::solr_options(); public function getExtrasPath()
return $this->extrasPath ? $this->extrasPath : $globalOptions['extraspath']; {
} $globalOptions = Solr::solr_options();
return $this->extrasPath ? $this->extrasPath : $globalOptions['extraspath'];
function generateSchema() { }
return $this->renderWith($this->getTemplatesPath() . '/schema.ss');
} public function generateSchema()
{
function getIndexName() { return $this->renderWith($this->getTemplatesPath() . '/schema.ss');
return get_class($this); }
}
public function getIndexName()
function getTypes() { {
return $this->renderWith($this->getTemplatesPath() . '/types.ss'); return get_class($this);
} }
/** public function getTypes()
* Index-time analyzer which is applied to a specific field. {
* Can be used to remove HTML tags, apply stemming, etc. return $this->renderWith($this->getTemplatesPath() . '/types.ss');
* }
* @see http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.WhitespaceTokenizerFactory
* /**
* @param String $field * Index-time analyzer which is applied to a specific field.
* @param String $type * Can be used to remove HTML tags, apply stemming, etc.
* @param Array $params Parameters for the analyzer, usually at least a "class" *
*/ * @see http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters#solr.WhitespaceTokenizerFactory
function addAnalyzer($field, $type, $params) { *
$fullFields = $this->fieldData($field); * @param String $field
if($fullFields) foreach($fullFields as $fullField => $spec) { * @param String $type
if(!isset($this->analyzerFields[$fullField])) $this->analyzerFields[$fullField] = array(); * @param Array $params Parameters for the analyzer, usually at least a "class"
$this->analyzerFields[$fullField][$type] = $params; */
} public function addAnalyzer($field, $type, $params)
} {
$fullFields = $this->fieldData($field);
/** if ($fullFields) {
* Get the default text field, normally '_text' foreach ($fullFields as $fullField => $spec) {
* if (!isset($this->analyzerFields[$fullField])) {
* @return string $this->analyzerFields[$fullField] = array();
*/ }
public function getDefaultField() { $this->analyzerFields[$fullField][$type] = $params;
return $this->config()->default_field; }
} }
}
/**
* Get list of fields each text field should be copied into. /**
* This will fallback to the default field if omitted. * Get the default text field, normally '_text'
* *
* @return array * @return string
*/ */
protected function getCopyDestinations() { public function getDefaultField()
$copyFields = $this->config()->copy_fields; {
if($copyFields) { return $this->config()->default_field;
return $copyFields; }
}
// Fallback to default field /**
$df = $this->getDefaultField(); * Get list of fields each text field should be copied into.
return array($df); * This will fallback to the default field if omitted.
} *
* @return array
public function getFieldDefinitions() { */
$xml = array(); protected function getCopyDestinations()
$stored = $this->getStoredDefault(); {
$copyFields = $this->config()->copy_fields;
$xml[] = ""; if ($copyFields) {
return $copyFields;
// Add the hardcoded field definitions }
// Fallback to default field
$xml[] = "<field name='_documentid' type='string' indexed='true' stored='true' required='true' />"; $df = $this->getDefaultField();
return array($df);
$xml[] = "<field name='ID' type='tint' indexed='true' stored='true' required='true' />"; }
$xml[] = "<field name='ClassName' type='string' indexed='true' stored='true' required='true' />";
$xml[] = "<field name='ClassHierarchy' type='string' indexed='true' stored='true' required='true' multiValued='true' />"; public function getFieldDefinitions()
{
// Add the fulltext collation field $xml = array();
$stored = $this->getStoredDefault();
$df = $this->getDefaultField();
$xml[] = "<field name='{$df}' type='htmltext' indexed='true' stored='{$stored}' multiValued='true' />" ; $xml[] = "";
// Add the user-specified fields // Add the hardcoded field definitions
foreach ($this->fulltextFields as $name => $field) { $xml[] = "<field name='_documentid' type='string' indexed='true' stored='true' required='true' />";
$xml[] = $this->getFieldDefinition($name, $field, self::$fulltextTypeMap);
} $xml[] = "<field name='ID' type='tint' indexed='true' stored='true' required='true' />";
$xml[] = "<field name='ClassName' type='string' indexed='true' stored='true' required='true' />";
foreach ($this->filterFields as $name => $field) { $xml[] = "<field name='ClassHierarchy' type='string' indexed='true' stored='true' required='true' multiValued='true' />";
if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') continue;
$xml[] = $this->getFieldDefinition($name, $field); // Add the fulltext collation field
}
$df = $this->getDefaultField();
foreach ($this->sortFields as $name => $field) { $xml[] = "<field name='{$df}' type='htmltext' indexed='true' stored='{$stored}' multiValued='true' />" ;
if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') continue;
$xml[] = $this->getFieldDefinition($name, $field); // Add the user-specified fields
}
foreach ($this->fulltextFields as $name => $field) {
return implode("\n\t\t", $xml); $xml[] = $this->getFieldDefinition($name, $field, self::$fulltextTypeMap);
} }
/** foreach ($this->filterFields as $name => $field) {
* Extract first suggestion text from collated values if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') {
* continue;
* @param mixed $collation }
* @return string $xml[] = $this->getFieldDefinition($name, $field);
*/ }
protected function getCollatedSuggestion($collation = '') {
if(is_string($collation)) { foreach ($this->sortFields as $name => $field) {
return $collation; if ($field['fullfield'] == 'ID' || $field['fullfield'] == 'ClassName') {
} continue;
if(is_object($collation)) { }
if(isset($collation->misspellingsAndCorrections)) { $xml[] = $this->getFieldDefinition($name, $field);
foreach($collation->misspellingsAndCorrections as $key => $value) { }
return $value;
} return implode("\n\t\t", $xml);
} }
}
return ''; /**
} * Extract first suggestion text from collated values
*
/** * @param mixed $collation
* Extract a human friendly spelling suggestion from a Solr spellcheck collation string. * @return string
* @param String $collation */
* @return String protected function getCollatedSuggestion($collation = '')
*/ {
protected function getNiceSuggestion($collation = '') { if (is_string($collation)) {
$collationParts = explode(' ', $collation); return $collation;
}
// Remove advanced query params from the beginning of each collation part. if (is_object($collation)) {
foreach ($collationParts as $key => &$part) { if (isset($collation->misspellingsAndCorrections)) {
$part = ltrim($part, '+'); foreach ($collation->misspellingsAndCorrections as $key => $value) {
} return $value;
}
return implode(' ', $collationParts); }
} }
return '';
/** }
* Extract a query string from a Solr spellcheck collation string.
* Useful for constructing 'Did you mean?' links, for example: /**
* <a href="http://example.com/search?q=$SuggestionQueryString">$SuggestionNice</a> * Extract a human friendly spelling suggestion from a Solr spellcheck collation string.
* @param String $collation * @param String $collation
* @return String * @return String
*/ */
protected function getSuggestionQueryString($collation = '') { protected function getNiceSuggestion($collation = '')
return str_replace(' ', '+', $this->getNiceSuggestion($collation)); {
} $collationParts = explode(' ', $collation);
/** // Remove advanced query params from the beginning of each collation part.
* Add a field that should be stored foreach ($collationParts as $key => &$part) {
* $part = ltrim($part, '+');
* @param string $field The field to add }
* @param string $forceType The type to force this field as (required in some cases, when not
* detectable from metadata) return implode(' ', $collationParts);
* @param array $extraOptions Dependent on search implementation }
*/
public function addStoredField($field, $forceType = null, $extraOptions = array()) { /**
$options = array_merge($extraOptions, array('stored' => 'true')); * Extract a query string from a Solr spellcheck collation string.
$this->addFulltextField($field, $forceType, $options); * Useful for constructing 'Did you mean?' links, for example:
} * <a href="http://example.com/search?q=$SuggestionQueryString">$SuggestionNice</a>
* @param String $collation
/** * @return String
* Add a fulltext field with a boosted value */
* protected function getSuggestionQueryString($collation = '')
* @param string $field The field to add {
* @param string $forceType The type to force this field as (required in some cases, when not return str_replace(' ', '+', $this->getNiceSuggestion($collation));
* detectable from metadata) }
* @param array $extraOptions Dependent on search implementation
* @param float $boost Numeric boosting value (defaults to 2) /**
*/ * Add a field that should be stored
public function addBoostedField($field, $forceType = null, $extraOptions = array(), $boost = 2) { *
$options = array_merge($extraOptions, array('boost' => $boost)); * @param string $field The field to add
$this->addFulltextField($field, $forceType, $options); * @param string $forceType The type to force this field as (required in some cases, when not
} * detectable from metadata)
* @param array $extraOptions Dependent on search implementation
*/
public function fieldData($field, $forceType = null, $extraOptions = array()) { public function addStoredField($field, $forceType = null, $extraOptions = array())
// Ensure that 'boost' is recorded here without being captured by solr {
$boost = null; $options = array_merge($extraOptions, array('stored' => 'true'));
if(array_key_exists('boost', $extraOptions)) { $this->addFulltextField($field, $forceType, $options);
$boost = $extraOptions['boost']; }
unset($extraOptions['boost']);
} /**
$data = parent::fieldData($field, $forceType, $extraOptions); * Add a fulltext field with a boosted value
*
// Boost all fields with this name * @param string $field The field to add
if(isset($boost)) { * @param string $forceType The type to force this field as (required in some cases, when not
foreach($data as $fieldName => $fieldInfo) { * detectable from metadata)
$this->boostedFields[$fieldName] = $boost; * @param array $extraOptions Dependent on search implementation
} * @param float $boost Numeric boosting value (defaults to 2)
} */
return $data; public function addBoostedField($field, $forceType = null, $extraOptions = array(), $boost = 2)
} {
$options = array_merge($extraOptions, array('boost' => $boost));
/** $this->addFulltextField($field, $forceType, $options);
* Set the default boosting level for a specific field. }
* Will control the default value for qf param (Query Fields), but will not
* override a query-specific value.
* public function fieldData($field, $forceType = null, $extraOptions = array())
* Fields must be added before having a field boosting specified {
* // Ensure that 'boost' is recorded here without being captured by solr
* @param string $field Full field key (Model_Field) $boost = null;
* @param float|null $level Numeric boosting value. Set to null to clear boost if (array_key_exists('boost', $extraOptions)) {
*/ $boost = $extraOptions['boost'];
public function setFieldBoosting($field, $level) { unset($extraOptions['boost']);
if(!isset($this->fulltextFields[$field])) { }
throw new InvalidArgumentException("No fulltext field $field exists on ".$this->getIndexName()); $data = parent::fieldData($field, $forceType, $extraOptions);
}
if($level === null) { // Boost all fields with this name
unset($this->boostedFields[$field]); if (isset($boost)) {
} else { foreach ($data as $fieldName => $fieldInfo) {
$this->boostedFields[$field] = $level; $this->boostedFields[$fieldName] = $boost;
} }
} }
return $data;
/** }
* Get all boosted fields
* /**
* @return array * Set the default boosting level for a specific field.
*/ * Will control the default value for qf param (Query Fields), but will not
public function getBoostedFields() { * override a query-specific value.
return $this->boostedFields; *
} * Fields must be added before having a field boosting specified
*
/** * @param string $field Full field key (Model_Field)
* Determine the best default value for the 'qf' parameter * @param float|null $level Numeric boosting value. Set to null to clear boost
* */
* @return array|null List of query fields, or null if not specified public function setFieldBoosting($field, $level)
*/ {
public function getQueryFields() { if (!isset($this->fulltextFields[$field])) {
// Not necessary to specify this unless boosting throw new InvalidArgumentException("No fulltext field $field exists on ".$this->getIndexName());
if(empty($this->boostedFields)) { }
return null; if ($level === null) {
} unset($this->boostedFields[$field]);
$queryFields = array(); } else {
foreach ($this->boostedFields as $fieldName => $boost) { $this->boostedFields[$field] = $level;
$queryFields[] = $fieldName . '^' . $boost; }
} }
// If any fields are queried, we must always include the default field, otherwise it will be excluded /**
$df = $this->getDefaultField(); * Get all boosted fields
if($queryFields && !isset($this->boostedFields[$df])) { *
$queryFields[] = $df; * @return array
} */
public function getBoostedFields()
return $queryFields; {
} return $this->boostedFields;
}
/**
* Gets the default 'stored' value for fields in this index /**
* * Determine the best default value for the 'qf' parameter
* @return string A default value for the 'stored' field option, either 'true' or 'false' *
*/ * @return array|null List of query fields, or null if not specified
protected function getStoredDefault() { */
return Director::isDev() ? 'true' : 'false'; public function getQueryFields()
} {
// Not necessary to specify this unless boosting
/** if (empty($this->boostedFields)) {
* @param String $name return null;
* @param Array $spec }
* @param Array $typeMap $queryFields = array();
* @return String XML foreach ($this->boostedFields as $fieldName => $boost) {
*/ $queryFields[] = $fieldName . '^' . $boost;
protected function getFieldDefinition($name, $spec, $typeMap = null) { }
if(!$typeMap) $typeMap = self::$filterTypeMap;
$multiValued = (isset($spec['multi_valued']) && $spec['multi_valued']) ? "true" : ''; // If any fields are queried, we must always include the default field, otherwise it will be excluded
$type = isset($typeMap[$spec['type']]) ? $typeMap[$spec['type']] : $typeMap['*']; $df = $this->getDefaultField();
if ($queryFields && !isset($this->boostedFields[$df])) {
$analyzerXml = ''; $queryFields[] = $df;
if(isset($this->analyzerFields[$name])) { }
foreach($this->analyzerFields[$name] as $analyzerType => $analyzerParams) {
$analyzerXml .= $this->toXmlTag($analyzerType, $analyzerParams); return $queryFields;
} }
}
/**
$fieldParams = array_merge( * Gets the default 'stored' value for fields in this index
array( *
'name' => $name, * @return string A default value for the 'stored' field option, either 'true' or 'false'
'type' => $type, */
'indexed' => 'true', protected function getStoredDefault()
'stored' => $this->getStoredDefault(), {
'multiValued' => $multiValued return Director::isDev() ? 'true' : 'false';
), }
isset($spec['extra_options']) ? $spec['extra_options'] : array()
); /**
* @param String $name
return $this->toXmlTag( * @param Array $spec
"field", * @param Array $typeMap
$fieldParams, * @return String XML
$analyzerXml ? "<analyzer>$analyzerXml</analyzer>" : null */
); protected function getFieldDefinition($name, $spec, $typeMap = null)
} {
if (!$typeMap) {
/** $typeMap = self::$filterTypeMap;
* Convert definition to XML tag }
* $multiValued = (isset($spec['multi_valued']) && $spec['multi_valued']) ? "true" : '';
* @param String $tag $type = isset($typeMap[$spec['type']]) ? $typeMap[$spec['type']] : $typeMap['*'];
* @param String $attrs Map of attributes
* @param String $content Inner content $analyzerXml = '';
* @return String XML tag if (isset($this->analyzerFields[$name])) {
*/ foreach ($this->analyzerFields[$name] as $analyzerType => $analyzerParams) {
protected function toXmlTag($tag, $attrs, $content = null) { $analyzerXml .= $this->toXmlTag($analyzerType, $analyzerParams);
$xml = "<$tag "; }
if($attrs) { }
$attrStrs = array();
foreach($attrs as $attrName => $attrVal) $attrStrs[] = "$attrName='$attrVal'"; $fieldParams = array_merge(
$xml .= $attrStrs ? implode(' ', $attrStrs) : ''; array(
} 'name' => $name,
$xml .= $content ? ">$content</$tag>" : '/>'; 'type' => $type,
return $xml; 'indexed' => 'true',
} 'stored' => $this->getStoredDefault(),
'multiValued' => $multiValued
/** ),
* @param String $source Composite field name (<class>_<fieldname>) isset($spec['extra_options']) ? $spec['extra_options'] : array()
* @param String $dest );
*/
function addCopyField($source, $dest, $extraOptions = array()) { return $this->toXmlTag(
if(!isset($this->copyFields[$source])) $this->copyFields[$source] = array(); "field",
$this->copyFields[$source][] = array_merge( $fieldParams,
array('source' => $source, 'dest' => $dest), $analyzerXml ? "<analyzer>$analyzerXml</analyzer>" : null
$extraOptions );
); }
}
/**
/** * Convert definition to XML tag
* Generate XML for copy field definitions *
* * @param String $tag
* @return string * @param String $attrs Map of attributes
*/ * @param String $content Inner content
public function getCopyFieldDefinitions() { * @return String XML tag
$xml = array(); */
protected function toXmlTag($tag, $attrs, $content = null)
// Default copy fields {
foreach($this->getCopyDestinations() as $copyTo) { $xml = "<$tag ";
foreach ($this->fulltextFields as $name => $field) { if ($attrs) {
$xml[] = "<copyField source='{$name}' dest='{$copyTo}' />"; $attrStrs = array();
} foreach ($attrs as $attrName => $attrVal) {
} $attrStrs[] = "$attrName='$attrVal'";
}
// Explicit copy fields $xml .= $attrStrs ? implode(' ', $attrStrs) : '';
foreach ($this->copyFields as $source => $fields) { }
foreach($fields as $fieldAttrs) { $xml .= $content ? ">$content</$tag>" : '/>';
$xml[] = $this->toXmlTag('copyField', $fieldAttrs); return $xml;
} }
}
/**
return implode("\n\t", $xml); * @param String $source Composite field name (<class>_<fieldname>)
} * @param String $dest
*/
protected function _addField($doc, $object, $field) { public function addCopyField($source, $dest, $extraOptions = array())
$class = get_class($object); {
if ($class != $field['origin'] && !is_subclass_of($class, $field['origin'])) return; if (!isset($this->copyFields[$source])) {
$this->copyFields[$source] = array();
$value = $this->_getFieldValue($object, $field); }
$this->copyFields[$source][] = array_merge(
$type = isset(self::$filterTypeMap[$field['type']]) ? self::$filterTypeMap[$field['type']] : self::$filterTypeMap['*']; array('source' => $source, 'dest' => $dest),
$extraOptions
if (is_array($value)) foreach($value as $sub) { );
/* Solr requires dates in the form 1995-12-31T23:59:59Z */ }
if ($type == 'tdate') {
if(!$sub) continue; /**
$sub = gmdate('Y-m-d\TH:i:s\Z', strtotime($sub)); * Generate XML for copy field definitions
} *
* @return string
/* Solr requires numbers to be valid if presented, not just empty */ */
if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($sub)) continue; public function getCopyFieldDefinitions()
{
$doc->addField($field['name'], $sub); $xml = array();
}
// Default copy fields
else { foreach ($this->getCopyDestinations() as $copyTo) {
/* Solr requires dates in the form 1995-12-31T23:59:59Z */ foreach ($this->fulltextFields as $name => $field) {
if ($type == 'tdate') { $xml[] = "<copyField source='{$name}' dest='{$copyTo}' />";
if(!$value) return; }
$value = gmdate('Y-m-d\TH:i:s\Z', strtotime($value)); }
}
// Explicit copy fields
/* Solr requires numbers to be valid if presented, not just empty */ foreach ($this->copyFields as $source => $fields) {
if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($value)) return; foreach ($fields as $fieldAttrs) {
$xml[] = $this->toXmlTag('copyField', $fieldAttrs);
$doc->setField($field['name'], $value); }
} }
}
return implode("\n\t", $xml);
protected function _addAs($object, $base, $options) { }
$includeSubs = $options['include_children'];
protected function _addField($doc, $object, $field)
$doc = new Apache_Solr_Document(); {
$class = get_class($object);
// Always present fields if ($class != $field['origin'] && !is_subclass_of($class, $field['origin'])) {
return;
$doc->setField('_documentid', $this->getDocumentID($object, $base, $includeSubs)); }
$doc->setField('ID', $object->ID);
$doc->setField('ClassName', $object->ClassName); $value = $this->_getFieldValue($object, $field);
foreach (SearchIntrospection::hierarchy(get_class($object), false) as $class) $doc->addField('ClassHierarchy', $class); $type = isset(self::$filterTypeMap[$field['type']]) ? self::$filterTypeMap[$field['type']] : self::$filterTypeMap['*'];
// Add the user-specified fields if (is_array($value)) {
foreach ($value as $sub) {
foreach ($this->getFieldsIterator() as $name => $field) { /* Solr requires dates in the form 1995-12-31T23:59:59Z */
if ($field['base'] == $base) $this->_addField($doc, $object, $field); if ($type == 'tdate') {
} if (!$sub) {
continue;
try { }
$this->getService()->addDocument($doc); $sub = gmdate('Y-m-d\TH:i:s\Z', strtotime($sub));
} catch (Exception $e) { }
SS_Log::log($e, SS_Log::WARN);
return false; /* Solr requires numbers to be valid if presented, not just empty */
} if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($sub)) {
continue;
return $doc; }
}
$doc->addField($field['name'], $sub);
function add($object) { }
$class = get_class($object); } else {
$docs = array(); /* Solr requires dates in the form 1995-12-31T23:59:59Z */
if ($type == 'tdate') {
foreach ($this->getClasses() as $searchclass => $options) { if (!$value) {
if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) { return;
$base = ClassInfo::baseDataClass($searchclass); }
$docs[] = $this->_addAs($object, $base, $options); $value = gmdate('Y-m-d\TH:i:s\Z', strtotime($value));
} }
}
/* Solr requires numbers to be valid if presented, not just empty */
return $docs; if (($type == 'tint' || $type == 'tfloat' || $type == 'tdouble') && !is_numeric($value)) {
} return;
}
function canAdd($class) {
foreach ($this->classes as $searchclass => $options) { $doc->setField($field['name'], $value);
if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) return true; }
} }
return false; protected function _addAs($object, $base, $options)
} {
$includeSubs = $options['include_children'];
function delete($base, $id, $state) {
$documentID = $this->getDocumentIDForState($base, $id, $state); $doc = new Apache_Solr_Document();
try { // Always present fields
$this->getService()->deleteById($documentID);
} catch (Exception $e) { $doc->setField('_documentid', $this->getDocumentID($object, $base, $includeSubs));
SS_Log::log($e, SS_Log::WARN); $doc->setField('ID', $object->ID);
return false; $doc->setField('ClassName', $object->ClassName);
}
} foreach (SearchIntrospection::hierarchy(get_class($object), false) as $class) {
$doc->addField('ClassHierarchy', $class);
/** }
* Clear all records which do not match the given classname whitelist.
* // Add the user-specified fields
* Can also be used to trim an index when reducing to a narrower set of classes.
* foreach ($this->getFieldsIterator() as $name => $field) {
* Ignores current state / variant. if ($field['base'] == $base) {
* $this->_addField($doc, $object, $field);
* @param array $classes List of non-obsolete classes in the same format as SolrIndex::getClasses() }
* @return bool Flag if successful }
*/
public function clearObsoleteClasses($classes) { try {
if(empty($classes)) { $this->getService()->addDocument($doc);
return false; } catch (Exception $e) {
} SS_Log::log($e, SS_Log::WARN);
return false;
// Delete all records which do not match the necessary classname rules }
$conditions = array();
foreach ($classes as $class => $options) { return $doc;
if ($options['include_children']) { }
$conditions[] = "ClassHierarchy:{$class}";
} else { public function add($object)
$conditions[] = "ClassName:{$class}"; {
} $class = get_class($object);
} $docs = array();
// Delete records which don't match any of these conditions in this index foreach ($this->getClasses() as $searchclass => $options) {
$deleteQuery = "-(" . implode(' ', $conditions) . ")"; if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) {
$this $base = ClassInfo::baseDataClass($searchclass);
->getService() $docs[] = $this->_addAs($object, $base, $options);
->deleteByQuery($deleteQuery); }
return true; }
}
return $docs;
function commit() { }
try {
$this->getService()->commit(false, false, false); public function canAdd($class)
} catch (Exception $e) { {
SS_Log::log($e, SS_Log::WARN); foreach ($this->classes as $searchclass => $options) {
return false; if ($searchclass == $class || ($options['include_children'] && is_subclass_of($class, $searchclass))) {
} return true;
} }
}
/**
* @param SearchQuery $query return false;
* @param integer $offset }
* @param integer $limit
* @param array $params Extra request parameters passed through to Solr public function delete($base, $id, $state)
* @return ArrayData Map with the following keys: {
* - 'Matches': ArrayList of the matched object instances $documentID = $this->getDocumentIDForState($base, $id, $state);
*/
public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = array()) { try {
$service = $this->getService(); $this->getService()->deleteById($documentID);
} catch (Exception $e) {
$searchClass = count($query->classes) == 1 SS_Log::log($e, SS_Log::WARN);
? $query->classes[0]['class'] return false;
: null; }
SearchVariant::with($searchClass) }
->call('alterQuery', $query, $this);
/**
$q = array(); // Query * Clear all records which do not match the given classname whitelist.
$fq = array(); // Filter query *
$qf = array(); // Query fields * Can also be used to trim an index when reducing to a narrower set of classes.
$hlq = array(); // Highlight query *
* Ignores current state / variant.
// Build the search itself *
$q = $this->getQueryComponent($query, $hlq); * @param array $classes List of non-obsolete classes in the same format as SolrIndex::getClasses()
* @return bool Flag if successful
// If using boosting, set the clean term separately for highlighting. */
// See https://issues.apache.org/jira/browse/SOLR-2632 public function clearObsoleteClasses($classes)
if(array_key_exists('hl', $params) && !array_key_exists('hl.q', $params)) { {
$params['hl.q'] = implode(' ', $hlq); if (empty($classes)) {
} return false;
}
// Filter by class if requested
$classq = array(); // Delete all records which do not match the necessary classname rules
foreach ($query->classes as $class) { $conditions = array();
if (!empty($class['includeSubclasses'])) { foreach ($classes as $class => $options) {
$classq[] = 'ClassHierarchy:'.$class['class']; if ($options['include_children']) {
} $conditions[] = "ClassHierarchy:{$class}";
else $classq[] = 'ClassName:'.$class['class']; } else {
} $conditions[] = "ClassName:{$class}";
if ($classq) $fq[] = '+('.implode(' ', $classq).')'; }
}
// Filter by filters
$fq = array_merge($fq, $this->getFiltersComponent($query)); // Delete records which don't match any of these conditions in this index
$deleteQuery = "-(" . implode(' ', $conditions) . ")";
// Prepare query fields unless specified explicitly $this
if(isset($params['qf'])) { ->getService()
$qf = $params['qf']; ->deleteByQuery($deleteQuery);
} else { return true;
$qf = $this->getQueryFields(); }
}
if(is_array($qf)) { public function commit()
$qf = implode(' ', $qf); {
} try {
if($qf) { $this->getService()->commit(false, false, false);
$params['qf'] = $qf; } catch (Exception $e) {
} SS_Log::log($e, SS_Log::WARN);
return false;
if(!headers_sent() && !Director::isLive()) { }
if ($q) header('X-Query: '.implode(' ', $q)); }
if ($fq) header('X-Filters: "'.implode('", "', $fq).'"');
if ($qf) header('X-QueryFields: '.$qf); /**
} * @param SearchQuery $query
* @param integer $offset
if ($offset == -1) $offset = $query->start; * @param integer $limit
if ($limit == -1) $limit = $query->limit; * @param array $params Extra request parameters passed through to Solr
if ($limit == -1) $limit = SearchQuery::$default_page_size; * @return ArrayData Map with the following keys:
* - 'Matches': ArrayList of the matched object instances
$params = array_merge($params, array('fq' => implode(' ', $fq))); */
public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = array())
$res = $service->search( {
$q ? implode(' ', $q) : '*:*', $service = $this->getService();
$offset,
$limit, $searchClass = count($query->classes) == 1
$params, ? $query->classes[0]['class']
Apache_Solr_Service::METHOD_POST : null;
); SearchVariant::with($searchClass)
->call('alterQuery', $query, $this);
$results = new ArrayList();
if($res->getHttpStatus() >= 200 && $res->getHttpStatus() < 300) { $q = array(); // Query
foreach ($res->response->docs as $doc) { $fq = array(); // Filter query
$result = DataObject::get_by_id($doc->ClassName, $doc->ID); $qf = array(); // Query fields
if($result) { $hlq = array(); // Highlight query
$results->push($result);
// Build the search itself
// Add highlighting (optional) $q = $this->getQueryComponent($query, $hlq);
$docId = $doc->_documentid;
if($res->highlighting && $res->highlighting->$docId) { // If using boosting, set the clean term separately for highlighting.
// TODO Create decorator class for search results rather than adding arbitrary object properties // See https://issues.apache.org/jira/browse/SOLR-2632
// TODO Allow specifying highlighted field, and lazy loading if (array_key_exists('hl', $params) && !array_key_exists('hl.q', $params)) {
// in case the search API needs another query (similar to SphinxSearchable->buildExcerpt()). $params['hl.q'] = implode(' ', $hlq);
$combinedHighlights = array(); }
foreach($res->highlighting->$docId as $field => $highlights) {
$combinedHighlights = array_merge($combinedHighlights, $highlights); // Filter by class if requested
} $classq = array();
foreach ($query->classes as $class) {
// Remove entity-encoded U+FFFD replacement character. It signifies non-displayable characters, if (!empty($class['includeSubclasses'])) {
// and shows up as an encoding error in browsers. $classq[] = 'ClassHierarchy:'.$class['class'];
$result->Excerpt = DBField::create_field( } else {
'HTMLText', $classq[] = 'ClassName:'.$class['class'];
str_replace( }
'&#65533;', }
'', if ($classq) {
implode(' ... ', $combinedHighlights) $fq[] = '+('.implode(' ', $classq).')';
) }
);
} // Filter by filters
} $fq = array_merge($fq, $this->getFiltersComponent($query));
}
$numFound = $res->response->numFound; // Prepare query fields unless specified explicitly
} else { if (isset($params['qf'])) {
$numFound = 0; $qf = $params['qf'];
} } else {
$qf = $this->getQueryFields();
$ret = array(); }
$ret['Matches'] = new PaginatedList($results); if (is_array($qf)) {
$ret['Matches']->setLimitItems(false); $qf = implode(' ', $qf);
// Tell PaginatedList how many results there are }
$ret['Matches']->setTotalItems($numFound); if ($qf) {
// Results for current page start at $offset $params['qf'] = $qf;
$ret['Matches']->setPageStart($offset); }
// Results per page
$ret['Matches']->setPageLength($limit); if (!headers_sent() && !Director::isLive()) {
if ($q) {
// Include spellcheck and suggestion data. Requires spellcheck=true in $params header('X-Query: '.implode(' ', $q));
if(isset($res->spellcheck)) { }
// Expose all spellcheck data, for custom handling. if ($fq) {
$ret['Spellcheck'] = $res->spellcheck; header('X-Filters: "'.implode('", "', $fq).'"');
}
// Suggestions. Requires spellcheck.collate=true in $params if ($qf) {
if(isset($res->spellcheck->suggestions->collation)) { header('X-QueryFields: '.$qf);
// Extract string suggestion }
$suggestion = $this->getCollatedSuggestion($res->spellcheck->suggestions->collation); }
// The collation, including advanced query params (e.g. +), suitable for making another query programmatically. if ($offset == -1) {
$ret['Suggestion'] = $suggestion; $offset = $query->start;
}
// A human friendly version of the suggestion, suitable for 'Did you mean $SuggestionNice?' display. if ($limit == -1) {
$ret['SuggestionNice'] = $this->getNiceSuggestion($suggestion); $limit = $query->limit;
}
// A string suitable for appending to an href as a query string. if ($limit == -1) {
// For example <a href="http://example.com/search?q=$SuggestionQueryString">$SuggestionNice</a> $limit = SearchQuery::$default_page_size;
$ret['SuggestionQueryString'] = $this->getSuggestionQueryString($suggestion); }
}
} $params = array_merge($params, array('fq' => implode(' ', $fq)));
return new ArrayData($ret); $res = $service->search(
} $q ? implode(' ', $q) : '*:*',
$offset,
$limit,
/** $params,
* Get the query (q) component for this search Apache_Solr_Service::METHOD_POST
* );
* @param SearchQuery $searchQuery
* @param array &$hlq Highlight query returned by reference $results = new ArrayList();
* @return array if ($res->getHttpStatus() >= 200 && $res->getHttpStatus() < 300) {
*/ foreach ($res->response->docs as $doc) {
protected function getQueryComponent(SearchQuery $searchQuery, &$hlq = array()) { $result = DataObject::get_by_id($doc->ClassName, $doc->ID);
$q = array(); if ($result) {
foreach ($searchQuery->search as $search) { $results->push($result);
$text = $search['text'];
preg_match_all('/"[^"]*"|\S+/', $text, $parts); // Add highlighting (optional)
$docId = $doc->_documentid;
$fuzzy = $search['fuzzy'] ? '~' : ''; if ($res->highlighting && $res->highlighting->$docId) {
// TODO Create decorator class for search results rather than adding arbitrary object properties
foreach ($parts[0] as $part) { // TODO Allow specifying highlighted field, and lazy loading
$fields = (isset($search['fields'])) ? $search['fields'] : array(); // in case the search API needs another query (similar to SphinxSearchable->buildExcerpt()).
if(isset($search['boost'])) { $combinedHighlights = array();
$fields = array_merge($fields, array_keys($search['boost'])); foreach ($res->highlighting->$docId as $field => $highlights) {
} $combinedHighlights = array_merge($combinedHighlights, $highlights);
if ($fields) { }
$searchq = array();
foreach ($fields as $field) { // Remove entity-encoded U+FFFD replacement character. It signifies non-displayable characters,
$boost = (isset($search['boost'][$field])) ? '^' . $search['boost'][$field] : ''; // and shows up as an encoding error in browsers.
$searchq[] = "{$field}:".$part.$fuzzy.$boost; $result->Excerpt = DBField::create_field(
} 'HTMLText',
$q[] = '+('.implode(' OR ', $searchq).')'; str_replace(
} '&#65533;',
else { '',
$q[] = '+'.$part.$fuzzy; implode(' ... ', $combinedHighlights)
} )
$hlq[] = $part; );
} }
} }
return $q; }
} $numFound = $res->response->numFound;
} else {
/** $numFound = 0;
* Parse all require constraints for inclusion in a filter query }
*
* @param SearchQuery $searchQuery $ret = array();
* @return array List of parsed string values for each require $ret['Matches'] = new PaginatedList($results);
*/ $ret['Matches']->setLimitItems(false);
protected function getRequireFiltersComponent(SearchQuery $searchQuery) { // Tell PaginatedList how many results there are
$fq = array(); $ret['Matches']->setTotalItems($numFound);
foreach ($searchQuery->require as $field => $values) { // Results for current page start at $offset
$requireq = array(); $ret['Matches']->setPageStart($offset);
// Results per page
foreach ($values as $value) { $ret['Matches']->setPageLength($limit);
if ($value === SearchQuery::$missing) {
$requireq[] = "(*:* -{$field}:[* TO *])"; // Include spellcheck and suggestion data. Requires spellcheck=true in $params
} if (isset($res->spellcheck)) {
else if ($value === SearchQuery::$present) { // Expose all spellcheck data, for custom handling.
$requireq[] = "{$field}:[* TO *]"; $ret['Spellcheck'] = $res->spellcheck;
}
else if ($value instanceof SearchQuery_Range) { // Suggestions. Requires spellcheck.collate=true in $params
$start = $value->start; if (isset($res->spellcheck->suggestions->collation)) {
if ($start === null) { // Extract string suggestion
$start = '*'; $suggestion = $this->getCollatedSuggestion($res->spellcheck->suggestions->collation);
}
$end = $value->end; // The collation, including advanced query params (e.g. +), suitable for making another query programmatically.
if ($end === null) { $ret['Suggestion'] = $suggestion;
$end = '*';
} // A human friendly version of the suggestion, suitable for 'Did you mean $SuggestionNice?' display.
$requireq[] = "$field:[$start TO $end]"; $ret['SuggestionNice'] = $this->getNiceSuggestion($suggestion);
}
else { // A string suitable for appending to an href as a query string.
$requireq[] = $field.':"'.$value.'"'; // For example <a href="http://example.com/search?q=$SuggestionQueryString">$SuggestionNice</a>
} $ret['SuggestionQueryString'] = $this->getSuggestionQueryString($suggestion);
} }
}
$fq[] = '+('.implode(' ', $requireq).')';
} return new ArrayData($ret);
return $fq; }
}
/** /**
* Parse all exclude constraints for inclusion in a filter query * Get the query (q) component for this search
* *
* @param SearchQuery $searchQuery * @param SearchQuery $searchQuery
* @return array List of parsed string values for each exclusion * @param array &$hlq Highlight query returned by reference
*/ * @return array
protected function getExcludeFiltersComponent(SearchQuery $searchQuery) { */
$fq = array(); protected function getQueryComponent(SearchQuery $searchQuery, &$hlq = array())
foreach ($searchQuery->exclude as $field => $values) { {
$excludeq = array(); $q = array();
$missing = false; foreach ($searchQuery->search as $search) {
$text = $search['text'];
foreach ($values as $value) { preg_match_all('/"[^"]*"|\S+/', $text, $parts);
if ($value === SearchQuery::$missing) {
$missing = true; $fuzzy = $search['fuzzy'] ? '~' : '';
}
else if ($value === SearchQuery::$present) { foreach ($parts[0] as $part) {
$excludeq[] = "{$field}:[* TO *]"; $fields = (isset($search['fields'])) ? $search['fields'] : array();
} if (isset($search['boost'])) {
else if ($value instanceof SearchQuery_Range) { $fields = array_merge($fields, array_keys($search['boost']));
$start = $value->start; }
if ($start === null) { if ($fields) {
$start = '*'; $searchq = array();
} foreach ($fields as $field) {
$end = $value->end; $boost = (isset($search['boost'][$field])) ? '^' . $search['boost'][$field] : '';
if ($end === null) { $searchq[] = "{$field}:".$part.$fuzzy.$boost;
$end = '*'; }
} $q[] = '+('.implode(' OR ', $searchq).')';
$excludeq[] = "$field:[$start TO $end]"; } else {
} $q[] = '+'.$part.$fuzzy;
else { }
$excludeq[] = $field.':"'.$value.'"'; $hlq[] = $part;
} }
} }
return $q;
$fq[] = ($missing ? "+{$field}:[* TO *] " : '') . '-('.implode(' ', $excludeq).')'; }
}
return $fq; /**
} * Parse all require constraints for inclusion in a filter query
*
/** * @param SearchQuery $searchQuery
* Get all filter conditions for this search * @return array List of parsed string values for each require
* */
* @param SearchQuery $searchQuery protected function getRequireFiltersComponent(SearchQuery $searchQuery)
* @return array {
*/ $fq = array();
public function getFiltersComponent(SearchQuery $searchQuery) { foreach ($searchQuery->require as $field => $values) {
return array_merge( $requireq = array();
$this->getRequireFiltersComponent($searchQuery),
$this->getExcludeFiltersComponent($searchQuery) foreach ($values as $value) {
); if ($value === SearchQuery::$missing) {
} $requireq[] = "(*:* -{$field}:[* TO *])";
} elseif ($value === SearchQuery::$present) {
protected $service; $requireq[] = "{$field}:[* TO *]";
} elseif ($value instanceof SearchQuery_Range) {
/** $start = $value->start;
* @return SolrService if ($start === null) {
*/ $start = '*';
public function getService() { }
if(!$this->service) $this->service = Solr::service(get_class($this)); $end = $value->end;
return $this->service; if ($end === null) {
} $end = '*';
}
public function setService(SolrService $service) { $requireq[] = "$field:[$start TO $end]";
$this->service = $service; } else {
return $this; $requireq[] = $field.':"'.$value.'"';
} }
}
/**
* Upload config for this index to the given store $fq[] = '+('.implode(' ', $requireq).')';
* }
* @param SolrConfigStore $store return $fq;
*/ }
public function uploadConfig($store) {
// Upload the config files for this index /**
$store->uploadString( * Parse all exclude constraints for inclusion in a filter query
$this->getIndexName(), *
'schema.xml', * @param SearchQuery $searchQuery
(string)$this->generateSchema() * @return array List of parsed string values for each exclusion
); */
protected function getExcludeFiltersComponent(SearchQuery $searchQuery)
// Upload additional files {
foreach (glob($this->getExtrasPath().'/*') as $file) { $fq = array();
if (is_file($file)) { foreach ($searchQuery->exclude as $field => $values) {
$store->uploadFile($this->getIndexName(), $file); $excludeq = array();
} $missing = false;
}
} foreach ($values as $value) {
if ($value === SearchQuery::$missing) {
$missing = true;
} elseif ($value === SearchQuery::$present) {
$excludeq[] = "{$field}:[* TO *]";
} elseif ($value instanceof SearchQuery_Range) {
$start = $value->start;
if ($start === null) {
$start = '*';
}
$end = $value->end;
if ($end === null) {
$end = '*';
}
$excludeq[] = "$field:[$start TO $end]";
} else {
$excludeq[] = $field.':"'.$value.'"';
}
}
$fq[] = ($missing ? "+{$field}:[* TO *] " : '') . '-('.implode(' ', $excludeq).')';
}
return $fq;
}
/**
* Get all filter conditions for this search
*
* @param SearchQuery $searchQuery
* @return array
*/
public function getFiltersComponent(SearchQuery $searchQuery)
{
return array_merge(
$this->getRequireFiltersComponent($searchQuery),
$this->getExcludeFiltersComponent($searchQuery)
);
}
protected $service;
/**
* @return SolrService
*/
public function getService()
{
if (!$this->service) {
$this->service = Solr::service(get_class($this));
}
return $this->service;
}
public function setService(SolrService $service)
{
$this->service = $service;
return $this;
}
/**
* Upload config for this index to the given store
*
* @param SolrConfigStore $store
*/
public function uploadConfig($store)
{
// Upload the config files for this index
$store->uploadString(
$this->getIndexName(),
'schema.xml',
(string)$this->generateSchema()
);
// Upload additional files
foreach (glob($this->getExtrasPath().'/*') as $file) {
if (is_file($file)) {
$store->uploadFile($this->getIndexName(), $file);
}
}
}
} }

121
code/solr/SolrService.php
View File

@ -5,7 +5,8 @@ Solr::include_client_api();
/** /**
* The API for accessing a specific core of a Solr server. Exactly the same as Apache_Solr_Service for now. * The API for accessing a specific core of a Solr server. Exactly the same as Apache_Solr_Service for now.
*/ */
class SolrService_Core extends Apache_Solr_Service { class SolrService_Core extends Apache_Solr_Service
{
} }
/** /**
@ -13,65 +14,77 @@ class SolrService_Core extends Apache_Solr_Service {
* plus extra methods for interrogating, creating, reloading and getting SolrService_Core instances * plus extra methods for interrogating, creating, reloading and getting SolrService_Core instances
* for Solr cores. * for Solr cores.
*/ */
class SolrService extends SolrService_Core { class SolrService extends SolrService_Core
private static $core_class = 'SolrService_Core'; {
private static $core_class = 'SolrService_Core';
/** /**
* Handle encoding the GET parameters and making the HTTP call to execute a core command * Handle encoding the GET parameters and making the HTTP call to execute a core command
*/ */
protected function coreCommand($command, $core, $params=array()) { protected function coreCommand($command, $core, $params=array())
$command = strtoupper($command); {
$command = strtoupper($command);
$params = array_merge($params, array('action' => $command, 'wt' => 'json')); $params = array_merge($params, array('action' => $command, 'wt' => 'json'));
$params[$command == 'CREATE' ? 'name' : 'core'] = $core; $params[$command == 'CREATE' ? 'name' : 'core'] = $core;
return $this->_sendRawGet($this->_constructUrl('admin/cores', $params)); return $this->_sendRawGet($this->_constructUrl('admin/cores', $params));
} }
/** /**
* Is the passed core active? * Is the passed core active?
* @param $core string - The name of the core * @param $core string - The name of the core
* @return boolean - True if that core exists & is active * @return boolean - True if that core exists & is active
*/ */
public function coreIsActive($core) { public function coreIsActive($core)
$result = $this->coreCommand('STATUS', $core); {
return isset($result->status->$core->uptime); $result = $this->coreCommand('STATUS', $core);
} return isset($result->status->$core->uptime);
}
/** /**
* Create a new core * Create a new core
* @param $core string - The name of the core * @param $core string - The name of the core
* @param $instancedir string - The base path of the core on the server * @param $instancedir string - The base path of the core on the server
* @param $config string - The filename of solrconfig.xml on the server. Default is $instancedir/solrconfig.xml * @param $config string - The filename of solrconfig.xml on the server. Default is $instancedir/solrconfig.xml
* @param $schema string - The filename of schema.xml on the server. Default is $instancedir/schema.xml * @param $schema string - The filename of schema.xml on the server. Default is $instancedir/schema.xml
* @param $datadir string - The path to store data for this core on the server. Default depends on solrconfig.xml * @param $datadir string - The path to store data for this core on the server. Default depends on solrconfig.xml
* @return Apache_Solr_Response * @return Apache_Solr_Response
*/ */
public function coreCreate($core, $instancedir, $config=null, $schema=null, $datadir=null) { public function coreCreate($core, $instancedir, $config=null, $schema=null, $datadir=null)
$args = array('instanceDir' => $instancedir); {
if ($config) $args['config'] = $config; $args = array('instanceDir' => $instancedir);
if ($schema) $args['schema'] = $schema; if ($config) {
if ($datadir) $args['dataDir'] = $datadir; $args['config'] = $config;
}
if ($schema) {
$args['schema'] = $schema;
}
if ($datadir) {
$args['dataDir'] = $datadir;
}
return $this->coreCommand('CREATE', $core, $args); return $this->coreCommand('CREATE', $core, $args);
} }
/** /**
* Reload a core * Reload a core
* @param $core string - The name of the core * @param $core string - The name of the core
* @return Apache_Solr_Response * @return Apache_Solr_Response
*/ */
public function coreReload($core) { public function coreReload($core)
return $this->coreCommand('RELOAD', $core); {
} return $this->coreCommand('RELOAD', $core);
}
/** /**
* Create a new Solr3Service_Core instance for the passed core * Create a new Solr3Service_Core instance for the passed core
* @param $core string - The name of the core * @param $core string - The name of the core
* @return Solr3Service_Core * @return Solr3Service_Core
*/ */
public function serviceForCore($core) { public function serviceForCore($core)
$klass = Config::inst()->get(get_called_class(), 'core_class'); {
return new $klass($this->_host, $this->_port, $this->_path.$core, $this->_httpTransport); $klass = Config::inst()->get(get_called_class(), 'core_class');
} return new $klass($this->_host, $this->_port, $this->_path.$core, $this->_httpTransport);
}
} }

404
code/solr/reindex/handlers/SolrReindexBase.php
View File

@ -5,226 +5,230 @@ use Psr\Log\LoggerInterface;
/** /**
* Base class for re-indexing of solr content * Base class for re-indexing of solr content
*/ */
abstract class SolrReindexBase implements SolrReindexHandler { abstract class SolrReindexBase implements SolrReindexHandler
{
public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null)
{
foreach (Solr::get_indexes() as $indexInstance) {
$this->processIndex($logger, $indexInstance, $batchSize, $taskName, $classes);
}
}
/**
* Process index for a single SolrIndex instance
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance
* @param int $batchSize
* @param string $taskName
* @param string $classes
*/
protected function processIndex(
LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null
) {
// Filter classes for this index
$indexClasses = $this->getClassesForIndex($indexInstance, $classes);
public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { // Clear all records in this index which do not contain the given classes
foreach (Solr::get_indexes() as $indexInstance) { $logger->info("Clearing obsolete classes from ".$indexInstance->getIndexName());
$this->processIndex($logger, $indexInstance, $batchSize, $taskName, $classes); $indexInstance->clearObsoleteClasses($indexClasses);
}
}
/**
* Process index for a single SolrIndex instance
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance
* @param int $batchSize
* @param string $taskName
* @param string $classes
*/
protected function processIndex(
LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null
) {
// Filter classes for this index
$indexClasses = $this->getClassesForIndex($indexInstance, $classes);
// Clear all records in this index which do not contain the given classes // Build queue for each class
$logger->info("Clearing obsolete classes from ".$indexInstance->getIndexName()); foreach ($indexClasses as $class => $options) {
$indexInstance->clearObsoleteClasses($indexClasses); $includeSubclasses = $options['include_children'];
// Build queue for each class foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) {
foreach ($indexClasses as $class => $options) { $this->processVariant($logger, $indexInstance, $state, $class, $includeSubclasses, $batchSize, $taskName);
$includeSubclasses = $options['include_children']; }
}
}
foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) { /**
$this->processVariant($logger, $indexInstance, $state, $class, $includeSubclasses, $batchSize, $taskName); * Get valid classes and options for an index with an optional filter
} *
} * @param SolrIndex $index
} * @param string|array $filterClasses Optional class or classes to limit to
* @return array List of classes, where the key is the classname and value is list of options
*/
protected function getClassesForIndex(SolrIndex $index, $filterClasses = null)
{
// Get base classes
$classes = $index->getClasses();
if (!$filterClasses) {
return $classes;
}
/** // Apply filter
* Get valid classes and options for an index with an optional filter if (!is_array($filterClasses)) {
* $filterClasses = explode(',', $filterClasses);
* @param SolrIndex $index }
* @param string|array $filterClasses Optional class or classes to limit to return array_intersect_key($classes, array_combine($filterClasses, $filterClasses));
* @return array List of classes, where the key is the classname and value is list of options }
*/
protected function getClassesForIndex(SolrIndex $index, $filterClasses = null) {
// Get base classes
$classes = $index->getClasses();
if(!$filterClasses) {
return $classes;
}
// Apply filter /**
if(!is_array($filterClasses)) { * Process re-index for a given variant state and class
$filterClasses = explode(',', $filterClasses); *
} * @param LoggerInterface $logger
return array_intersect_key($classes, array_combine($filterClasses, $filterClasses)); * @param SolrIndex $indexInstance
} * @param array $state Variant state
* @param string $class
* @param bool $includeSubclasses
* @param int $batchSize
* @param string $taskName
*/
protected function processVariant(
LoggerInterface $logger, SolrIndex $indexInstance, $state,
$class, $includeSubclasses, $batchSize, $taskName
) {
// Set state
SearchVariant::activate_state($state);
/** // Count records
* Process re-index for a given variant state and class $query = $class::get();
* if (!$includeSubclasses) {
* @param LoggerInterface $logger $query = $query->filter('ClassName', $class);
* @param SolrIndex $indexInstance }
* @param array $state Variant state $total = $query->count();
* @param string $class
* @param bool $includeSubclasses
* @param int $batchSize
* @param string $taskName
*/
protected function processVariant(
LoggerInterface $logger, SolrIndex $indexInstance, $state,
$class, $includeSubclasses, $batchSize, $taskName
) {
// Set state
SearchVariant::activate_state($state);
// Count records // Skip this variant if nothing to process, or if there are no records
$query = $class::get(); if ($total == 0 || $indexInstance->variantStateExcluded($state)) {
if(!$includeSubclasses) { // Remove all records in the current state, since there are no groups to process
$query = $query->filter('ClassName', $class); $logger->info("Clearing all records of type {$class} in the current state: " . json_encode($state));
} $this->clearRecords($indexInstance, $class);
$total = $query->count(); return;
}
// Skip this variant if nothing to process, or if there are no records // For each group, run processing
if ($total == 0 || $indexInstance->variantStateExcluded($state)) { $groups = (int)(($total + $batchSize - 1) / $batchSize);
// Remove all records in the current state, since there are no groups to process for ($group = 0; $group < $groups; $group++) {
$logger->info("Clearing all records of type {$class} in the current state: " . json_encode($state)); $this->processGroup($logger, $indexInstance, $state, $class, $groups, $group, $taskName);
$this->clearRecords($indexInstance, $class); }
return; }
}
// For each group, run processing /**
$groups = (int)(($total + $batchSize - 1) / $batchSize); * Initiate the processing of a single group
for ($group = 0; $group < $groups; $group++) { *
$this->processGroup($logger, $indexInstance, $state, $class, $groups, $group, $taskName); * @param LoggerInterface $logger
} * @param SolrIndex $indexInstance Index instance
} * @param array $state Variant state
* @param string $class Class to index
* @param int $groups Total groups
* @param int $group Index of group to process
* @param string $taskName Name of task script to run
*/
abstract protected function processGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName
);
/** /**
* Initiate the processing of a single group * Explicitly invoke the process that performs the group
* * processing. Can be run either by a background task or a queuedjob.
* @param LoggerInterface $logger *
* @param SolrIndex $indexInstance Index instance * Does not commit changes to the index, so this must be controlled externally.
* @param array $state Variant state *
* @param string $class Class to index * @param LoggerInterface $logger
* @param int $groups Total groups * @param SolrIndex $indexInstance
* @param int $group Index of group to process * @param array $state
* @param string $taskName Name of task script to run * @param string $class
*/ * @param int $groups
abstract protected function processGroup( * @param int $group
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName */
); public function runGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group
) {
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($state);
$logger->info("Adding $class");
/** // Prior to adding these records to solr, delete existing solr records
* Explicitly invoke the process that performs the group $this->clearRecords($indexInstance, $class, $groups, $group);
* processing. Can be run either by a background task or a queuedjob.
*
* Does not commit changes to the index, so this must be controlled externally.
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance
* @param array $state
* @param string $class
* @param int $groups
* @param int $group
*/
public function runGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group
) {
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($state);
$logger->info("Adding $class");
// Prior to adding these records to solr, delete existing solr records // Process selected records in this class
$this->clearRecords($indexInstance, $class, $groups, $group); $items = $this->getRecordsInGroup($indexInstance, $class, $groups, $group);
$processed = array();
foreach ($items as $item) {
$processed[] = $item->ID;
// Process selected records in this class // By this point, obsolete classes/states have been removed in processVariant
$items = $this->getRecordsInGroup($indexInstance, $class, $groups, $group); // and obsolete records have been removed in clearRecords
$processed = array(); $indexInstance->add($item);
foreach ($items as $item) { $item->destroy();
$processed[] = $item->ID; }
$logger->info("Updated ".implode(',', $processed));
// By this point, obsolete classes/states have been removed in processVariant // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex
// and obsolete records have been removed in clearRecords DB::query('SELECT 1');
$indexInstance->add($item);
$item->destroy(); $logger->info("Done");
} }
$logger->info("Updated ".implode(',', $processed));
// This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex /**
DB::query('SELECT 1'); * Gets the datalist of records in the given group in the current state
*
$logger->info("Done"); * Assumes that the desired variant state is in effect.
} *
* @param SolrIndex $indexInstance
* @param string $class
* @param int $groups
* @param int $group
* @return DataList
*/
protected function getRecordsInGroup(SolrIndex $indexInstance, $class, $groups, $group)
{
// Generate filtered list of local records
$baseClass = ClassInfo::baseDataClass($class);
$items = DataList::create($class)
->where(sprintf(
'"%s"."ID" %% \'%d\' = \'%d\'',
$baseClass,
intval($groups),
intval($group)
))
->sort("ID");
/** // Add child filter
* Gets the datalist of records in the given group in the current state $classes = $indexInstance->getClasses();
* $options = $classes[$class];
* Assumes that the desired variant state is in effect. if (!$options['include_children']) {
* $items = $items->filter('ClassName', $class);
* @param SolrIndex $indexInstance }
* @param string $class
* @param int $groups
* @param int $group
* @return DataList
*/
protected function getRecordsInGroup(SolrIndex $indexInstance, $class, $groups, $group) {
// Generate filtered list of local records
$baseClass = ClassInfo::baseDataClass($class);
$items = DataList::create($class)
->where(sprintf(
'"%s"."ID" %% \'%d\' = \'%d\'',
$baseClass,
intval($groups),
intval($group)
))
->sort("ID");
// Add child filter return $items;
$classes = $indexInstance->getClasses(); }
$options = $classes[$class];
if(!$options['include_children']) {
$items = $items->filter('ClassName', $class);
}
return $items; /**
} * Clear all records of the given class in the current state ONLY.
*
* Optionally delete from a given group (where the group is defined as the ID % total groups)
*
* @param SolrIndex $indexInstance Index instance
* @param string $class Class name
* @param int $groups Number of groups, if clearing from a striped group
* @param int $group Group number, if clearing from a striped group
*/
protected function clearRecords(SolrIndex $indexInstance, $class, $groups = null, $group = null)
{
// Clear by classname
$conditions = array("+(ClassHierarchy:{$class})");
/** // If grouping, delete from this group only
* Clear all records of the given class in the current state ONLY. if ($groups) {
* $conditions[] = "+_query_:\"{!frange l={$group} u={$group}}mod(ID, {$groups})\"";
* Optionally delete from a given group (where the group is defined as the ID % total groups) }
*
* @param SolrIndex $indexInstance Index instance
* @param string $class Class name
* @param int $groups Number of groups, if clearing from a striped group
* @param int $group Group number, if clearing from a striped group
*/
protected function clearRecords(SolrIndex $indexInstance, $class, $groups = null, $group = null) {
// Clear by classname
$conditions = array("+(ClassHierarchy:{$class})");
// If grouping, delete from this group only // Also filter by state (suffix on document ID)
if($groups) { $query = new SearchQuery();
$conditions[] = "+_query_:\"{!frange l={$group} u={$group}}mod(ID, {$groups})\""; SearchVariant::with($class)
} ->call('alterQuery', $query, $indexInstance);
if ($query->isfiltered()) {
$conditions = array_merge($conditions, $indexInstance->getFiltersComponent($query));
}
// Also filter by state (suffix on document ID) // Invoke delete on index
$query = new SearchQuery(); $deleteQuery = implode(' ', $conditions);
SearchVariant::with($class) $indexInstance
->call('alterQuery', $query, $indexInstance); ->getService()
if($query->isfiltered()) { ->deleteByQuery($deleteQuery);
$conditions = array_merge($conditions, $indexInstance->getFiltersComponent($query)); }
}
// Invoke delete on index
$deleteQuery = implode(' ', $conditions);
$indexInstance
->getService()
->deleteByQuery($deleteQuery);
}
} }

64
code/solr/reindex/handlers/SolrReindexHandler.php
View File

@ -5,38 +5,38 @@ use Psr\Log\LoggerInterface;
/** /**
* Provides interface for queueing a solr reindex * Provides interface for queueing a solr reindex
*/ */
interface SolrReindexHandler { interface SolrReindexHandler
{
/**
* Trigger a solr-reindex
*
* @param LoggerInterface $logger
* @param int $batchSize Records to run each process
* @param string $taskName Name of devtask to run
* @param string|array|null $classes Optional class or classes to limit index to
*/
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null);
/** /**
* Trigger a solr-reindex * Begin an immediate re-index
* *
* @param LoggerInterface $logger * @param LoggerInterface $logger
* @param int $batchSize Records to run each process * @param int $batchSize Records to run each process
* @param string $taskName Name of devtask to run * @param string $taskName Name of devtask to run
* @param string|array|null $classes Optional class or classes to limit index to * @param string|array|null $classes Optional class or classes to limit index to
*/ */
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null); public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null);
/** /**
* Begin an immediate re-index * Do an immediate re-index on the given group, where the group is defined as the list of items
* * where ID mod $groups = $group, in the given $state and optional $class filter.
* @param LoggerInterface $logger *
* @param int $batchSize Records to run each process * @param LoggerInterface $logger
* @param string $taskName Name of devtask to run * @param SolrIndex $indexInstance
* @param string|array|null $classes Optional class or classes to limit index to * @param array $state
*/ * @param string $class
public function runReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null); * @param int $groups
* @param int $group
/** */
* Do an immediate re-index on the given group, where the group is defined as the list of items public function runGroup(LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group);
* where ID mod $groups = $group, in the given $state and optional $class filter.
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance
* @param array $state
* @param string $class
* @param int $groups
* @param int $group
*/
public function runGroup(LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group);
} }

115
code/solr/reindex/handlers/SolrReindexImmediateHandler.php
View File

@ -7,68 +7,69 @@ use Psr\Log\LoggerInterface;
* *
* Internally batches of records will be invoked via shell tasks in the background * Internally batches of records will be invoked via shell tasks in the background
*/ */
class SolrReindexImmediateHandler extends SolrReindexBase { class SolrReindexImmediateHandler extends SolrReindexBase
{
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null)
{
$this->runReindex($logger, $batchSize, $taskName, $classes);
}
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { protected function processIndex(
$this->runReindex($logger, $batchSize, $taskName, $classes); LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null
} ) {
parent::processIndex($logger, $indexInstance, $batchSize, $taskName, $classes);
protected function processIndex( // Immediate processor needs to immediately commit after each index
LoggerInterface $logger, SolrIndex $indexInstance, $batchSize, $taskName, $classes = null $indexInstance->getService()->commit();
) { }
parent::processIndex($logger, $indexInstance, $batchSize, $taskName, $classes);
// Immediate processor needs to immediately commit after each index /**
$indexInstance->getService()->commit(); * Process a single group.
} *
* Without queuedjobs, it's necessary to shell this out to a background task as this is
* very memory intensive.
*
* The sub-process will then invoke $processor->runGroup() in {@see Solr_Reindex::doReindex}
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance Index instance
* @param array $state Variant state
* @param string $class Class to index
* @param int $groups Total groups
* @param int $group Index of group to process
* @param string $taskName Name of task script to run
*/
protected function processGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName
) {
// Build state
$statevar = json_encode($state);
if (strpos(PHP_OS, "WIN") !== false) {
$statevar = '"'.str_replace('"', '\\"', $statevar).'"';
} else {
$statevar = "'".$statevar."'";
}
/** // Build script
* Process a single group. $indexName = $indexInstance->getIndexName();
* $scriptPath = sprintf("%s%sframework%scli-script.php", BASE_PATH, DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR);
* Without queuedjobs, it's necessary to shell this out to a background task as this is $scriptTask = "php {$scriptPath} dev/tasks/{$taskName}";
* very memory intensive. $cmd = "{$scriptTask} index={$indexName} class={$class} group={$group} groups={$groups} variantstate={$statevar}";
* $cmd .= " verbose=1 2>&1";
* The sub-process will then invoke $processor->runGroup() in {@see Solr_Reindex::doReindex} $logger->info("Running '$cmd'");
*
* @param LoggerInterface $logger
* @param SolrIndex $indexInstance Index instance
* @param array $state Variant state
* @param string $class Class to index
* @param int $groups Total groups
* @param int $group Index of group to process
* @param string $taskName Name of task script to run
*/
protected function processGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName
) {
// Build state
$statevar = json_encode($state);
if (strpos(PHP_OS, "WIN") !== false) {
$statevar = '"'.str_replace('"', '\\"', $statevar).'"';
} else {
$statevar = "'".$statevar."'";
}
// Build script // Execute script via shell
$indexName = $indexInstance->getIndexName(); $res = $logger ? passthru($cmd) : `$cmd`;
$scriptPath = sprintf("%s%sframework%scli-script.php", BASE_PATH, DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR); if ($logger) {
$scriptTask = "php {$scriptPath} dev/tasks/{$taskName}"; $logger->info(preg_replace('/\r\n|\n/', '$0 ', $res));
$cmd = "{$scriptTask} index={$indexName} class={$class} group={$group} groups={$groups} variantstate={$statevar}"; }
$cmd .= " verbose=1 2>&1";
$logger->info("Running '$cmd'");
// Execute script via shell // If we're in dev mode, commit more often for fun and profit
$res = $logger ? passthru($cmd) : `$cmd`; if (Director::isDev()) {
if($logger) { Solr::service($indexName)->commit();
$logger->info(preg_replace('/\r\n|\n/', '$0 ', $res)); }
}
// If we're in dev mode, commit more often for fun and profit // This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex
if (Director::isDev()) { DB::query('SELECT 1');
Solr::service($indexName)->commit(); }
}
// This will slow down things a tiny bit, but it is done so that we don't timeout to the database during a reindex
DB::query('SELECT 1');
}
} }

74
code/solr/reindex/handlers/SolrReindexMessageHandler.php
View File

@ -2,39 +2,43 @@
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
if(!class_exists('MessageQueue')) return; if (!class_exists('MessageQueue')) {
return;
class SolrReindexMessageHandler extends SolrReindexImmediateHandler { }
/** class SolrReindexMessageHandler extends SolrReindexImmediateHandler
* The MessageQueue to use when processing updates {
* @config /**
* @var string * The MessageQueue to use when processing updates
*/ * @config
private static $reindex_queue = "search_indexing"; * @var string
*/
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { private static $reindex_queue = "search_indexing";
$queue = Config::inst()->get(__CLASS__, 'reindex_queue');
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null)
$logger->info('Queuing message'); {
MessageQueue::send( $queue = Config::inst()->get(__CLASS__, 'reindex_queue');
$queue,
new MethodInvocationMessage('SolrReindexMessageHandler', 'run_reindex', $batchSize, $taskName, $classes) $logger->info('Queuing message');
); MessageQueue::send(
} $queue,
new MethodInvocationMessage('SolrReindexMessageHandler', 'run_reindex', $batchSize, $taskName, $classes)
/** );
* Entry point for message queue }
*
* @param int $batchSize /**
* @param string $taskName * Entry point for message queue
* @param array|string|null $classes *
*/ * @param int $batchSize
public static function run_reindex($batchSize, $taskName, $classes = null) { * @param string $taskName
// @todo Logger for message queue? * @param array|string|null $classes
$logger = Injector::inst()->createWithArgs('Monolog\Logger', array(strtolower(get_class()))); */
public static function run_reindex($batchSize, $taskName, $classes = null)
$inst = Injector::inst()->get(get_class()); {
$inst->runReindex($logger, $batchSize, $taskName, $classes); // @todo Logger for message queue?
} $logger = Injector::inst()->createWithArgs('Monolog\Logger', array(strtolower(get_class())));
$inst = Injector::inst()->get(get_class());
$inst->runReindex($logger, $batchSize, $taskName, $classes);
}
} }

158
code/solr/reindex/handlers/SolrReindexQueuedHandler.php
View File

@ -2,93 +2,97 @@
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
}
/** /**
* Represents a queued task to start the reindex job * Represents a queued task to start the reindex job
*/ */
class SolrReindexQueuedHandler extends SolrReindexBase { class SolrReindexQueuedHandler extends SolrReindexBase
{
/**
* @return QueuedJobService
*/
protected function getQueuedJobService()
{
return singleton('QueuedJobService');
}
/** /**
* @return QueuedJobService * Cancel any cancellable jobs
*/ *
protected function getQueuedJobService() { * @param string $type Type of job to cancel
return singleton('QueuedJobService'); * @return int Number of jobs cleared
} */
protected function cancelExistingJobs($type)
{
$clearable = array(
// Paused jobs need to be discarded
QueuedJob::STATUS_PAUSED,
// These types would be automatically started
QueuedJob::STATUS_NEW,
QueuedJob::STATUS_WAIT,
/** // Cancel any in-progress job
* Cancel any cancellable jobs QueuedJob::STATUS_INIT,
* QueuedJob::STATUS_RUN
* @param string $type Type of job to cancel );
* @return int Number of jobs cleared DB::query(sprintf(
*/ 'UPDATE "QueuedJobDescriptor" '
protected function cancelExistingJobs($type) { . ' SET "JobStatus" = \'%s\''
$clearable = array( . ' WHERE "JobStatus" IN (\'%s\')'
// Paused jobs need to be discarded . ' AND "Implementation" = \'%s\'',
QueuedJob::STATUS_PAUSED, Convert::raw2sql(QueuedJob::STATUS_CANCELLED),
implode("','", Convert::raw2sql($clearable)),
// These types would be automatically started Convert::raw2sql($type)
QueuedJob::STATUS_NEW, ));
QueuedJob::STATUS_WAIT, return DB::affectedRows();
}
// Cancel any in-progress job public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null)
QueuedJob::STATUS_INIT, {
QueuedJob::STATUS_RUN // Cancel existing jobs
); $queues = $this->cancelExistingJobs('SolrReindexQueuedJob');
DB::query(sprintf( $groups = $this->cancelExistingJobs('SolrReindexGroupQueuedJob');
'UPDATE "QueuedJobDescriptor" ' $logger->info("Cancelled {$queues} re-index tasks and {$groups} re-index groups");
. ' SET "JobStatus" = \'%s\''
. ' WHERE "JobStatus" IN (\'%s\')'
. ' AND "Implementation" = \'%s\'',
Convert::raw2sql(QueuedJob::STATUS_CANCELLED),
implode("','", Convert::raw2sql($clearable)),
Convert::raw2sql($type)
));
return DB::affectedRows();
}
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) { // Although this class is used as a service (singleton) it may also be instantiated
// Cancel existing jobs // as a queuedjob
$queues = $this->cancelExistingJobs('SolrReindexQueuedJob'); $job = Injector::inst()->create('SolrReindexQueuedJob', $batchSize, $taskName, $classes);
$groups = $this->cancelExistingJobs('SolrReindexGroupQueuedJob'); $this
$logger->info("Cancelled {$queues} re-index tasks and {$groups} re-index groups"); ->getQueuedJobService()
->queueJob($job);
// Although this class is used as a service (singleton) it may also be instantiated $title = $job->getTitle();
// as a queuedjob $logger->info("Queued {$title}");
$job = Injector::inst()->create('SolrReindexQueuedJob', $batchSize, $taskName, $classes); }
$this
->getQueuedJobService()
->queueJob($job);
$title = $job->getTitle(); protected function processGroup(
$logger->info("Queued {$title}"); LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName
} ) {
// Trigger another job for this group
$job = Injector::inst()->create(
'SolrReindexGroupQueuedJob',
$indexInstance->getIndexName(), $state, $class, $groups, $group
);
$this
->getQueuedJobService()
->queueJob($job);
$title = $job->getTitle();
$logger->info("Queued {$title}");
}
protected function processGroup( public function runGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group
) { ) {
// Trigger another job for this group parent::runGroup($logger, $indexInstance, $state, $class, $groups, $group);
$job = Injector::inst()->create(
'SolrReindexGroupQueuedJob',
$indexInstance->getIndexName(), $state, $class, $groups, $group
);
$this
->getQueuedJobService()
->queueJob($job);
$title = $job->getTitle();
$logger->info("Queued {$title}");
}
public function runGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group
) {
parent::runGroup($logger, $indexInstance, $state, $class, $groups, $group);
// After any changes have been made, mark all indexes as dirty for commit
// see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers
$logger->info("Queuing commit on all changes");
SearchUpdateCommitJobProcessor::queue();
}
// After any changes have been made, mark all indexes as dirty for commit
// see http://stackoverflow.com/questions/7512945/how-to-fix-exceeded-limit-of-maxwarmingsearchers
$logger->info("Queuing commit on all changes");
SearchUpdateCommitJobProcessor::queue();
}
} }

187
code/solr/reindex/jobs/SolrReindexGroupQueuedJob.php
View File

@ -1,6 +1,8 @@
<?php <?php
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
}
/** /**
* Queuedjob to re-index a small group within an index. * Queuedjob to re-index a small group within an index.
@ -11,107 +13,112 @@ if(!interface_exists('QueuedJob')) return;
* list of IDs. Instead groups are segmented by ID. Additionally, this task does incremental * list of IDs. Instead groups are segmented by ID. Additionally, this task does incremental
* deletions of records. * deletions of records.
*/ */
class SolrReindexGroupQueuedJob extends SolrReindexQueuedJobBase { class SolrReindexGroupQueuedJob extends SolrReindexQueuedJobBase
{
/**
* Name of index to reindex
*
* @var string
*/
protected $indexName;
/** /**
* Name of index to reindex * Variant state that this group belongs to
* *
* @var string * @var type
*/ */
protected $indexName; protected $state;
/** /**
* Variant state that this group belongs to * Single class name to index
* *
* @var type * @var string
*/ */
protected $state; protected $class;
/** /**
* Single class name to index * Total number of groups
* *
* @var string * @var int
*/ */
protected $class; protected $groups;
/** /**
* Total number of groups * Group index
* *
* @var int * @var int
*/ */
protected $groups; protected $group;
/** public function __construct($indexName = null, $state = null, $class = null, $groups = null, $group = null)
* Group index {
* parent::__construct();
* @var int $this->indexName = $indexName;
*/ $this->state = $state;
protected $group; $this->class = $class;
$this->groups = $groups;
$this->group = $group;
}
public function __construct($indexName = null, $state = null, $class = null, $groups = null, $group = null) { public function getJobData()
parent::__construct(); {
$this->indexName = $indexName; $data = parent::getJobData();
$this->state = $state;
$this->class = $class;
$this->groups = $groups;
$this->group = $group;
}
public function getJobData() { // Custom data
$data = parent::getJobData(); $data->jobData->indexName = $this->indexName;
$data->jobData->state = $this->state;
$data->jobData->class = $this->class;
$data->jobData->groups = $this->groups;
$data->jobData->group = $this->group;
return $data;
}
// Custom data public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages)
$data->jobData->indexName = $this->indexName; {
$data->jobData->state = $this->state; parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages);
$data->jobData->class = $this->class;
$data->jobData->groups = $this->groups;
$data->jobData->group = $this->group;
return $data;
}
public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { // Custom data
parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages); $this->indexName = $jobData->indexName;
$this->state = $jobData->state;
$this->class = $jobData->class;
$this->groups = $jobData->groups;
$this->group = $jobData->group;
}
// Custom data public function getSignature()
$this->indexName = $jobData->indexName; {
$this->state = $jobData->state; return md5(get_class($this) . time() . mt_rand(0, 100000));
$this->class = $jobData->class; }
$this->groups = $jobData->groups;
$this->group = $jobData->group;
}
public function getSignature() { public function getTitle()
return md5(get_class($this) . time() . mt_rand(0, 100000)); {
} return sprintf(
'Solr Reindex Group (%d/%d) of %s in %s',
($this->group+1),
$this->groups,
$this->class,
json_encode($this->state)
);
}
public function getTitle() { public function process()
return sprintf( {
'Solr Reindex Group (%d/%d) of %s in %s', $logger = $this->getLogger();
($this->group+1), if ($this->jobFinished()) {
$this->groups, $logger->notice("reindex group already complete");
$this->class, return;
json_encode($this->state) }
);
}
public function process() { // Get instance of index
$logger = $this->getLogger(); $indexInstance = singleton($this->indexName);
if($this->jobFinished()) {
$logger->notice("reindex group already complete");
return;
}
// Get instance of index
$indexInstance = singleton($this->indexName);
// Send back to processor
$logger->info("Beginning reindex group");
$this
->getHandler()
->runGroup($logger, $indexInstance, $this->state, $this->class, $this->groups, $this->group);
$logger->info("Completed reindex group");
$this->isComplete = true;
}
// Send back to processor
$logger->info("Beginning reindex group");
$this
->getHandler()
->runGroup($logger, $indexInstance, $this->state, $this->class, $this->groups, $this->group);
$logger->info("Completed reindex group");
$this->isComplete = true;
}
} }

153
code/solr/reindex/jobs/SolrReindexQueuedJob.php
View File

@ -1,91 +1,100 @@
<?php <?php
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
}
/** /**
* Represents a queuedjob which invokes a reindex * Represents a queuedjob which invokes a reindex
*/ */
class SolrReindexQueuedJob extends SolrReindexQueuedJobBase { class SolrReindexQueuedJob extends SolrReindexQueuedJobBase
{
/**
* Size of each batch to run
*
* @var int
*/
protected $batchSize;
/** /**
* Size of each batch to run * Name of devtask Which invoked this
* * Not necessary for re-index processing performed entirely by queuedjobs
* @var int *
*/ * @var string
protected $batchSize; */
protected $taskName;
/** /**
* Name of devtask Which invoked this * List of classes to filter
* Not necessary for re-index processing performed entirely by queuedjobs *
* * @var array|string
* @var string */
*/ protected $classes;
protected $taskName;
/** public function __construct($batchSize = null, $taskName = null, $classes = null)
* List of classes to filter {
* $this->batchSize = $batchSize;
* @var array|string $this->taskName = $taskName;
*/ $this->classes = $classes;
protected $classes; parent::__construct();
}
public function __construct($batchSize = null, $taskName = null, $classes = null) { public function getJobData()
$this->batchSize = $batchSize; {
$this->taskName = $taskName; $data = parent::getJobData();
$this->classes = $classes;
parent::__construct();
}
public function getJobData() { // Custom data
$data = parent::getJobData(); $data->jobData->batchSize = $this->batchSize;
$data->jobData->taskName = $this->taskName;
$data->jobData->classes = $this->classes;
return $data;
}
// Custom data public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages)
$data->jobData->batchSize = $this->batchSize; {
$data->jobData->taskName = $this->taskName; parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages);
$data->jobData->classes = $this->classes;
return $data;
}
public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { // Custom data
parent::setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages); $this->batchSize = $jobData->batchSize;
$this->taskName = $jobData->taskName;
$this->classes = $jobData->classes;
}
// Custom data public function getSignature()
$this->batchSize = $jobData->batchSize; {
$this->taskName = $jobData->taskName; return __CLASS__;
$this->classes = $jobData->classes; }
}
public function getSignature() { public function getTitle()
return __CLASS__; {
} return 'Solr Reindex Job';
}
public function getTitle() { public function process()
return 'Solr Reindex Job'; {
} $logger = $this->getLogger();
if ($this->jobFinished()) {
$logger->notice("reindex already complete");
return;
}
public function process() { // Send back to processor
$logger = $this->getLogger(); $logger->info("Beginning init of reindex");
if($this->jobFinished()) { $this
$logger->notice("reindex already complete"); ->getHandler()
return; ->runReindex($logger, $this->batchSize, $this->taskName, $this->classes);
} $logger->info("Completed init of reindex");
$this->isComplete = true;
}
// Send back to processor /**
$logger->info("Beginning init of reindex"); * Get size of batch
$this *
->getHandler() * @return int
->runReindex($logger, $this->batchSize, $this->taskName, $this->classes); */
$logger->info("Completed init of reindex"); public function getBatchSize()
$this->isComplete = true; {
} return $this->batchSize;
}
/**
* Get size of batch
*
* @return int
*/
public function getBatchSize() {
return $this->batchSize;
}
} }

205
code/solr/reindex/jobs/SolrReindexQueuedJobBase.php
View File

@ -3,121 +3,136 @@
use Monolog\Logger; use Monolog\Logger;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
}
/** /**
* Base class for jobs which perform re-index * Base class for jobs which perform re-index
*/ */
abstract class SolrReindexQueuedJobBase implements QueuedJob { abstract class SolrReindexQueuedJobBase implements QueuedJob
{
/**
* Flag whether this job is done
*
* @var bool
*/
protected $isComplete;
/** /**
* Flag whether this job is done * List of messages
* *
* @var bool * @var array
*/ */
protected $isComplete; protected $messages;
/** /**
* List of messages * Logger to use for this job
* *
* @var array * @var LoggerInterface
*/ */
protected $messages; protected $logger;
/** public function __construct()
* Logger to use for this job {
* $this->isComplete = false;
* @var LoggerInterface $this->messages = array();
*/ }
protected $logger;
public function __construct() { /**
$this->isComplete = false; * @return SearchLogFactory
$this->messages = array(); */
} protected function getLoggerFactory()
{
return Injector::inst()->get('SearchLogFactory');
}
/** /**
* @return SearchLogFactory * Gets a logger for this job
*/ *
protected function getLoggerFactory() { * @return LoggerInterface
return Injector::inst()->get('SearchLogFactory'); */
} protected function getLogger()
{
if ($this->logger) {
return $this->logger;
}
/** // Set logger for this job
* Gets a logger for this job $this->logger = $this
* ->getLoggerFactory()
* @return LoggerInterface ->getQueuedJobLogger($this);
*/ return $this->logger;
protected function getLogger() { }
if($this->logger) {
return $this->logger;
}
// Set logger for this job /**
$this->logger = $this * Assign custom logger for this job
->getLoggerFactory() *
->getQueuedJobLogger($this); * @param LoggerInterface $logger
return $this->logger; */
} public function setLogger($logger)
{
$this->logger = $logger;
}
/** public function getJobData()
* Assign custom logger for this job {
* $data = new stdClass();
* @param LoggerInterface $logger
*/
public function setLogger($logger) {
$this->logger = $logger;
}
public function getJobData() { // Standard fields
$data = new stdClass(); $data->totalSteps = 1;
$data->currentStep = $this->isComplete ? 0 : 1;
$data->isComplete = $this->isComplete;
$data->messages = $this->messages;
// Standard fields // Custom data
$data->totalSteps = 1; $data->jobData = new stdClass();
$data->currentStep = $this->isComplete ? 0 : 1; return $data;
$data->isComplete = $this->isComplete; }
$data->messages = $this->messages;
// Custom data public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages)
$data->jobData = new stdClass(); {
return $data; $this->isComplete = $isComplete;
} $this->messages = $messages;
}
public function setJobData($totalSteps, $currentStep, $isComplete, $jobData, $messages) { /**
$this->isComplete = $isComplete; * Get the reindex handler
$this->messages = $messages; *
} * @return SolrReindexHandler
*/
protected function getHandler()
{
return Injector::inst()->get('SolrReindexHandler');
}
/** public function jobFinished()
* Get the reindex handler {
* return $this->isComplete;
* @return SolrReindexHandler }
*/
protected function getHandler() {
return Injector::inst()->get('SolrReindexHandler');
}
public function jobFinished() { public function prepareForRestart()
return $this->isComplete; {
} // NOOP
}
public function prepareForRestart() { public function setup()
// NOOP {
} // NOOP
}
public function setup() { public function afterComplete()
// NOOP {
} // NOOP
}
public function afterComplete() { public function getJobType()
// NOOP {
} return QueuedJob::QUEUED;
}
public function getJobType() { public function addMessage($message)
return QueuedJob::QUEUED; {
} $this->messages[] = $message;
}
public function addMessage($message) {
$this->messages[] = $message;
}
} }

120
code/utils/CombinationsArrayIterator.php
View File

@ -1,66 +1,80 @@
<?php <?php
class CombinationsArrayIterator implements Iterator { class CombinationsArrayIterator implements Iterator
protected $arrays; {
protected $keys; protected $arrays;
protected $numArrays; protected $keys;
protected $numArrays;
protected $isValid = false; protected $isValid = false;
protected $k = 0; protected $k = 0;
function __construct($args) { public function __construct($args)
$this->arrays = array(); {
$this->keys = array(); $this->arrays = array();
$this->keys = array();
$keys = array_keys($args); $keys = array_keys($args);
$values = array_values($args); $values = array_values($args);
foreach ($values as $i => $arg) { foreach ($values as $i => $arg) {
if (is_array($arg) && count($arg)) { if (is_array($arg) && count($arg)) {
$this->arrays[] = $arg; $this->arrays[] = $arg;
$this->keys[] = $keys[$i]; $this->keys[] = $keys[$i];
} }
} }
$this->numArrays = count($this->arrays); $this->numArrays = count($this->arrays);
$this->rewind(); $this->rewind();
} }
function rewind() { public function rewind()
if (!$this->numArrays) { {
$this->isValid = false; if (!$this->numArrays) {
} $this->isValid = false;
else { } else {
$this->isValid = true; $this->isValid = true;
$this->k = 0; $this->k = 0;
for ($i = 0; $i < $this->numArrays; $i++) reset($this->arrays[$i]); for ($i = 0; $i < $this->numArrays; $i++) {
} reset($this->arrays[$i]);
} }
}
}
function valid() { public function valid()
return $this->isValid; {
} return $this->isValid;
}
function next() { public function next()
$this->k++; {
$this->k++;
for ($i = 0; $i < $this->numArrays; $i++) { for ($i = 0; $i < $this->numArrays; $i++) {
if (next($this->arrays[$i]) === false) { if (next($this->arrays[$i]) === false) {
if ($i == $this->numArrays-1) $this->isValid = false; if ($i == $this->numArrays-1) {
else reset($this->arrays[$i]); $this->isValid = false;
} } else {
else break; reset($this->arrays[$i]);
} }
} } else {
break;
}
}
}
function current() { public function current()
$res = array(); {
for ($i = 0; $i < $this->numArrays; $i++) $res[$this->keys[$i]] = current($this->arrays[$i]); $res = array();
return $res; for ($i = 0; $i < $this->numArrays; $i++) {
} $res[$this->keys[$i]] = current($this->arrays[$i]);
}
return $res;
}
function key() { public function key()
return $this->k; {
} return $this->k;
} }
}

78
code/utils/MultipleArrayIterator.php
View File

@ -1,44 +1,58 @@
<?php <?php
class MultipleArrayIterator implements Iterator { class MultipleArrayIterator implements Iterator
{
protected $arrays;
protected $active;
protected $arrays; public function __construct()
protected $active; {
$args = func_get_args();
function __construct() { $this->arrays = array();
$args = func_get_args(); foreach ($args as $arg) {
if (is_array($arg) && count($arg)) {
$this->arrays[] = $arg;
}
}
$this->arrays = array(); $this->rewind();
foreach ($args as $arg) { }
if (is_array($arg) && count($arg)) $this->arrays[] = $arg;
}
$this->rewind(); public function rewind()
} {
$this->active = $this->arrays;
if ($this->active) {
reset($this->active[0]);
}
}
function rewind() { public function current()
$this->active = $this->arrays; {
if ($this->active) reset($this->active[0]); return $this->active ? current($this->active[0]) : false;
} }
function current() { public function key()
return $this->active ? current($this->active[0]) : false; {
} return $this->active ? key($this->active[0]) : false;
}
function key() { public function next()
return $this->active ? key($this->active[0]) : false; {
} if (!$this->active) {
return;
}
function next() { if (next($this->active[0]) === false) {
if (!$this->active) return; array_shift($this->active);
if ($this->active) {
reset($this->active[0]);
}
}
}
if (next($this->active[0]) === false) { public function valid()
array_shift($this->active); {
if ($this->active) reset($this->active[0]); return $this->active && (current($this->active[0]) !== false);
} }
}
function valid() {
return $this->active && (current($this->active[0]) !== false);
}
} }

106
code/utils/WebDAV.php
View File

@ -1,68 +1,76 @@
<?php <?php
class WebDAV { class WebDAV
{
public static function curl_init($url, $method)
{
$ch = curl_init($url);
static function curl_init($url, $method) { curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$ch = curl_init($url); curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); return $ch;
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY); }
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method);
return $ch; public static function exists($url)
} {
// WebDAV expects that checking a directory exists has a trailing slash
if (substr($url, -1) != '/') {
$url .= '/';
}
static function exists($url) { $ch = self::curl_init($url, 'PROPFIND');
// WebDAV expects that checking a directory exists has a trailing slash
if (substr($url, -1) != '/') { $res = curl_exec($ch);
$url .= '/'; $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
}
$ch = self::curl_init($url, 'PROPFIND'); if ($code == 404) {
return false;
$res = curl_exec($ch); }
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($code == 200 || $code == 207) {
return true;
}
if ($code == 404) return false; user_error("Got error from webdav server - ".$code, E_USER_ERROR);
if ($code == 200 || $code == 207) return true; }
user_error("Got error from webdav server - ".$code, E_USER_ERROR); public static function mkdir($url)
} {
$ch = self::curl_init(rtrim($url, '/').'/', 'MKCOL');
static function mkdir($url) { $res = curl_exec($ch);
$ch = self::curl_init(rtrim($url, '/').'/', 'MKCOL'); $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$res = curl_exec($ch); return $code == 201;
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE); }
return $code == 201; public static function put($handle, $url)
} {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
static function put($handle, $url) { curl_setopt($ch, CURLOPT_PUT, true);
$ch = curl_init($url); curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_setopt($ch, CURLOPT_PUT, true); curl_setopt($ch, CURLOPT_INFILE, $handle);
curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
curl_setopt($ch, CURLOPT_INFILE, $handle); $res = curl_exec($ch);
fclose($handle);
$res = curl_exec($ch); return curl_getinfo($ch, CURLINFO_HTTP_CODE);
fclose($handle); }
return curl_getinfo($ch, CURLINFO_HTTP_CODE);
}
static function upload_from_string($string, $url) {
$fh = tmpfile();
fwrite($fh, $string);
fseek($fh, 0);
return self::put($fh, $url);
}
static function upload_from_file($string, $url) {
return self::put(fopen($string, 'rb'), $url);
}
public static function upload_from_string($string, $url)
{
$fh = tmpfile();
fwrite($fh, $string);
fseek($fh, 0);
return self::put($fh, $url);
}
public static function upload_from_file($string, $url)
{
return self::put(fopen($string, 'rb'), $url);
}
} }

166
code/utils/logging/MonologFactory.php
View File

@ -8,91 +8,97 @@ use Monolog\Logger;
/** /**
* Provides logging based on monolog * Provides logging based on monolog
*/ */
class MonologFactory implements SearchLogFactory { class MonologFactory implements SearchLogFactory
{
public function getOutputLogger($name, $verbose)
{
$logger = $this->getLoggerFor($name);
$formatter = $this->getFormatter();
public function getOutputLogger($name, $verbose) { // Notice handling
$logger = $this->getLoggerFor($name); if ($verbose) {
$formatter = $this->getFormatter(); $messageHandler = $this->getStreamHandler($formatter, 'php://stdout', Logger::INFO);
$logger->pushHandler($messageHandler);
}
// Notice handling // Error handling. buble is false so that errors aren't logged twice
if($verbose) { $errorHandler = $this->getStreamHandler($formatter, 'php://stderr', Logger::ERROR, false);
$messageHandler = $this->getStreamHandler($formatter, 'php://stdout', Logger::INFO); $logger->pushHandler($errorHandler);
$logger->pushHandler($messageHandler); return $logger;
} }
// Error handling. buble is false so that errors aren't logged twice public function getQueuedJobLogger($job)
$errorHandler = $this->getStreamHandler($formatter, 'php://stderr', Logger::ERROR, false); {
$logger->pushHandler($errorHandler); $logger = $this->getLoggerFor(get_class($job));
return $logger; $handler = $this->getJobHandler($job);
} $logger->pushHandler($handler);
return $logger;
}
public function getQueuedJobLogger($job) { /**
$logger = $this->getLoggerFor(get_class($job)); * Generate a handler for the given stream
$handler = $this->getJobHandler($job); *
$logger->pushHandler($handler); * @param FormatterInterface $formatter
return $logger; * @param string $stream Name of preferred stream
} * @param int $level
* @param bool $bubble
* @return HandlerInterface
*/
protected function getStreamHandler(FormatterInterface $formatter, $stream, $level = Logger::DEBUG, $bubble = true)
{
// Unless cli, force output to php://output
$stream = Director::is_cli() ? $stream : 'php://output';
$handler = Injector::inst()->createWithArgs(
'Monolog\Handler\StreamHandler',
array($stream, $level, $bubble)
);
$handler->setFormatter($formatter);
return $handler;
}
/** /**
* Generate a handler for the given stream * Gets a formatter for standard output
* *
* @param FormatterInterface $formatter * @return FormatterInterface
* @param string $stream Name of preferred stream */
* @param int $level protected function getFormatter()
* @param bool $bubble {
* @return HandlerInterface // Get formatter
*/ $format = LineFormatter::SIMPLE_FORMAT;
protected function getStreamHandler(FormatterInterface $formatter, $stream, $level = Logger::DEBUG, $bubble = true) { if (!Director::is_cli()) {
// Unless cli, force output to php://output $format = "<p>$format</p>";
$stream = Director::is_cli() ? $stream : 'php://output'; }
$handler = Injector::inst()->createWithArgs( return Injector::inst()->createWithArgs(
'Monolog\Handler\StreamHandler', 'Monolog\Formatter\LineFormatter',
array($stream, $level, $bubble) array($format)
); );
$handler->setFormatter($formatter); }
return $handler;
}
/** /**
* Gets a formatter for standard output * Get a logger for a named class
* *
* @return FormatterInterface * @param string $name
*/ * @return Logger
protected function getFormatter() { */
// Get formatter protected function getLoggerFor($name)
$format = LineFormatter::SIMPLE_FORMAT; {
if(!Director::is_cli()) { return Injector::inst()->createWithArgs(
$format = "<p>$format</p>"; 'Monolog\Logger',
} array(strtolower($name))
return Injector::inst()->createWithArgs( );
'Monolog\Formatter\LineFormatter', }
array($format)
);
}
/** /**
* Get a logger for a named class * Generate handler for a job object
* *
* @param string $name * @param QueuedJob $job
* @return Logger * @return HandlerInterface
*/ */
protected function getLoggerFor($name) { protected function getJobHandler($job)
return Injector::inst()->createWithArgs( {
'Monolog\Logger', return Injector::inst()->createWithArgs(
array(strtolower($name)) 'QueuedJobLogHandler',
); array($job, Logger::INFO)
} );
}
/**
* Generate handler for a job object
*
* @param QueuedJob $job
* @return HandlerInterface
*/
protected function getJobHandler($job) {
return Injector::inst()->createWithArgs(
'QueuedJobLogHandler',
array($job, Logger::INFO)
);
}
} }

83
code/utils/logging/QueuedJobLogHandler.php
View File

@ -3,51 +3,56 @@
use Monolog\Handler\AbstractProcessingHandler; use Monolog\Handler\AbstractProcessingHandler;
use Monolog\Logger; use Monolog\Logger;
if(!interface_exists('QueuedJob')) return; if (!interface_exists('QueuedJob')) {
return;
}
/** /**
* Handler for logging events into QueuedJob message data * Handler for logging events into QueuedJob message data
*/ */
class QueuedJobLogHandler extends AbstractProcessingHandler { class QueuedJobLogHandler extends AbstractProcessingHandler
{
/**
* Job to log to
*
* @var QueuedJob
*/
protected $queuedJob;
/** /**
* Job to log to * @param QueuedJob $queuedJob Job to log to
* * @param integer $level The minimum logging level at which this handler will be triggered
* @var QueuedJob * @param Boolean $bubble Whether the messages that are handled can bubble up the stack or not
*/ */
protected $queuedJob; public function __construct(QueuedJob $queuedJob, $level = Logger::DEBUG, $bubble = true)
{
parent::__construct($level, $bubble);
$this->setQueuedJob($queuedJob);
}
/** /**
* @param QueuedJob $queuedJob Job to log to * Set a new queuedjob
* @param integer $level The minimum logging level at which this handler will be triggered *
* @param Boolean $bubble Whether the messages that are handled can bubble up the stack or not * @param QueuedJob $queuedJob
*/ */
public function __construct(QueuedJob $queuedJob, $level = Logger::DEBUG, $bubble = true) { public function setQueuedJob(QueuedJob $queuedJob)
parent::__construct($level, $bubble); {
$this->setQueuedJob($queuedJob); $this->queuedJob = $queuedJob;
} }
/** /**
* Set a new queuedjob * Get queuedjob
* *
* @param QueuedJob $queuedJob * @return QueuedJob
*/ */
public function setQueuedJob(QueuedJob $queuedJob) { public function getQueuedJob()
$this->queuedJob = $queuedJob; {
} return $this->queuedJob;
}
/**
* Get queuedjob
*
* @return QueuedJob
*/
public function getQueuedJob() {
return $this->queuedJob;
}
protected function write(array $record) {
// Write formatted message
$this->getQueuedJob()->addMessage($record['formatted']);
}
protected function write(array $record)
{
// Write formatted message
$this->getQueuedJob()->addMessage($record['formatted']);
}
} }

34
code/utils/logging/SearchLogFactory.php
View File

@ -2,22 +2,22 @@
use Psr\Log; use Psr\Log;
interface SearchLogFactory { interface SearchLogFactory
{
/**
* Make a logger for a queuedjob
*
* @param QueuedJob $job
* @return Log
*/
public function getQueuedJobLogger($job);
/** /**
* Make a logger for a queuedjob * Get an output logger with the given verbosity
* *
* @param QueuedJob $job * @param string $name
* @return Log * @param bool $verbose
*/ * @return Log
public function getQueuedJobLogger($job); */
public function getOutputLogger($name, $verbose);
/**
* Get an output logger with the given verbosity
*
* @param string $name
* @param bool $verbose
* @return Log
*/
public function getOutputLogger($name, $verbose);
} }

439
tests/BatchedProcessorTest.php
View File

@ -1,246 +1,259 @@
<?php <?php
class BatchedProcessorTest_Object extends SiteTree implements TestOnly { class BatchedProcessorTest_Object extends SiteTree implements TestOnly
private static $db = array( {
'TestText' => 'Varchar' private static $db = array(
); 'TestText' => 'Varchar'
);
} }
class BatchedProcessorTest_Index extends SearchIndex_Recording implements TestOnly { class BatchedProcessorTest_Index extends SearchIndex_Recording implements TestOnly
function init() { {
$this->addClass('BatchedProcessorTest_Object'); public function init()
$this->addFilterField('TestText'); {
} $this->addClass('BatchedProcessorTest_Object');
$this->addFilterField('TestText');
}
} }
class BatchedProcessor_QueuedJobService { class BatchedProcessor_QueuedJobService
protected $jobs = array(); {
protected $jobs = array();
public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null) { public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null)
$this->jobs[] = array( {
'job' => $job, $this->jobs[] = array(
'startAfter' => $startAfter 'job' => $job,
); 'startAfter' => $startAfter
return $job; );
} return $job;
}
public function getJobs() { public function getJobs()
return $this->jobs; {
} return $this->jobs;
}
} }
/** /**
* Tests {@see SearchUpdateQueuedJobProcessor} * Tests {@see SearchUpdateQueuedJobProcessor}
*/ */
class BatchedProcessorTest extends SapphireTest { class BatchedProcessorTest extends SapphireTest
{
protected $oldProcessor; protected $oldProcessor;
protected $extraDataObjects = array( protected $extraDataObjects = array(
'BatchedProcessorTest_Object' 'BatchedProcessorTest_Object'
); );
protected $illegalExtensions = array( protected $illegalExtensions = array(
'SiteTree' => array( 'SiteTree' => array(
'SiteTreeSubsites', 'SiteTreeSubsites',
'Translatable' 'Translatable'
) )
); );
public function setUpOnce() { public function setUpOnce()
// Disable illegal extensions if skipping this test {
if(class_exists('Subsite') || !interface_exists('QueuedJob')) { // Disable illegal extensions if skipping this test
$this->illegalExtensions = array(); if (class_exists('Subsite') || !interface_exists('QueuedJob')) {
} $this->illegalExtensions = array();
parent::setUpOnce(); }
} parent::setUpOnce();
}
public function setUp() { public function setUp()
parent::setUp(); {
Config::nest(); parent::setUp();
Config::nest();
if (!interface_exists('QueuedJob')) {
$this->skipTest = true; if (!interface_exists('QueuedJob')) {
$this->markTestSkipped("These tests need the QueuedJobs module installed to run"); $this->skipTest = true;
} $this->markTestSkipped("These tests need the QueuedJobs module installed to run");
}
if(class_exists('Subsite')) { if (class_exists('Subsite')) {
$this->skipTest = true; $this->skipTest = true;
$this->markTestSkipped(get_class() . ' skipped when running with subsites'); $this->markTestSkipped(get_class() . ' skipped when running with subsites');
} }
SS_Datetime::set_mock_now('2015-05-07 06:00:00'); SS_Datetime::set_mock_now('2015-05-07 06:00:00');
Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_size', 5); Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_size', 5);
Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 0); Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 0);
Config::inst()->update('SearchUpdateCommitJobProcessor', 'cooldown', 600); Config::inst()->update('SearchUpdateCommitJobProcessor', 'cooldown', 600);
Versioned::reading_stage("Stage"); Versioned::reading_stage("Stage");
Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), 'QueuedJobService'); Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), 'QueuedJobService');
FullTextSearch::force_index_list('BatchedProcessorTest_Index'); FullTextSearch::force_index_list('BatchedProcessorTest_Index');
SearchUpdateCommitJobProcessor::$dirty_indexes = array(); SearchUpdateCommitJobProcessor::$dirty_indexes = array();
SearchUpdateCommitJobProcessor::$has_run = false; SearchUpdateCommitJobProcessor::$has_run = false;
$this->oldProcessor = SearchUpdater::$processor; $this->oldProcessor = SearchUpdater::$processor;
SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor(); SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor();
} }
public function tearDown() { public function tearDown()
if($this->oldProcessor) { {
SearchUpdater::$processor = $this->oldProcessor; if ($this->oldProcessor) {
} SearchUpdater::$processor = $this->oldProcessor;
Config::unnest(); }
Injector::inst()->unregisterNamedObject('QueuedJobService'); Config::unnest();
FullTextSearch::force_index_list(); Injector::inst()->unregisterNamedObject('QueuedJobService');
parent::tearDown(); FullTextSearch::force_index_list();
} parent::tearDown();
}
/** /**
* @return SearchUpdateQueuedJobProcessor * @return SearchUpdateQueuedJobProcessor
*/ */
protected function generateDirtyIds() { protected function generateDirtyIds()
$processor = SearchUpdater::$processor; {
for($id = 1; $id <= 42; $id++) { $processor = SearchUpdater::$processor;
// Save to db for ($id = 1; $id <= 42; $id++) {
$object = new BatchedProcessorTest_Object(); // Save to db
$object->TestText = 'Object ' . $id; $object = new BatchedProcessorTest_Object();
$object->write(); $object->TestText = 'Object ' . $id;
// Add to index manually $object->write();
$processor->addDirtyIDs( // Add to index manually
'BatchedProcessorTest_Object', $processor->addDirtyIDs(
array(array( 'BatchedProcessorTest_Object',
'id' => $id, array(array(
'state' => array('SearchVariantVersioned' => 'Stage') 'id' => $id,
)), 'state' => array('SearchVariantVersioned' => 'Stage')
'BatchedProcessorTest_Index' )),
); 'BatchedProcessorTest_Index'
} );
$processor->batchData(); }
return $processor; $processor->batchData();
} return $processor;
}
/**
* Tests that large jobs are broken up into a suitable number of batches /**
*/ * Tests that large jobs are broken up into a suitable number of batches
public function testBatching() { */
$index = singleton('BatchedProcessorTest_Index'); public function testBatching()
$index->reset(); {
$processor = $this->generateDirtyIds(); $index = singleton('BatchedProcessorTest_Index');
$index->reset();
// Check initial state $processor = $this->generateDirtyIds();
$data = $processor->getJobData();
$this->assertEquals(9, $data->totalSteps); // Check initial state
$this->assertEquals(0, $data->currentStep); $data = $processor->getJobData();
$this->assertEmpty($data->isComplete); $this->assertEquals(9, $data->totalSteps);
$this->assertEquals(0, count($index->getAdded())); $this->assertEquals(0, $data->currentStep);
$this->assertEmpty($data->isComplete);
// Advance state $this->assertEquals(0, count($index->getAdded()));
for($pass = 1; $pass <= 8; $pass++) {
$processor->process(); // Advance state
$data = $processor->getJobData(); for ($pass = 1; $pass <= 8; $pass++) {
$this->assertEquals($pass, $data->currentStep); $processor->process();
$this->assertEquals($pass * 5, count($index->getAdded())); $data = $processor->getJobData();
} $this->assertEquals($pass, $data->currentStep);
$this->assertEquals($pass * 5, count($index->getAdded()));
// Last run should have two hanging items }
$processor->process();
$data = $processor->getJobData(); // Last run should have two hanging items
$this->assertEquals(9, $data->currentStep); $processor->process();
$this->assertEquals(42, count($index->getAdded())); $data = $processor->getJobData();
$this->assertTrue($data->isComplete); $this->assertEquals(9, $data->currentStep);
$this->assertEquals(42, count($index->getAdded()));
$this->assertTrue($data->isComplete);
// Check any additional queued jobs // Check any additional queued jobs
$processor->afterComplete(); $processor->afterComplete();
$service = singleton('QueuedJobService'); $service = singleton('QueuedJobService');
$jobs = $service->getJobs(); $jobs = $service->getJobs();
$this->assertEquals(1, count($jobs)); $this->assertEquals(1, count($jobs));
$this->assertInstanceOf('SearchUpdateCommitJobProcessor', $jobs[0]['job']); $this->assertInstanceOf('SearchUpdateCommitJobProcessor', $jobs[0]['job']);
} }
/** /**
* Test creation of multiple commit jobs * Test creation of multiple commit jobs
*/ */
public function testMultipleCommits() { public function testMultipleCommits()
$index = singleton('BatchedProcessorTest_Index'); {
$index->reset(); $index = singleton('BatchedProcessorTest_Index');
$index->reset();
// Test that running a commit immediately after submitting to the indexes // Test that running a commit immediately after submitting to the indexes
// correctly commits // correctly commits
$first = SearchUpdateCommitJobProcessor::queue(); $first = SearchUpdateCommitJobProcessor::queue();
$second = SearchUpdateCommitJobProcessor::queue(); $second = SearchUpdateCommitJobProcessor::queue();
$this->assertFalse($index->getIsCommitted()); $this->assertFalse($index->getIsCommitted());
// First process will cause the commit // First process will cause the commit
$this->assertFalse($first->jobFinished()); $this->assertFalse($first->jobFinished());
$first->process(); $first->process();
$allMessages = $first->getMessages(); $allMessages = $first->getMessages();
$this->assertTrue($index->getIsCommitted()); $this->assertTrue($index->getIsCommitted());
$this->assertTrue($first->jobFinished()); $this->assertTrue($first->jobFinished());
$this->assertStringEndsWith('All indexes committed', $allMessages[2]); $this->assertStringEndsWith('All indexes committed', $allMessages[2]);
// Executing the subsequent processor should not re-trigger a commit // Executing the subsequent processor should not re-trigger a commit
$index->reset(); $index->reset();
$this->assertFalse($second->jobFinished()); $this->assertFalse($second->jobFinished());
$second->process(); $second->process();
$allMessages = $second->getMessages(); $allMessages = $second->getMessages();
$this->assertFalse($index->getIsCommitted()); $this->assertFalse($index->getIsCommitted());
$this->assertTrue($second->jobFinished()); $this->assertTrue($second->jobFinished());
$this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]); $this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]);
// Given that a third job is created, and the indexes are dirtied, attempting to run this job // Given that a third job is created, and the indexes are dirtied, attempting to run this job
// should result in a delay // should result in a delay
$index->reset(); $index->reset();
$third = SearchUpdateCommitJobProcessor::queue(); $third = SearchUpdateCommitJobProcessor::queue();
$this->assertFalse($third->jobFinished()); $this->assertFalse($third->jobFinished());
$third->process(); $third->process();
$this->assertTrue($third->jobFinished()); $this->assertTrue($third->jobFinished());
$allMessages = $third->getMessages(); $allMessages = $third->getMessages();
$this->assertStringEndsWith( $this->assertStringEndsWith(
'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00', 'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00',
$allMessages[0] $allMessages[0]
); );
} }
/** /**
* Tests that the batch_soft_cap setting is properly respected * Tests that the batch_soft_cap setting is properly respected
*/ */
public function testSoftCap() { public function testSoftCap()
$index = singleton('BatchedProcessorTest_Index'); {
$index->reset(); $index = singleton('BatchedProcessorTest_Index');
$processor = $this->generateDirtyIds(); $index->reset();
$processor = $this->generateDirtyIds();
// Test that increasing the soft cap to 2 will reduce the number of batches
Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 2); // Test that increasing the soft cap to 2 will reduce the number of batches
$processor->batchData(); Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 2);
$data = $processor->getJobData(); $processor->batchData();
//Debug::dump($data);die; $data = $processor->getJobData();
$this->assertEquals(8, $data->totalSteps); //Debug::dump($data);die;
$this->assertEquals(8, $data->totalSteps);
// A soft cap of 1 should not fit in the hanging two items
Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 1); // A soft cap of 1 should not fit in the hanging two items
$processor->batchData(); Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 1);
$data = $processor->getJobData(); $processor->batchData();
$this->assertEquals(9, $data->totalSteps); $data = $processor->getJobData();
$this->assertEquals(9, $data->totalSteps);
// Extra large soft cap should fit both items
Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 4); // Extra large soft cap should fit both items
$processor->batchData(); Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 4);
$data = $processor->getJobData(); $processor->batchData();
$this->assertEquals(8, $data->totalSteps); $data = $processor->getJobData();
$this->assertEquals(8, $data->totalSteps);
// Process all data and ensure that all are processed adequately
for($pass = 1; $pass <= 8; $pass++) { // Process all data and ensure that all are processed adequately
$processor->process(); for ($pass = 1; $pass <= 8; $pass++) {
} $processor->process();
$data = $processor->getJobData(); }
$this->assertEquals(8, $data->currentStep); $data = $processor->getJobData();
$this->assertEquals(42, count($index->getAdded())); $this->assertEquals(8, $data->currentStep);
$this->assertTrue($data->isComplete); $this->assertEquals(42, count($index->getAdded()));
} $this->assertTrue($data->isComplete);
}
} }

318
tests/SearchUpdaterTest.php
View File

@ -1,215 +1,229 @@
<?php <?php
class SearchUpdaterTest_Container extends DataObject { class SearchUpdaterTest_Container extends DataObject
private static $db = array( {
'Field1' => 'Varchar', private static $db = array(
'Field2' => 'Varchar', 'Field1' => 'Varchar',
'MyDate' => 'Date', 'Field2' => 'Varchar',
); 'MyDate' => 'Date',
);
private static $has_one = array( private static $has_one = array(
'HasOneObject' => 'SearchUpdaterTest_HasOne' 'HasOneObject' => 'SearchUpdaterTest_HasOne'
); );
private static $has_many = array( private static $has_many = array(
'HasManyObjects' => 'SearchUpdaterTest_HasMany' 'HasManyObjects' => 'SearchUpdaterTest_HasMany'
); );
private static $many_many = array( private static $many_many = array(
'ManyManyObjects' => 'SearchUpdaterTest_ManyMany' 'ManyManyObjects' => 'SearchUpdaterTest_ManyMany'
); );
} }
class SearchUpdaterTest_HasOne extends DataObject { class SearchUpdaterTest_HasOne extends DataObject
private static $db = array( {
'Field1' => 'Varchar', private static $db = array(
'Field2' => 'Varchar' 'Field1' => 'Varchar',
); 'Field2' => 'Varchar'
);
private static $has_many = array( private static $has_many = array(
'HasManyContainers' => 'SearchUpdaterTest_Container' 'HasManyContainers' => 'SearchUpdaterTest_Container'
); );
} }
class SearchUpdaterTest_HasMany extends DataObject { class SearchUpdaterTest_HasMany extends DataObject
private static $db = array( {
'Field1' => 'Varchar', private static $db = array(
'Field2' => 'Varchar' 'Field1' => 'Varchar',
); 'Field2' => 'Varchar'
);
private static $has_one = array( private static $has_one = array(
'HasManyContainer' => 'SearchUpdaterTest_Container' 'HasManyContainer' => 'SearchUpdaterTest_Container'
); );
} }
class SearchUpdaterTest_ManyMany extends DataObject { class SearchUpdaterTest_ManyMany extends DataObject
private static $db = array( {
'Field1' => 'Varchar', private static $db = array(
'Field2' => 'Varchar' 'Field1' => 'Varchar',
); 'Field2' => 'Varchar'
);
private static $belongs_many_many = array( private static $belongs_many_many = array(
'ManyManyContainer' => 'SearchUpdaterTest_Container' 'ManyManyContainer' => 'SearchUpdaterTest_Container'
); );
} }
class SearchUpdaterTest_Index extends SearchIndex_Recording { class SearchUpdaterTest_Index extends SearchIndex_Recording
function init() { {
$this->addClass('SearchUpdaterTest_Container'); public function init()
{
$this->addClass('SearchUpdaterTest_Container');
$this->addFilterField('Field1'); $this->addFilterField('Field1');
$this->addFilterField('HasOneObject.Field1'); $this->addFilterField('HasOneObject.Field1');
$this->addFilterField('HasManyObjects.Field1'); $this->addFilterField('HasManyObjects.Field1');
} }
} }
class SearchUpdaterTest extends SapphireTest { class SearchUpdaterTest extends SapphireTest
{
protected $usesDatabase = true;
protected $usesDatabase = true; private static $index = null;
public function setUp()
{
parent::setUp();
private static $index = null; if (self::$index === null) {
self::$index = singleton(get_class($this).'_Index');
function setUp() { } else {
parent::setUp(); self::$index->reset();
}
if (self::$index === null) self::$index = singleton(get_class($this).'_Index'); SearchUpdater::bind_manipulation_capture();
else self::$index->reset();
SearchUpdater::bind_manipulation_capture(); Config::nest();
Config::nest(); Config::inst()->update('Injector', 'SearchUpdateProcessor', array(
'class' => 'SearchUpdateImmediateProcessor'
));
Config::inst()->update('Injector', 'SearchUpdateProcessor', array( FullTextSearch::force_index_list(self::$index);
'class' => 'SearchUpdateImmediateProcessor' SearchUpdater::clear_dirty_indexes();
)); }
FullTextSearch::force_index_list(self::$index); public function tearDown()
SearchUpdater::clear_dirty_indexes(); {
} Config::unnest();
function tearDown() { parent::tearDown();
Config::unnest(); }
parent::tearDown(); public function testBasic()
} {
$item = new SearchUpdaterTest_Container();
$item->write();
function testBasic() { // TODO: Make sure changing field1 updates item.
$item = new SearchUpdaterTest_Container(); // TODO: Get updating just field2 to not update item (maybe not possible - variants complicate)
$item->write(); }
// TODO: Make sure changing field1 updates item. public function testHasOneHook()
// TODO: Get updating just field2 to not update item (maybe not possible - variants complicate) {
} $hasOne = new SearchUpdaterTest_HasOne();
$hasOne->write();
function testHasOneHook() { $alternateHasOne = new SearchUpdaterTest_HasOne();
$hasOne = new SearchUpdaterTest_HasOne(); $alternateHasOne->write();
$hasOne->write();
$alternateHasOne = new SearchUpdaterTest_HasOne(); $container1 = new SearchUpdaterTest_Container();
$alternateHasOne->write(); $container1->HasOneObjectID = $hasOne->ID;
$container1->write();
$container1 = new SearchUpdaterTest_Container(); $container2 = new SearchUpdaterTest_Container();
$container1->HasOneObjectID = $hasOne->ID; $container2->HasOneObjectID = $hasOne->ID;
$container1->write(); $container2->write();
$container2 = new SearchUpdaterTest_Container(); $container3 = new SearchUpdaterTest_Container();
$container2->HasOneObjectID = $hasOne->ID; $container3->HasOneObjectID = $alternateHasOne->ID;
$container2->write(); $container3->write();
$container3 = new SearchUpdaterTest_Container(); // Check the default "writing a document updates the document"
$container3->HasOneObjectID = $alternateHasOne->ID; SearchUpdater::flush_dirty_indexes();
$container3->write();
// Check the default "writing a document updates the document"
SearchUpdater::flush_dirty_indexes();
$added = self::$index->getAdded(array('ID')); $added = self::$index->getAdded(array('ID'));
// Some databases don't output $added in a consistent order; that's okay // Some databases don't output $added in a consistent order; that's okay
usort($added, function($a,$b) {return $a['ID']-$b['ID']; }); usort($added, function ($a, $b) {return $a['ID']-$b['ID']; });
$this->assertEquals($added, array( $this->assertEquals($added, array(
array('ID' => $container1->ID), array('ID' => $container1->ID),
array('ID' => $container2->ID), array('ID' => $container2->ID),
array('ID' => $container3->ID) array('ID' => $container3->ID)
)); ));
// Check writing a has_one tracks back to the origin documents // Check writing a has_one tracks back to the origin documents
self::$index->reset(); self::$index->reset();
$hasOne->Field1 = "Updated"; $hasOne->Field1 = "Updated";
$hasOne->write(); $hasOne->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$added = self::$index->getAdded(array('ID')); $added = self::$index->getAdded(array('ID'));
// Some databases don't output $added in a consistent order; that's okay // Some databases don't output $added in a consistent order; that's okay
usort($added, function($a,$b) {return $a['ID']-$b['ID']; }); usort($added, function ($a, $b) {return $a['ID']-$b['ID']; });
$this->assertEquals($added, array( $this->assertEquals($added, array(
array('ID' => $container1->ID), array('ID' => $container1->ID),
array('ID' => $container2->ID) array('ID' => $container2->ID)
)); ));
// Check updating an unrelated field doesn't track back // Check updating an unrelated field doesn't track back
self::$index->reset(); self::$index->reset();
$hasOne->Field2 = "Updated"; $hasOne->Field2 = "Updated";
$hasOne->write(); $hasOne->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID')), array()); $this->assertEquals(self::$index->getAdded(array('ID')), array());
// Check writing a has_one tracks back to the origin documents // Check writing a has_one tracks back to the origin documents
self::$index->reset(); self::$index->reset();
$alternateHasOne->Field1= "Updated"; $alternateHasOne->Field1= "Updated";
$alternateHasOne->write(); $alternateHasOne->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID')), array( $this->assertEquals(self::$index->getAdded(array('ID')), array(
array('ID' => $container3->ID) array('ID' => $container3->ID)
)); ));
} }
function testHasManyHook() { public function testHasManyHook()
$container1 = new SearchUpdaterTest_Container(); {
$container1->write(); $container1 = new SearchUpdaterTest_Container();
$container1->write();
$container2 = new SearchUpdaterTest_Container(); $container2 = new SearchUpdaterTest_Container();
$container2->write(); $container2->write();
//self::$index->reset(); //self::$index->reset();
//SearchUpdater::clear_dirty_indexes(); //SearchUpdater::clear_dirty_indexes();
$hasMany1 = new SearchUpdaterTest_HasMany(); $hasMany1 = new SearchUpdaterTest_HasMany();
$hasMany1->HasManyContainerID = $container1->ID; $hasMany1->HasManyContainerID = $container1->ID;
$hasMany1->write(); $hasMany1->write();
$hasMany2 = new SearchUpdaterTest_HasMany(); $hasMany2 = new SearchUpdaterTest_HasMany();
$hasMany2->HasManyContainerID = $container1->ID; $hasMany2->HasManyContainerID = $container1->ID;
$hasMany2->write(); $hasMany2->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID')), array( $this->assertEquals(self::$index->getAdded(array('ID')), array(
array('ID' => $container1->ID), array('ID' => $container1->ID),
array('ID' => $container2->ID) array('ID' => $container2->ID)
)); ));
self::$index->reset(); self::$index->reset();
$hasMany1->Field1 = 'Updated'; $hasMany1->Field1 = 'Updated';
$hasMany1->write(); $hasMany1->write();
$hasMany2->Field1 = 'Updated'; $hasMany2->Field1 = 'Updated';
$hasMany2->write(); $hasMany2->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID')), array( $this->assertEquals(self::$index->getAdded(array('ID')), array(
array('ID' => $container1->ID) array('ID' => $container1->ID)
)); ));
} }
} }

119
tests/SearchVariantSiteTreeSubsitesPolyhomeTest.php
View File

@ -1,76 +1,83 @@
<?php <?php
class SearchVariantSiteTreeSubsitesPolyhomeTest_Item extends SiteTree { class SearchVariantSiteTreeSubsitesPolyhomeTest_Item extends SiteTree
// TODO: Currently theres a failure if you addClass a non-table class {
private static $db = array( // TODO: Currently theres a failure if you addClass a non-table class
'TestText' => 'Varchar' private static $db = array(
); 'TestText' => 'Varchar'
);
} }
class SearchVariantSiteTreeSubsitesPolyhomeTest_Index extends SearchIndex_Recording { class SearchVariantSiteTreeSubsitesPolyhomeTest_Index extends SearchIndex_Recording
function init() { {
$this->addClass('SearchVariantSiteTreeSubsitesPolyhomeTest_Item'); public function init()
$this->addFilterField('TestText'); {
} $this->addClass('SearchVariantSiteTreeSubsitesPolyhomeTest_Item');
$this->addFilterField('TestText');
}
} }
class SearchVariantSiteTreeSubsitesPolyhomeTest extends SapphireTest { class SearchVariantSiteTreeSubsitesPolyhomeTest extends SapphireTest
{
private static $index = null;
private static $index = null; private static $subsite_a = null;
private static $subsite_b = null;
private static $subsite_a = null; public function setUp()
private static $subsite_b = null; {
parent::setUp();
function setUp() { // Check subsites installed
parent::setUp(); if (!class_exists('Subsite') || !class_exists('SubsitePolyhome')) {
return $this->markTestSkipped('The subsites polyhome module is not installed');
}
// Check subsites installed if (self::$index === null) {
if(!class_exists('Subsite') || !class_exists('SubsitePolyhome')) { self::$index = singleton('SearchVariantSiteTreeSubsitesPolyhomeTest_Index');
return $this->markTestSkipped('The subsites polyhome module is not installed'); }
}
if (self::$index === null) self::$index = singleton('SearchVariantSiteTreeSubsitesPolyhomeTest_Index'); if (self::$subsite_a === null) {
self::$subsite_a = new Subsite();
self::$subsite_a->write();
self::$subsite_b = new Subsite();
self::$subsite_b->write();
}
if (self::$subsite_a === null) { FullTextSearch::force_index_list(self::$index);
self::$subsite_a = new Subsite(); self::$subsite_a->write(); SearchUpdater::clear_dirty_indexes();
self::$subsite_b = new Subsite(); self::$subsite_b->write(); }
}
FullTextSearch::force_index_list(self::$index); public function testSavingDirect()
SearchUpdater::clear_dirty_indexes(); {
} // Initial add
function testSavingDirect() { $item = new SearchVariantSiteTreeSubsitesPolyhomeTest_Item();
// Initial add $item->write();
$item = new SearchVariantSiteTreeSubsitesPolyhomeTest_Item(); SearchUpdater::flush_dirty_indexes();
$item->write(); $this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array(
array('ID' => $item->ID, '_subsite' => 0)
));
SearchUpdater::flush_dirty_indexes(); // Check that adding to subsites works
$this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array(
array('ID' => $item->ID, '_subsite' => 0)
));
// Check that adding to subsites works self::$index->reset();
self::$index->reset(); $item->setField('AddToSubsite[0]', 1);
$item->setField('AddToSubsite['.(self::$subsite_a->ID).']', 1);
$item->setField('AddToSubsite[0]', 1); $item->write();
$item->setField('AddToSubsite['.(self::$subsite_a->ID).']', 1);
$item->write(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array(
SearchUpdater::flush_dirty_indexes(); array('ID' => $item->ID, '_subsite' => 0),
$this->assertEquals(self::$index->getAdded(array('ID', '_subsite')), array( array('ID' => $item->ID, '_subsite' => self::$subsite_a->ID)
array('ID' => $item->ID, '_subsite' => 0), ));
array('ID' => $item->ID, '_subsite' => self::$subsite_a->ID) $this->assertEquals(self::$index->deleted, array(
)); array('base' => 'SiteTree', 'id' => $item->ID, 'state' => array(
$this->assertEquals(self::$index->deleted, array( 'SearchVariantVersioned' => 'Stage', 'SearchVariantSiteTreeSubsitesPolyhome' => self::$subsite_b->ID
array('base' => 'SiteTree', 'id' => $item->ID, 'state' => array( ))
'SearchVariantVersioned' => 'Stage', 'SearchVariantSiteTreeSubsitesPolyhome' => self::$subsite_b->ID ));
)) }
)); }
}
}

175
tests/SearchVariantVersionedTest.php
View File

@ -1,114 +1,125 @@
<?php <?php
class SearchVariantVersionedTest extends SapphireTest { class SearchVariantVersionedTest extends SapphireTest
{
private static $index = null;
private static $index = null; protected $extraDataObjects = array(
'SearchVariantVersionedTest_Item'
);
protected $extraDataObjects = array( public function setUp()
'SearchVariantVersionedTest_Item' {
); parent::setUp();
function setUp() { // Check versioned available
parent::setUp(); if (!class_exists('Versioned')) {
return $this->markTestSkipped('The versioned decorator is not installed');
}
// Check versioned available if (self::$index === null) {
if(!class_exists('Versioned')) { self::$index = singleton('SearchVariantVersionedTest_Index');
return $this->markTestSkipped('The versioned decorator is not installed'); }
}
if (self::$index === null) self::$index = singleton('SearchVariantVersionedTest_Index'); SearchUpdater::bind_manipulation_capture();
SearchUpdater::bind_manipulation_capture(); Config::nest();
Config::nest(); Config::inst()->update('Injector', 'SearchUpdateProcessor', array(
'class' => 'SearchUpdateImmediateProcessor'
));
Config::inst()->update('Injector', 'SearchUpdateProcessor', array( FullTextSearch::force_index_list(self::$index);
'class' => 'SearchUpdateImmediateProcessor' SearchUpdater::clear_dirty_indexes();
)); }
FullTextSearch::force_index_list(self::$index); public function tearDown()
SearchUpdater::clear_dirty_indexes(); {
} Config::unnest();
function tearDown() { parent::tearDown();
Config::unnest(); }
parent::tearDown(); public function testPublishing()
} {
// Check that write updates Stage
function testPublishing() { $item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo'));
// Check that write updates Stage $item->write();
$item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo'));
$item->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array(
array('ID' => $item->ID, '_versionedstage' => 'Stage') array('ID' => $item->ID, '_versionedstage' => 'Stage')
)); ));
// Check that publish updates Live // Check that publish updates Live
self::$index->reset(); self::$index->reset();
$item->publish("Stage", "Live"); $item->publish("Stage", "Live");
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array(
array('ID' => $item->ID, '_versionedstage' => 'Live') array('ID' => $item->ID, '_versionedstage' => 'Live')
)); ));
// Just update a SiteTree field, and check it updates Stage // Just update a SiteTree field, and check it updates Stage
self::$index->reset(); self::$index->reset();
$item->Title = "Pow!"; $item->Title = "Pow!";
$item->write(); $item->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array( $this->assertEquals(self::$index->getAdded(array('ID', '_versionedstage')), array(
array('ID' => $item->ID, '_versionedstage' => 'Stage') array('ID' => $item->ID, '_versionedstage' => 'Stage')
)); ));
} }
function testExcludeVariantState() { public function testExcludeVariantState()
$index = singleton('SearchVariantVersionedTest_IndexNoStage'); {
FullTextSearch::force_index_list($index); $index = singleton('SearchVariantVersionedTest_IndexNoStage');
FullTextSearch::force_index_list($index);
// Check that write doesn't update stage // Check that write doesn't update stage
$item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo')); $item = new SearchVariantVersionedTest_Item(array('TestText' => 'Foo'));
$item->write(); $item->write();
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array()); $this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array());
// Check that publish updates Live // Check that publish updates Live
$index->reset(); $index->reset();
$item->publish("Stage", "Live"); $item->publish("Stage", "Live");
SearchUpdater::flush_dirty_indexes(); SearchUpdater::flush_dirty_indexes();
$this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array( $this->assertEquals($index->getAdded(array('ID', '_versionedstage')), array(
array('ID' => $item->ID, '_versionedstage' => 'Live') array('ID' => $item->ID, '_versionedstage' => 'Live')
)); ));
} }
} }
class SearchVariantVersionedTest_Item extends SiteTree implements TestOnly { class SearchVariantVersionedTest_Item extends SiteTree implements TestOnly
// TODO: Currently theres a failure if you addClass a non-table class {
private static $db = array( // TODO: Currently theres a failure if you addClass a non-table class
'TestText' => 'Varchar' private static $db = array(
); 'TestText' => 'Varchar'
);
} }
class SearchVariantVersionedTest_Index extends SearchIndex_Recording { class SearchVariantVersionedTest_Index extends SearchIndex_Recording
function init() { {
$this->addClass('SearchVariantVersionedTest_Item'); public function init()
$this->addFilterField('TestText'); {
} $this->addClass('SearchVariantVersionedTest_Item');
$this->addFilterField('TestText');
}
} }
class SearchVariantVersionedTest_IndexNoStage extends SearchIndex_Recording { class SearchVariantVersionedTest_IndexNoStage extends SearchIndex_Recording
function init() { {
$this->addClass('SearchVariantVersionedTest_Item'); public function init()
$this->addFilterField('TestText'); {
$this->excludeVariantState(array('SearchVariantVersioned' => 'Stage')); $this->addClass('SearchVariantVersionedTest_Item');
} $this->addFilterField('TestText');
} $this->excludeVariantState(array('SearchVariantVersioned' => 'Stage'));
}
}

123
tests/Solr4ServiceTest.php
View File

@ -3,64 +3,71 @@
/** /**
* Test solr 4.0 compatibility * Test solr 4.0 compatibility
*/ */
class Solr4ServiceTest extends SapphireTest { class Solr4ServiceTest extends SapphireTest
{
/** /**
* *
* @return Solr4ServiceTest_RecordingService * @return Solr4ServiceTest_RecordingService
*/ */
protected function getMockService() { protected function getMockService()
return new Solr4ServiceTest_RecordingService(); {
} return new Solr4ServiceTest_RecordingService();
}
protected function getMockDocument($id) {
$document = new Apache_Solr_Document(); protected function getMockDocument($id)
$document->setField('id', $id); {
$document->setField('title', "Item $id"); $document = new Apache_Solr_Document();
return $document; $document->setField('id', $id);
} $document->setField('title', "Item $id");
return $document;
public function testAddDocument() { }
$service = $this->getMockService();
$sent = $service->addDocument($this->getMockDocument('A'), false); public function testAddDocument()
$this->assertEquals( {
'<add overwrite="true"><doc><field name="id">A</field><field name="title">Item A</field></doc></add>', $service = $this->getMockService();
$sent $sent = $service->addDocument($this->getMockDocument('A'), false);
); $this->assertEquals(
$sent = $service->addDocument($this->getMockDocument('B'), true); '<add overwrite="true"><doc><field name="id">A</field><field name="title">Item A</field></doc></add>',
$this->assertEquals( $sent
'<add overwrite="false"><doc><field name="id">B</field><field name="title">Item B</field></doc></add>', );
$sent $sent = $service->addDocument($this->getMockDocument('B'), true);
); $this->assertEquals(
} '<add overwrite="false"><doc><field name="id">B</field><field name="title">Item B</field></doc></add>',
$sent
public function testAddDocuments() { );
$service = $this->getMockService(); }
$sent = $service->addDocuments(array(
$this->getMockDocument('C'), public function testAddDocuments()
$this->getMockDocument('D') {
), false); $service = $this->getMockService();
$this->assertEquals( $sent = $service->addDocuments(array(
'<add overwrite="true"><doc><field name="id">C</field><field name="title">Item C</field></doc><doc><field name="id">D</field><field name="title">Item D</field></doc></add>', $this->getMockDocument('C'),
$sent $this->getMockDocument('D')
); ), false);
$sent = $service->addDocuments(array( $this->assertEquals(
$this->getMockDocument('E'), '<add overwrite="true"><doc><field name="id">C</field><field name="title">Item C</field></doc><doc><field name="id">D</field><field name="title">Item D</field></doc></add>',
$this->getMockDocument('F') $sent
), true); );
$this->assertEquals( $sent = $service->addDocuments(array(
'<add overwrite="false"><doc><field name="id">E</field><field name="title">Item E</field></doc><doc><field name="id">F</field><field name="title">Item F</field></doc></add>', $this->getMockDocument('E'),
$sent $this->getMockDocument('F')
); ), true);
} $this->assertEquals(
'<add overwrite="false"><doc><field name="id">E</field><field name="title">Item E</field></doc><doc><field name="id">F</field><field name="title">Item F</field></doc></add>',
$sent
);
}
} }
class Solr4ServiceTest_RecordingService extends Solr4Service_Core { class Solr4ServiceTest_RecordingService extends Solr4Service_Core
protected function _sendRawPost($url, $rawPost, $timeout = FALSE, $contentType = 'text/xml; charset=UTF-8') { {
return $rawPost; protected function _sendRawPost($url, $rawPost, $timeout = false, $contentType = 'text/xml; charset=UTF-8')
} {
return $rawPost;
protected function _sendRawGet($url, $timeout = FALSE) { }
return $url;
} protected function _sendRawGet($url, $timeout = false)
{
return $url;
}
} }

531
tests/SolrIndexTest.php
View File

@ -1,307 +1,330 @@
<?php <?php
class SolrIndexTest extends SapphireTest { class SolrIndexTest extends SapphireTest
{
public function setUpOnce()
{
parent::setUpOnce();
function setUpOnce() { if (class_exists('Phockito')) {
parent::setUpOnce(); Phockito::include_hamcrest();
}
}
if (class_exists('Phockito')) Phockito::include_hamcrest(); public function setUp()
} {
if (!class_exists('Phockito')) {
$this->markTestSkipped("These tests need the Phockito module installed to run");
$this->skipTest = true;
}
function setUp() { parent::setUp();
if (!class_exists('Phockito')) { }
$this->markTestSkipped("These tests need the Phockito module installed to run");
$this->skipTest = true;
}
parent::setUp(); public function testFieldDataHasOne()
} {
$index = new SolrIndexTest_FakeIndex();
$data = $index->fieldData('HasOneObject.Field1');
$data = $data['SearchUpdaterTest_Container_HasOneObject_Field1'];
function testFieldDataHasOne() { $this->assertEquals('SearchUpdaterTest_Container', $data['origin']);
$index = new SolrIndexTest_FakeIndex(); $this->assertEquals('SearchUpdaterTest_Container', $data['base']);
$data = $index->fieldData('HasOneObject.Field1'); $this->assertEquals('SearchUpdaterTest_HasOne', $data['class']);
$data = $data['SearchUpdaterTest_Container_HasOneObject_Field1']; }
$this->assertEquals('SearchUpdaterTest_Container', $data['origin']); public function testFieldDataHasMany()
$this->assertEquals('SearchUpdaterTest_Container', $data['base']); {
$this->assertEquals('SearchUpdaterTest_HasOne', $data['class']); $index = new SolrIndexTest_FakeIndex();
} $data = $index->fieldData('HasManyObjects.Field1');
$data = $data['SearchUpdaterTest_Container_HasManyObjects_Field1'];
function testFieldDataHasMany() { $this->assertEquals('SearchUpdaterTest_Container', $data['origin']);
$index = new SolrIndexTest_FakeIndex(); $this->assertEquals('SearchUpdaterTest_Container', $data['base']);
$data = $index->fieldData('HasManyObjects.Field1'); $this->assertEquals('SearchUpdaterTest_HasMany', $data['class']);
$data = $data['SearchUpdaterTest_Container_HasManyObjects_Field1']; }
$this->assertEquals('SearchUpdaterTest_Container', $data['origin']); public function testFieldDataManyMany()
$this->assertEquals('SearchUpdaterTest_Container', $data['base']); {
$this->assertEquals('SearchUpdaterTest_HasMany', $data['class']); $index = new SolrIndexTest_FakeIndex();
} $data = $index->fieldData('ManyManyObjects.Field1');
$data = $data['SearchUpdaterTest_Container_ManyManyObjects_Field1'];
function testFieldDataManyMany() { $this->assertEquals('SearchUpdaterTest_Container', $data['origin']);
$index = new SolrIndexTest_FakeIndex(); $this->assertEquals('SearchUpdaterTest_Container', $data['base']);
$data = $index->fieldData('ManyManyObjects.Field1'); $this->assertEquals('SearchUpdaterTest_ManyMany', $data['class']);
$data = $data['SearchUpdaterTest_Container_ManyManyObjects_Field1']; }
$this->assertEquals('SearchUpdaterTest_Container', $data['origin']); /**
$this->assertEquals('SearchUpdaterTest_Container', $data['base']); * Test boosting on SearchQuery
$this->assertEquals('SearchUpdaterTest_ManyMany', $data['class']); */
} public function testBoostedQuery()
{
$serviceMock = $this->getServiceMock();
Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse());
/** $index = new SolrIndexTest_FakeIndex();
* Test boosting on SearchQuery $index->setService($serviceMock);
*/
function testBoostedQuery() {
$serviceMock = $this->getServiceMock();
Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse());
$index = new SolrIndexTest_FakeIndex(); $query = new SearchQuery();
$index->setService($serviceMock); $query->search(
'term',
null,
array('Field1' => 1.5, 'HasOneObject_Field1' => 3)
);
$index->search($query);
$query = new SearchQuery(); Phockito::verify($serviceMock)->search('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', anything(), anything(), anything(), anything());
$query->search( }
'term',
null, /**
array('Field1' => 1.5, 'HasOneObject_Field1' => 3) * Test boosting on field schema (via queried fields parameter)
); */
$index->search($query); public function testBoostedField()
{
$serviceMock = $this->getServiceMock();
Phockito::when($serviceMock)
->search(anything(), anything(), anything(), anything(), anything())
->return($this->getFakeRawSolrResponse());
Phockito::verify($serviceMock)->search('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', anything(), anything(), anything(), anything()); $index = new SolrIndexTest_BoostedIndex();
} $index->setService($serviceMock);
/**
* Test boosting on field schema (via queried fields parameter)
*/
public function testBoostedField() {
$serviceMock = $this->getServiceMock();
Phockito::when($serviceMock)
->search(anything(), anything(), anything(), anything(), anything())
->return($this->getFakeRawSolrResponse());
$index = new SolrIndexTest_BoostedIndex(); $query = new SearchQuery();
$index->setService($serviceMock); $query->search('term');
$index->search($query);
$query = new SearchQuery(); // Ensure matcher contains correct boost in 'qf' parameter
$query->search('term'); $matcher = new Hamcrest_Array_IsArrayContainingKeyValuePair(
$index->search($query); new Hamcrest_Core_IsEqual('qf'),
new Hamcrest_Core_IsEqual('SearchUpdaterTest_Container_Field1^1.5 SearchUpdaterTest_Container_Field2^2.1 _text')
);
Phockito::verify($serviceMock)
->search('+term', anything(), anything(), $matcher, anything());
}
// Ensure matcher contains correct boost in 'qf' parameter public function testHighlightQueryOnBoost()
$matcher = new Hamcrest_Array_IsArrayContainingKeyValuePair( {
new Hamcrest_Core_IsEqual('qf'), $serviceMock = $this->getServiceMock();
new Hamcrest_Core_IsEqual('SearchUpdaterTest_Container_Field1^1.5 SearchUpdaterTest_Container_Field2^2.1 _text') Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse());
);
Phockito::verify($serviceMock)
->search('+term', anything(), anything(), $matcher, anything());
}
function testHighlightQueryOnBoost() { $index = new SolrIndexTest_FakeIndex();
$serviceMock = $this->getServiceMock(); $index->setService($serviceMock);
Phockito::when($serviceMock)->search(anything(), anything(), anything(), anything(), anything())->return($this->getFakeRawSolrResponse());
$index = new SolrIndexTest_FakeIndex(); // Search without highlighting
$index->setService($serviceMock); $query = new SearchQuery();
$query->search(
'term',
null,
array('Field1' => 1.5, 'HasOneObject_Field1' => 3)
);
$index->search($query);
Phockito::verify(
$serviceMock)->search(
'+(Field1:term^1.5 OR HasOneObject_Field1:term^3)',
anything(),
anything(),
not(hasKeyInArray('hl.q')),
anything()
);
// Search without highlighting // Search with highlighting
$query = new SearchQuery(); $query = new SearchQuery();
$query->search( $query->search(
'term', 'term',
null, null,
array('Field1' => 1.5, 'HasOneObject_Field1' => 3) array('Field1' => 1.5, 'HasOneObject_Field1' => 3)
); );
$index->search($query); $index->search($query, -1, -1, array('hl' => true));
Phockito::verify( Phockito::verify(
$serviceMock)->search( $serviceMock)->search(
'+(Field1:term^1.5 OR HasOneObject_Field1:term^3)', '+(Field1:term^1.5 OR HasOneObject_Field1:term^3)',
anything(), anything(),
anything(), anything(),
not(hasKeyInArray('hl.q')), hasKeyInArray('hl.q'),
anything() anything()
); );
}
// Search with highlighting public function testIndexExcludesNullValues()
$query = new SearchQuery(); {
$query->search( $serviceMock = $this->getServiceMock();
'term', $index = new SolrIndexTest_FakeIndex();
null, $index->setService($serviceMock);
array('Field1' => 1.5, 'HasOneObject_Field1' => 3) $obj = new SearchUpdaterTest_Container();
);
$index->search($query, -1, -1, array('hl' => true));
Phockito::verify(
$serviceMock)->search(
'+(Field1:term^1.5 OR HasOneObject_Field1:term^3)',
anything(),
anything(),
hasKeyInArray('hl.q'),
anything()
);
}
function testIndexExcludesNullValues() { $obj->Field1 = 'Field1 val';
$serviceMock = $this->getServiceMock(); $obj->Field2 = null;
$index = new SolrIndexTest_FakeIndex(); $obj->MyDate = null;
$index->setService($serviceMock); $docs = $index->add($obj);
$obj = new SearchUpdaterTest_Container(); $value = $docs[0]->getField('SearchUpdaterTest_Container_Field1');
$this->assertEquals('Field1 val', $value['value'], 'Writes non-NULL string fields');
$value = $docs[0]->getField('SearchUpdaterTest_Container_Field2');
$this->assertFalse($value, 'Ignores string fields if they are NULL');
$value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate');
$this->assertFalse($value, 'Ignores date fields if they are NULL');
$obj->Field1 = 'Field1 val'; $obj->MyDate = '2010-12-30';
$obj->Field2 = null; $docs = $index->add($obj);
$obj->MyDate = null; $value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate');
$docs = $index->add($obj); $this->assertEquals('2010-12-30T00:00:00Z', $value['value'], 'Writes non-NULL dates');
$value = $docs[0]->getField('SearchUpdaterTest_Container_Field1'); }
$this->assertEquals('Field1 val', $value['value'], 'Writes non-NULL string fields');
$value = $docs[0]->getField('SearchUpdaterTest_Container_Field2');
$this->assertFalse($value, 'Ignores string fields if they are NULL');
$value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate');
$this->assertFalse($value, 'Ignores date fields if they are NULL');
$obj->MyDate = '2010-12-30'; public function testAddFieldExtraOptions()
$docs = $index->add($obj); {
$value = $docs[0]->getField('SearchUpdaterTest_Container_MyDate'); Config::inst()->nest();
$this->assertEquals('2010-12-30T00:00:00Z', $value['value'], 'Writes non-NULL dates'); Config::inst()->update('Director', 'environment_type', 'live'); // dev mode sets stored=true for everything
}
function testAddFieldExtraOptions() { $index = new SolrIndexTest_FakeIndex();
Config::inst()->nest();
Config::inst()->update('Director', 'environment_type', 'live'); // dev mode sets stored=true for everything
$index = new SolrIndexTest_FakeIndex(); $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
$defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]');
$this->assertEquals((string)$defField1[0]['stored'], 'false');
$defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>'); $index->addFilterField('Field1', null, array('stored' => 'true'));
$defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
$this->assertEquals((string)$defField1[0]['stored'], 'false'); $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]');
$this->assertEquals((string)$defField1[0]['stored'], 'true');
$index->addFilterField('Field1', null, array('stored' => 'true')); Config::inst()->unnest();
$defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>'); }
$defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]');
$this->assertEquals((string)$defField1[0]['stored'], 'true');
Config::inst()->unnest(); public function testAddAnalyzer()
} {
$index = new SolrIndexTest_FakeIndex();
function testAddAnalyzer() { $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
$index = new SolrIndexTest_FakeIndex(); $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]');
$analyzers = $defField1[0]->analyzer;
$this->assertFalse((bool)$analyzers);
$defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>'); $index->addAnalyzer('Field1', 'charFilter', array('class' => 'solr.HTMLStripCharFilterFactory'));
$defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
$analyzers = $defField1[0]->analyzer; $defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]');
$this->assertFalse((bool)$analyzers); $analyzers = $defField1[0]->analyzer;
$this->assertTrue((bool)$analyzers);
$this->assertEquals('solr.HTMLStripCharFilterFactory', $analyzers[0]->charFilter[0]['class']);
}
$index->addAnalyzer('Field1', 'charFilter', array('class' => 'solr.HTMLStripCharFilterFactory')); public function testAddCopyField()
$defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>'); {
$defField1 = $defs->xpath('field[@name="SearchUpdaterTest_Container_Field1"]'); $index = new SolrIndexTest_FakeIndex();
$analyzers = $defField1[0]->analyzer; $index->addCopyField('sourceField', 'destField');
$this->assertTrue((bool)$analyzers);
$this->assertEquals('solr.HTMLStripCharFilterFactory', $analyzers[0]->charFilter[0]['class']);
}
function testAddCopyField() { $defs = simplexml_load_string('<fields>' . $index->getCopyFieldDefinitions() . '</fields>');
$index = new SolrIndexTest_FakeIndex(); $copyField = $defs->xpath('copyField');
$index->addCopyField('sourceField', 'destField');
$defs = simplexml_load_string('<fields>' . $index->getCopyFieldDefinitions() . '</fields>'); $this->assertEquals('sourceField', $copyField[0]['source']);
$copyField = $defs->xpath('copyField'); $this->assertEquals('destField', $copyField[0]['dest']);
}
$this->assertEquals('sourceField', $copyField[0]['source']); /**
$this->assertEquals('destField', $copyField[0]['dest']); * Tests the setting of the 'stored' flag
} */
public function testStoredFields()
{
// Test two fields
$index = new SolrIndexTest_FakeIndex2();
$index->addStoredField('Field1');
$index->addFulltextField('Field2');
$schema = $index->getFieldDefinitions();
$this->assertContains(
"<field name='SearchUpdaterTest_Container_Field1' type='text' indexed='true' stored='true'",
$schema
);
$this->assertContains(
"<field name='SearchUpdaterTest_Container_Field2' type='text' indexed='true' stored='false'",
$schema
);
/** // Test with addAllFulltextFields
* Tests the setting of the 'stored' flag $index2 = new SolrIndexTest_FakeIndex2();
*/ $index2->addAllFulltextFields();
public function testStoredFields() { $index2->addStoredField('Field2');
// Test two fields $schema2 = $index2->getFieldDefinitions();
$index = new SolrIndexTest_FakeIndex2(); $this->assertContains(
$index->addStoredField('Field1'); "<field name='SearchUpdaterTest_Container_Field1' type='text' indexed='true' stored='false'",
$index->addFulltextField('Field2'); $schema2
$schema = $index->getFieldDefinitions(); );
$this->assertContains( $this->assertContains(
"<field name='SearchUpdaterTest_Container_Field1' type='text' indexed='true' stored='true'", "<field name='SearchUpdaterTest_Container_Field2' type='text' indexed='true' stored='true'",
$schema $schema2
); );
$this->assertContains( }
"<field name='SearchUpdaterTest_Container_Field2' type='text' indexed='true' stored='false'",
$schema
);
// Test with addAllFulltextFields /**
$index2 = new SolrIndexTest_FakeIndex2(); * @return Solr3Service
$index2->addAllFulltextFields(); */
$index2->addStoredField('Field2'); protected function getServiceMock()
$schema2 = $index2->getFieldDefinitions(); {
$this->assertContains( return Phockito::mock('Solr3Service');
"<field name='SearchUpdaterTest_Container_Field1' type='text' indexed='true' stored='false'", }
$schema2
);
$this->assertContains(
"<field name='SearchUpdaterTest_Container_Field2' type='text' indexed='true' stored='true'",
$schema2
);
}
/** protected function getServiceSpy()
* @return Solr3Service {
*/ $serviceSpy = Phockito::spy('Solr3Service');
protected function getServiceMock() { Phockito::when($serviceSpy)->_sendRawPost()->return($this->getFakeRawSolrResponse());
return Phockito::mock('Solr3Service');
}
protected function getServiceSpy() { return $serviceSpy;
$serviceSpy = Phockito::spy('Solr3Service'); }
Phockito::when($serviceSpy)->_sendRawPost()->return($this->getFakeRawSolrResponse());
return $serviceSpy; protected function getFakeRawSolrResponse()
} {
return new Apache_Solr_Response(
protected function getFakeRawSolrResponse() { new Apache_Solr_HttpTransport_Response(
return new Apache_Solr_Response( null,
new Apache_Solr_HttpTransport_Response( null,
null, '{}'
null, )
'{}' );
) }
);
}
} }
class SolrIndexTest_FakeIndex extends SolrIndex { class SolrIndexTest_FakeIndex extends SolrIndex
function init() { {
$this->addClass('SearchUpdaterTest_Container'); public function init()
{
$this->addClass('SearchUpdaterTest_Container');
$this->addFilterField('Field1'); $this->addFilterField('Field1');
$this->addFilterField('MyDate', 'Date'); $this->addFilterField('MyDate', 'Date');
$this->addFilterField('HasOneObject.Field1'); $this->addFilterField('HasOneObject.Field1');
$this->addFilterField('HasManyObjects.Field1'); $this->addFilterField('HasManyObjects.Field1');
$this->addFilterField('ManyManyObjects.Field1'); $this->addFilterField('ManyManyObjects.Field1');
} }
} }
class SolrIndexTest_FakeIndex2 extends SolrIndex { class SolrIndexTest_FakeIndex2 extends SolrIndex
{
protected function getStoredDefault() { protected function getStoredDefault()
// Override isDev defaulting to stored {
return 'false'; // Override isDev defaulting to stored
} return 'false';
}
function init() { public function init()
$this->addClass('SearchUpdaterTest_Container'); {
$this->addFilterField('MyDate', 'Date'); $this->addClass('SearchUpdaterTest_Container');
$this->addFilterField('HasOneObject.Field1'); $this->addFilterField('MyDate', 'Date');
$this->addFilterField('HasManyObjects.Field1'); $this->addFilterField('HasOneObject.Field1');
$this->addFilterField('ManyManyObjects.Field1'); $this->addFilterField('HasManyObjects.Field1');
} $this->addFilterField('ManyManyObjects.Field1');
}
} }
class SolrIndexTest_BoostedIndex extends SolrIndex { class SolrIndexTest_BoostedIndex extends SolrIndex
{
protected function getStoredDefault() { protected function getStoredDefault()
// Override isDev defaulting to stored {
return 'false'; // Override isDev defaulting to stored
} return 'false';
}
function init() { public function init()
$this->addClass('SearchUpdaterTest_Container'); {
$this->addAllFulltextFields(); $this->addClass('SearchUpdaterTest_Container');
$this->setFieldBoosting('SearchUpdaterTest_Container_Field1', 1.5); $this->addAllFulltextFields();
$this->addBoostedField('Field2', null, array(), 2.1); $this->setFieldBoosting('SearchUpdaterTest_Container_Field1', 1.5);
} $this->addBoostedField('Field2', null, array(), 2.1);
}
} }

314
tests/SolrIndexVersionedTest.php
View File

@ -1,165 +1,183 @@
<?php <?php
if (class_exists('Phockito')) Phockito::include_hamcrest(); if (class_exists('Phockito')) {
Phockito::include_hamcrest();
}
class SolrIndexVersionedTest extends SapphireTest { class SolrIndexVersionedTest extends SapphireTest
{
protected $oldMode = null; protected $oldMode = null;
protected static $index = null; protected static $index = null;
protected $extraDataObjects = array( protected $extraDataObjects = array(
'SearchVariantVersionedTest_Item' 'SearchVariantVersionedTest_Item'
); );
public function setUp() { public function setUp()
{
parent::setUp();
if (!class_exists('Phockito')) {
$this->skipTest = true;
return $this->markTestSkipped("These tests need the Phockito module installed to run");
}
parent::setUp(); // Check versioned available
if (!class_exists('Versioned')) {
if (!class_exists('Phockito')) { $this->skipTest = true;
$this->skipTest = true; return $this->markTestSkipped('The versioned decorator is not installed');
return $this->markTestSkipped("These tests need the Phockito module installed to run"); }
}
// Check versioned available if (self::$index === null) {
if(!class_exists('Versioned')) { self::$index = singleton('SolrVersionedTest_Index');
$this->skipTest = true; }
return $this->markTestSkipped('The versioned decorator is not installed');
}
if (self::$index === null) self::$index = singleton('SolrVersionedTest_Index'); SearchUpdater::bind_manipulation_capture();
SearchUpdater::bind_manipulation_capture(); Config::nest();
Config::nest(); Config::inst()->update('Injector', 'SearchUpdateProcessor', array(
'class' => 'SearchUpdateImmediateProcessor'
));
Config::inst()->update('Injector', 'SearchUpdateProcessor', array( FullTextSearch::force_index_list(self::$index);
'class' => 'SearchUpdateImmediateProcessor' SearchUpdater::clear_dirty_indexes();
));
$this->oldMode = Versioned::get_reading_mode();
Versioned::reading_stage('Stage');
}
public function tearDown()
{
Versioned::set_reading_mode($this->oldMode);
Config::unnest();
parent::tearDown();
}
FullTextSearch::force_index_list(self::$index); protected function getServiceMock()
SearchUpdater::clear_dirty_indexes(); {
return Phockito::mock('Solr3Service');
$this->oldMode = Versioned::get_reading_mode(); }
Versioned::reading_stage('Stage');
} protected function getExpectedDocumentId($id, $stage)
{
public function tearDown() { // Prevent subsites from breaking tests
Versioned::set_reading_mode($this->oldMode); $subsites = class_exists('Subsite') ? '"SearchVariantSubsites":"0",' : '';
Config::unnest(); return $id.'-SiteTree-{'.$subsites.'"SearchVariantVersioned":"'.$stage.'"}';
parent::tearDown(); }
}
public function testPublishing()
protected function getServiceMock() { {
return Phockito::mock('Solr3Service');
} // Setup mocks
$serviceMock = $this->getServiceMock();
protected function getExpectedDocumentId($id, $stage) { self::$index->setService($serviceMock);
// Prevent subsites from breaking tests
$subsites = class_exists('Subsite') ? '"SearchVariantSubsites":"0",' : ''; // Check that write updates Stage
return $id.'-SiteTree-{'.$subsites.'"SearchVariantVersioned":"'.$stage.'"}'; Versioned::reading_stage('Stage');
} Phockito::reset($serviceMock);
$item = new SearchVariantVersionedTest_Item(array('Title' => 'Foo'));
public function testPublishing() { $item->write();
SearchUpdater::flush_dirty_indexes();
// Setup mocks $doc = new SolrDocumentMatcher(array(
$serviceMock = $this->getServiceMock(); '_documentid' => $this->getExpectedDocumentId($item->ID, 'Stage'),
self::$index->setService($serviceMock); 'ClassName' => 'SearchVariantVersionedTest_Item'
));
// Check that write updates Stage Phockito::verify($serviceMock)->addDocument($doc);
Versioned::reading_stage('Stage');
Phockito::reset($serviceMock); // Check that write updates Live
$item = new SearchVariantVersionedTest_Item(array('Title' => 'Foo')); Versioned::reading_stage('Stage');
$item->write(); Phockito::reset($serviceMock);
SearchUpdater::flush_dirty_indexes(); $item = new SearchVariantVersionedTest_Item(array('Title' => 'Bar'));
$doc = new SolrDocumentMatcher(array( $item->write();
'_documentid' => $this->getExpectedDocumentId($item->ID, 'Stage'), $item->publish('Stage', 'Live');
'ClassName' => 'SearchVariantVersionedTest_Item' SearchUpdater::flush_dirty_indexes();
)); $doc = new SolrDocumentMatcher(array(
Phockito::verify($serviceMock)->addDocument($doc); '_documentid' => $this->getExpectedDocumentId($item->ID, 'Live'),
'ClassName' => 'SearchVariantVersionedTest_Item'
// Check that write updates Live ));
Versioned::reading_stage('Stage'); Phockito::verify($serviceMock)->addDocument($doc);
Phockito::reset($serviceMock); }
$item = new SearchVariantVersionedTest_Item(array('Title' => 'Bar'));
$item->write(); public function testDelete()
$item->publish('Stage', 'Live'); {
SearchUpdater::flush_dirty_indexes();
$doc = new SolrDocumentMatcher(array( // Setup mocks
'_documentid' => $this->getExpectedDocumentId($item->ID, 'Live'), $serviceMock = $this->getServiceMock();
'ClassName' => 'SearchVariantVersionedTest_Item' self::$index->setService($serviceMock);
));
Phockito::verify($serviceMock)->addDocument($doc); // Delete the live record (not the stage)
} Versioned::reading_stage('Stage');
Phockito::reset($serviceMock);
public function testDelete() { $item = new SearchVariantVersionedTest_Item(array('Title' => 'Too'));
$item->write();
// Setup mocks $item->publish('Stage', 'Live');
$serviceMock = $this->getServiceMock(); Versioned::reading_stage('Live');
self::$index->setService($serviceMock); $id = $item->ID;
$item->delete();
// Delete the live record (not the stage) SearchUpdater::flush_dirty_indexes();
Versioned::reading_stage('Stage'); Phockito::verify($serviceMock, 1)
Phockito::reset($serviceMock); ->deleteById($this->getExpectedDocumentId($id, 'Live'));
$item = new SearchVariantVersionedTest_Item(array('Title' => 'Too')); Phockito::verify($serviceMock, 0)
$item->write(); ->deleteById($this->getExpectedDocumentId($id, 'Stage'));
$item->publish('Stage', 'Live');
Versioned::reading_stage('Live'); // Delete the stage record
$id = $item->ID; Versioned::reading_stage('Stage');
$item->delete(); Phockito::reset($serviceMock);
SearchUpdater::flush_dirty_indexes(); $item = new SearchVariantVersionedTest_Item(array('Title' => 'Too'));
Phockito::verify($serviceMock, 1) $item->write();
->deleteById($this->getExpectedDocumentId($id, 'Live')); $item->publish('Stage', 'Live');
Phockito::verify($serviceMock, 0) $id = $item->ID;
->deleteById($this->getExpectedDocumentId($id, 'Stage')); $item->delete();
SearchUpdater::flush_dirty_indexes();
// Delete the stage record Phockito::verify($serviceMock, 1)
Versioned::reading_stage('Stage'); ->deleteById($this->getExpectedDocumentId($id, 'Stage'));
Phockito::reset($serviceMock); Phockito::verify($serviceMock, 0)
$item = new SearchVariantVersionedTest_Item(array('Title' => 'Too')); ->deleteById($this->getExpectedDocumentId($id, 'Live'));
$item->write(); }
$item->publish('Stage', 'Live');
$id = $item->ID;
$item->delete();
SearchUpdater::flush_dirty_indexes();
Phockito::verify($serviceMock, 1)
->deleteById($this->getExpectedDocumentId($id, 'Stage'));
Phockito::verify($serviceMock, 0)
->deleteById($this->getExpectedDocumentId($id, 'Live'));
}
} }
class SolrVersionedTest_Index extends SolrIndex { class SolrVersionedTest_Index extends SolrIndex
function init() { {
$this->addClass('SearchVariantVersionedTest_Item'); public function init()
$this->addFilterField('TestText'); {
} $this->addClass('SearchVariantVersionedTest_Item');
$this->addFilterField('TestText');
}
} }
if (!class_exists('Phockito')) return; if (!class_exists('Phockito')) {
return;
class SolrDocumentMatcher extends Hamcrest_BaseMatcher { }
protected $properties; class SolrDocumentMatcher extends Hamcrest_BaseMatcher
{
public function __construct($properties) { protected $properties;
$this->properties = $properties;
} public function __construct($properties)
{
public function describeTo(\Hamcrest_Description $description) { $this->properties = $properties;
$description->appendText('Apache_Solr_Document with properties '.var_export($this->properties, true)); }
}
public function describeTo(\Hamcrest_Description $description)
public function matches($item) { {
$description->appendText('Apache_Solr_Document with properties '.var_export($this->properties, true));
if(! ($item instanceof Apache_Solr_Document)) return false; }
foreach($this->properties as $key => $value) { public function matches($item)
if($item->{$key} != $value) return false; {
} if (! ($item instanceof Apache_Solr_Document)) {
return false;
return true; }
}
foreach ($this->properties as $key => $value) {
if ($item->{$key} != $value) {
return false;
}
}
return true;
}
} }

370
tests/SolrReindexQueuedTest.php
View File

@ -3,217 +3,227 @@
/** /**
* Additional tests of solr reindexing processes when run with queuedjobs * Additional tests of solr reindexing processes when run with queuedjobs
*/ */
class SolrReindexQueuedTest extends SapphireTest { class SolrReindexQueuedTest extends SapphireTest
{
protected $usesDatabase = true;
protected $usesDatabase = true; protected $extraDataObjects = array(
'SolrReindexTest_Item'
);
protected $extraDataObjects = array( /**
'SolrReindexTest_Item' * Forced index for testing
); *
* @var SolrReindexTest_Index
*/
protected $index = null;
/** /**
* Forced index for testing * Mock service
* *
* @var SolrReindexTest_Index * @var SolrService
*/ */
protected $index = null; protected $service = null;
/** public function setUp()
* Mock service {
* parent::setUp();
* @var SolrService
*/
protected $service = null;
public function setUp() { if (!class_exists('Phockito')) {
parent::setUp(); $this->skipTest = true;
return $this->markTestSkipped("These tests need the Phockito module installed to run");
}
if (!class_exists('Phockito')) { if (!interface_exists('QueuedJob')) {
$this->skipTest = true; $this->skipTest = true;
return $this->markTestSkipped("These tests need the Phockito module installed to run"); return $this->markTestSkipped("These tests need the QueuedJobs module installed to run");
} }
if(!interface_exists('QueuedJob')) { // Set queued handler for reindex
$this->skipTest = true; Config::inst()->update('Injector', 'SolrReindexHandler', array(
return $this->markTestSkipped("These tests need the QueuedJobs module installed to run"); 'class' => 'SolrReindexQueuedHandler'
} ));
Injector::inst()->registerService(new SolrReindexQueuedHandler(), 'SolrReindexHandler');
// Set queued handler for reindex // Set test variant
Config::inst()->update('Injector', 'SolrReindexHandler', array( SolrReindexTest_Variant::enable();
'class' => 'SolrReindexQueuedHandler'
));
Injector::inst()->registerService(new SolrReindexQueuedHandler(), 'SolrReindexHandler');
// Set test variant // Set index list
SolrReindexTest_Variant::enable(); $this->service = $this->getServiceMock();
$this->index = singleton('SolrReindexTest_Index');
$this->index->setService($this->service);
FullTextSearch::force_index_list($this->index);
}
// Set index list /**
$this->service = $this->getServiceMock(); * Populate database with dummy dataset
$this->index = singleton('SolrReindexTest_Index'); *
$this->index->setService($this->service); * @param int $number Number of records to create in each variant
FullTextSearch::force_index_list($this->index); */
} protected function createDummyData($number)
{
// Populate dataobjects. Use truncate to generate predictable IDs
DB::query('TRUNCATE "SolrReindexTest_Item"');
/** // Note that we don't create any records in variant = 2, to represent a variant
* Populate database with dummy dataset // that should be cleared without any re-indexes performed
* foreach (array(0, 1) as $variant) {
* @param int $number Number of records to create in each variant for ($i = 1; $i <= $number; $i++) {
*/ $item = new SolrReindexTest_Item();
protected function createDummyData($number) { $item->Variant = $variant;
// Populate dataobjects. Use truncate to generate predictable IDs $item->Title = "Item $variant / $i";
DB::query('TRUNCATE "SolrReindexTest_Item"'); $item->write();
}
}
}
// Note that we don't create any records in variant = 2, to represent a variant /**
// that should be cleared without any re-indexes performed * Mock service
foreach(array(0, 1) as $variant) { *
for($i = 1; $i <= $number; $i++) { * @return SolrService
$item = new SolrReindexTest_Item(); */
$item->Variant = $variant; protected function getServiceMock()
$item->Title = "Item $variant / $i"; {
$item->write(); return Phockito::mock('Solr4Service');
} }
}
}
/** public function tearDown()
* Mock service {
* FullTextSearch::force_index_list();
* @return SolrService SolrReindexTest_Variant::disable();
*/ parent::tearDown();
protected function getServiceMock() { }
return Phockito::mock('Solr4Service');
}
public function tearDown() { /**
FullTextSearch::force_index_list(); * Get the reindex handler
SolrReindexTest_Variant::disable(); *
parent::tearDown(); * @return SolrReindexHandler
} */
protected function getHandler()
{
return Injector::inst()->get('SolrReindexHandler');
}
/** /**
* Get the reindex handler * @return SolrReindexQueuedTest_Service
* */
* @return SolrReindexHandler protected function getQueuedJobService()
*/ {
protected function getHandler() { return singleton('SolrReindexQueuedTest_Service');
return Injector::inst()->get('SolrReindexHandler'); }
}
/** /**
* @return SolrReindexQueuedTest_Service * Test that reindex will generate a top top level queued job, and executing this will perform
*/ * the necessary initialisation of the grouped queued jobs
protected function getQueuedJobService() { */
return singleton('SolrReindexQueuedTest_Service'); public function testReindexSegmentsGroups()
} {
$this->createDummyData(18);
/** // Create pre-existing jobs
* Test that reindex will generate a top top level queued job, and executing this will perform $this->getQueuedJobService()->queueJob(new SolrReindexQueuedJob());
* the necessary initialisation of the grouped queued jobs $this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob());
*/ $this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob());
public function testReindexSegmentsGroups() {
$this->createDummyData(18);
// Create pre-existing jobs // Initiate re-index
$this->getQueuedJobService()->queueJob(new SolrReindexQueuedJob()); $logger = new SolrReindexTest_RecordingLogger();
$this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob()); $this->getHandler()->triggerReindex($logger, 6, 'Solr_Reindex');
$this->getQueuedJobService()->queueJob(new SolrReindexGroupQueuedJob());
// Initiate re-index // Old jobs should be cancelled
$logger = new SolrReindexTest_RecordingLogger(); $this->assertEquals(1, $logger->countMessages('Cancelled 1 re-index tasks and 2 re-index groups'));
$this->getHandler()->triggerReindex($logger, 6, 'Solr_Reindex'); $this->assertEquals(1, $logger->countMessages('Queued Solr Reindex Job'));
// Old jobs should be cancelled // Next job should be queue job
$this->assertEquals(1, $logger->countMessages('Cancelled 1 re-index tasks and 2 re-index groups')); $job = $this->getQueuedJobService()->getNextJob();
$this->assertEquals(1, $logger->countMessages('Queued Solr Reindex Job')); $this->assertInstanceOf('SolrReindexQueuedJob', $job);
$this->assertEquals(6, $job->getBatchSize());
// Next job should be queue job // Test that necessary items are created
$job = $this->getQueuedJobService()->getNextJob(); $logger->clear();
$this->assertInstanceOf('SolrReindexQueuedJob', $job); $job->setLogger($logger);
$this->assertEquals(6, $job->getBatchSize()); $job->process();
// Test that necessary items are created // Deletes are performed in the main task prior to individual groups being processed
$logger->clear(); // 18 records means 3 groups of 6 in each variant (6 total)
$job->setLogger($logger); Phockito::verify($this->service, 2)
$job->process(); ->deleteByQuery(anything());
$this->assertEquals(1, $logger->countMessages('Beginning init of reindex'));
$this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group '));
$this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}'));
$this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"1"}'));
$this->assertEquals(1, $logger->countMessages('Completed init of reindex'));
// Deletes are performed in the main task prior to individual groups being processed
// 18 records means 3 groups of 6 in each variant (6 total) // Test that invalid classes are removed
Phockito::verify($this->service, 2) $this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index'));
->deleteByQuery(anything()); Phockito::verify($this->service, 1)
$this->assertEquals(1, $logger->countMessages('Beginning init of reindex')); ->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)');
$this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group '));
$this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}'));
$this->assertEquals(3, $logger->countMessages(' of SolrReindexTest_Item in {"SolrReindexTest_Variant":"1"}'));
$this->assertEquals(1, $logger->countMessages('Completed init of reindex'));
// Test that valid classes in invalid variants are removed
// Test that invalid classes are removed $this->assertNotEmpty($logger->getMessages(
$this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index')); 'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}'
Phockito::verify($this->service, 1) ));
->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)'); Phockito::verify($this->service, 1)
->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")');
}
// Test that valid classes in invalid variants are removed /**
$this->assertNotEmpty($logger->getMessages( * Test index processing on individual groups
'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}' */
)); public function testRunGroup()
Phockito::verify($this->service, 1) {
->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")'); $this->createDummyData(18);
}
/** // Just do what the SolrReindexQueuedJob would do to create each sub
* Test index processing on individual groups $logger = new SolrReindexTest_RecordingLogger();
*/ $this->getHandler()->runReindex($logger, 6, 'Solr_Reindex');
public function testRunGroup() {
$this->createDummyData(18);
// Just do what the SolrReindexQueuedJob would do to create each sub // Assert jobs are created
$logger = new SolrReindexTest_RecordingLogger(); $this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group'));
$this->getHandler()->runReindex($logger, 6, 'Solr_Reindex');
// Assert jobs are created // Check next job is a group queued job
$this->assertEquals(6, $logger->countMessages('Queued Solr Reindex Group')); $job = $this->getQueuedJobService()->getNextJob();
$this->assertInstanceOf('SolrReindexGroupQueuedJob', $job);
$this->assertEquals(
'Solr Reindex Group (1/3) of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}',
$job->getTitle()
);
// Check next job is a group queued job // Running this job performs the necessary reindex
$job = $this->getQueuedJobService()->getNextJob(); $logger->clear();
$this->assertInstanceOf('SolrReindexGroupQueuedJob', $job); $job->setLogger($logger);
$this->assertEquals( $job->process();
'Solr Reindex Group (1/3) of SolrReindexTest_Item in {"SolrReindexTest_Variant":"0"}',
$job->getTitle()
);
// Running this job performs the necessary reindex // Check tasks completed (as per non-queuedjob version)
$logger->clear(); $this->assertEquals(1, $logger->countMessages('Beginning reindex group'));
$job->setLogger($logger); $this->assertEquals(1, $logger->countMessages('Adding SolrReindexTest_Item'));
$job->process(); $this->assertEquals(1, $logger->countMessages('Queuing commit on all changes'));
$this->assertEquals(1, $logger->countMessages('Completed reindex group'));
// Check tasks completed (as per non-queuedjob version) // Check IDs
$this->assertEquals(1, $logger->countMessages('Beginning reindex group')); $idMessage = $logger->filterMessages('Updated ');
$this->assertEquals(1, $logger->countMessages('Adding SolrReindexTest_Item')); $this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/i', $idMessage[0], $matches));
$this->assertEquals(1, $logger->countMessages('Queuing commit on all changes')); $ids = array_unique(explode(',', $matches['ids']));
$this->assertEquals(1, $logger->countMessages('Completed reindex group')); $this->assertEquals(6, count($ids));
foreach ($ids as $id) {
// Check IDs // Each id should be % 3 == 0
$idMessage = $logger->filterMessages('Updated '); $this->assertEquals(0, $id % 3, "ID $id Should match pattern ID % 3 = 0");
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/i', $idMessage[0], $matches)); }
$ids = array_unique(explode(',', $matches['ids'])); }
$this->assertEquals(6, count($ids));
foreach($ids as $id) {
// Each id should be % 3 == 0
$this->assertEquals(0, $id % 3, "ID $id Should match pattern ID % 3 = 0");
}
}
} }
if(!class_exists('QueuedJobService')) return; if (!class_exists('QueuedJobService')) {
return;
}
class SolrReindexQueuedTest_Service extends QueuedJobService implements TestOnly { class SolrReindexQueuedTest_Service extends QueuedJobService implements TestOnly
{
/** /**
* @return QueuedJob * @return QueuedJob
*/ */
public function getNextJob() { public function getNextJob()
$job = $this->getNextPendingJob(); {
return $this->initialiseJob($job); $job = $this->getNextPendingJob();
} return $this->initialiseJob($job);
}
} }

893
tests/SolrReindexTest.php
View File

@ -5,335 +5,348 @@ use Monolog\Handler\HandlerInterface;
use Monolog\Logger; use Monolog\Logger;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
if (class_exists('Phockito')) Phockito::include_hamcrest(); if (class_exists('Phockito')) {
Phockito::include_hamcrest();
}
class SolrReindexTest extends SapphireTest { class SolrReindexTest extends SapphireTest
{
protected $usesDatabase = true;
protected $usesDatabase = true; protected $extraDataObjects = array(
'SolrReindexTest_Item'
);
protected $extraDataObjects = array( /**
'SolrReindexTest_Item' * Forced index for testing
); *
* @var SolrReindexTest_Index
*/
protected $index = null;
/** /**
* Forced index for testing * Mock service
* *
* @var SolrReindexTest_Index * @var SolrService
*/ */
protected $index = null; protected $service = null;
/** public function setUp()
* Mock service {
* parent::setUp();
* @var SolrService
*/
protected $service = null;
public function setUp() { if (!class_exists('Phockito')) {
parent::setUp(); $this->skipTest = true;
return $this->markTestSkipped("These tests need the Phockito module installed to run");
}
// Set test handler for reindex
Config::inst()->update('Injector', 'SolrReindexHandler', array(
'class' => 'SolrReindexTest_TestHandler'
));
Injector::inst()->registerService(new SolrReindexTest_TestHandler(), 'SolrReindexHandler');
if (!class_exists('Phockito')) { // Set test variant
$this->skipTest = true; SolrReindexTest_Variant::enable();
return $this->markTestSkipped("These tests need the Phockito module installed to run");
}
// Set test handler for reindex
Config::inst()->update('Injector', 'SolrReindexHandler', array(
'class' => 'SolrReindexTest_TestHandler'
));
Injector::inst()->registerService(new SolrReindexTest_TestHandler(), 'SolrReindexHandler');
// Set test variant // Set index list
SolrReindexTest_Variant::enable(); $this->service = $this->getServiceMock();
$this->index = singleton('SolrReindexTest_Index');
$this->index->setService($this->service);
FullTextSearch::force_index_list($this->index);
}
// Set index list /**
$this->service = $this->getServiceMock(); * Populate database with dummy dataset
$this->index = singleton('SolrReindexTest_Index'); *
$this->index->setService($this->service); * @param int $number Number of records to create in each variant
FullTextSearch::force_index_list($this->index); */
} protected function createDummyData($number)
{
// Populate dataobjects. Use truncate to generate predictable IDs
DB::query('TRUNCATE "SolrReindexTest_Item"');
/** // Note that we don't create any records in variant = 2, to represent a variant
* Populate database with dummy dataset // that should be cleared without any re-indexes performed
* foreach (array(0, 1) as $variant) {
* @param int $number Number of records to create in each variant for ($i = 1; $i <= $number; $i++) {
*/ $item = new SolrReindexTest_Item();
protected function createDummyData($number) { $item->Variant = $variant;
// Populate dataobjects. Use truncate to generate predictable IDs $item->Title = "Item $variant / $i";
DB::query('TRUNCATE "SolrReindexTest_Item"'); $item->write();
}
}
}
// Note that we don't create any records in variant = 2, to represent a variant /**
// that should be cleared without any re-indexes performed * Mock service
foreach(array(0, 1) as $variant) { *
for($i = 1; $i <= $number; $i++) { * @return SolrService
$item = new SolrReindexTest_Item(); */
$item->Variant = $variant; protected function getServiceMock()
$item->Title = "Item $variant / $i"; {
$item->write(); return Phockito::mock('Solr4Service');
} }
}
}
/** public function tearDown()
* Mock service {
* FullTextSearch::force_index_list();
* @return SolrService SolrReindexTest_Variant::disable();
*/ parent::tearDown();
protected function getServiceMock() { }
return Phockito::mock('Solr4Service');
}
public function tearDown() { /**
FullTextSearch::force_index_list(); * Get the reindex handler
SolrReindexTest_Variant::disable(); *
parent::tearDown(); * @return SolrReindexHandler
} */
protected function getHandler()
{
return Injector::inst()->get('SolrReindexHandler');
}
/** /**
* Get the reindex handler * Ensure the test variant is up and running properly
* */
* @return SolrReindexHandler public function testVariant()
*/ {
protected function getHandler() { // State defaults to 0
return Injector::inst()->get('SolrReindexHandler'); $variant = SearchVariant::current_state();
} $this->assertEquals(
array(
"SolrReindexTest_Variant" => "0"
),
$variant
);
/** // All states enumerated
* Ensure the test variant is up and running properly $allStates = iterator_to_array(SearchVariant::reindex_states());
*/ $this->assertEquals(
public function testVariant() { array(
// State defaults to 0 array(
$variant = SearchVariant::current_state(); "SolrReindexTest_Variant" => "0"
$this->assertEquals( ),
array( array(
"SolrReindexTest_Variant" => "0" "SolrReindexTest_Variant" => "1"
), ),
$variant array(
); "SolrReindexTest_Variant" => "2"
)
),
$allStates
);
// All states enumerated // Check correct items created and that filtering on variant works
$allStates = iterator_to_array(SearchVariant::reindex_states()); $this->createDummyData(120);
$this->assertEquals( SolrReindexTest_Variant::set_current(2);
array( $this->assertEquals(0, SolrReindexTest_Item::get()->count());
array( SolrReindexTest_Variant::set_current(1);
"SolrReindexTest_Variant" => "0" $this->assertEquals(120, SolrReindexTest_Item::get()->count());
), SolrReindexTest_Variant::set_current(0);
array( $this->assertEquals(120, SolrReindexTest_Item::get()->count());
"SolrReindexTest_Variant" => "1" SolrReindexTest_Variant::disable();
), $this->assertEquals(240, SolrReindexTest_Item::get()->count());
array( }
"SolrReindexTest_Variant" => "2"
)
),
$allStates
);
// Check correct items created and that filtering on variant works
$this->createDummyData(120);
SolrReindexTest_Variant::set_current(2);
$this->assertEquals(0, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(1);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(0);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::disable();
$this->assertEquals(240, SolrReindexTest_Item::get()->count());
}
/** /**
* Given the invocation of a new re-index with a given set of data, ensure that the necessary * Given the invocation of a new re-index with a given set of data, ensure that the necessary
* list of groups are created and segmented for each state * list of groups are created and segmented for each state
* *
* Test should work fine with any variants (versioned, subsites, etc) specified * Test should work fine with any variants (versioned, subsites, etc) specified
*/ */
public function testReindexSegmentsGroups() { public function testReindexSegmentsGroups()
$this->createDummyData(120); {
$this->createDummyData(120);
// Initiate re-index // Initiate re-index
$logger = new SolrReindexTest_RecordingLogger(); $logger = new SolrReindexTest_RecordingLogger();
$this->getHandler()->runReindex($logger, 21, 'Solr_Reindex'); $this->getHandler()->runReindex($logger, 21, 'Solr_Reindex');
// Test that invalid classes are removed // Test that invalid classes are removed
$this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index')); $this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index'));
Phockito::verify($this->service, 1) Phockito::verify($this->service, 1)
->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)'); ->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)');
// Test that valid classes in invalid variants are removed // Test that valid classes in invalid variants are removed
$this->assertNotEmpty($logger->getMessages( $this->assertNotEmpty($logger->getMessages(
'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}' 'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}'
)); ));
Phockito::verify($this->service, 1) Phockito::verify($this->service, 1)
->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")'); ->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")');
// 120x2 grouped into groups of 21 results in 12 groups // 120x2 grouped into groups of 21 results in 12 groups
$this->assertEquals(12, $logger->countMessages('Called processGroup with ')); $this->assertEquals(12, $logger->countMessages('Called processGroup with '));
$this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"0"}')); $this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"0"}'));
$this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"1"}')); $this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"1"}'));
// Given that there are two variants, there should be two group ids of each number // Given that there are two variants, there should be two group ids of each number
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 0 of 6')); $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 0 of 6'));
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 1 of 6')); $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 1 of 6'));
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 2 of 6')); $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 2 of 6'));
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 3 of 6')); $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 3 of 6'));
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 4 of 6')); $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 4 of 6'));
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 5 of 6')); $this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 5 of 6'));
// Check various group sizes // Check various group sizes
$logger->clear(); $logger->clear();
$this->getHandler()->runReindex($logger, 120, 'Solr_Reindex'); $this->getHandler()->runReindex($logger, 120, 'Solr_Reindex');
$this->assertEquals(2, $logger->countMessages('Called processGroup with ')); $this->assertEquals(2, $logger->countMessages('Called processGroup with '));
$logger->clear(); $logger->clear();
$this->getHandler()->runReindex($logger, 119, 'Solr_Reindex'); $this->getHandler()->runReindex($logger, 119, 'Solr_Reindex');
$this->assertEquals(4, $logger->countMessages('Called processGroup with ')); $this->assertEquals(4, $logger->countMessages('Called processGroup with '));
$logger->clear(); $logger->clear();
$this->getHandler()->runReindex($logger, 121, 'Solr_Reindex'); $this->getHandler()->runReindex($logger, 121, 'Solr_Reindex');
$this->assertEquals(2, $logger->countMessages('Called processGroup with ')); $this->assertEquals(2, $logger->countMessages('Called processGroup with '));
$logger->clear(); $logger->clear();
$this->getHandler()->runReindex($logger, 2, 'Solr_Reindex'); $this->getHandler()->runReindex($logger, 2, 'Solr_Reindex');
$this->assertEquals(120, $logger->countMessages('Called processGroup with ')); $this->assertEquals(120, $logger->countMessages('Called processGroup with '));
} }
/** /**
* Test index processing on individual groups * Test index processing on individual groups
*/ */
public function testRunGroup() { public function testRunGroup()
$this->createDummyData(120); {
$logger = new SolrReindexTest_RecordingLogger(); $this->createDummyData(120);
$logger = new SolrReindexTest_RecordingLogger();
// Initiate re-index of third group (index 2 of 6) // Initiate re-index of third group (index 2 of 6)
$state = array('SolrReindexTest_Variant' => '1'); $state = array('SolrReindexTest_Variant' => '1');
$this->getHandler()->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, 2); $this->getHandler()->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, 2);
$idMessage = $logger->filterMessages('Updated '); $idMessage = $logger->filterMessages('Updated ');
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/i', $idMessage[0], $matches)); $this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/i', $idMessage[0], $matches));
$ids = array_unique(explode(',', $matches['ids'])); $ids = array_unique(explode(',', $matches['ids']));
// Test successful // Test successful
$this->assertNotEmpty($logger->getMessages('Adding SolrReindexTest_Item')); $this->assertNotEmpty($logger->getMessages('Adding SolrReindexTest_Item'));
$this->assertNotEmpty($logger->getMessages('Done')); $this->assertNotEmpty($logger->getMessages('Done'));
// Test that items in this variant / group are cleared from solr // Test that items in this variant / group are cleared from solr
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")'
); );
// Test that items in this variant / group are re-indexed // Test that items in this variant / group are re-indexed
// 120 divided into 6 groups should be 20 at least (max 21) // 120 divided into 6 groups should be 20 at least (max 21)
$this->assertEquals(21, count($ids), 'Group size is about 20', 1); $this->assertEquals(21, count($ids), 'Group size is about 20', 1);
foreach($ids as $id) { foreach ($ids as $id) {
// Each id should be % 6 == 2 // Each id should be % 6 == 2
$this->assertEquals(2, $id % 6, "ID $id Should match pattern ID % 6 = 2"); $this->assertEquals(2, $id % 6, "ID $id Should match pattern ID % 6 = 2");
} }
} }
/** /**
* Test that running all groups covers the entire range of dataobject IDs * Test that running all groups covers the entire range of dataobject IDs
*/ */
public function testRunAllGroups() { public function testRunAllGroups()
$this->createDummyData(120); {
$logger = new SolrReindexTest_RecordingLogger(); $this->createDummyData(120);
$logger = new SolrReindexTest_RecordingLogger();
// Test that running all groups covers the complete set of ids // Test that running all groups covers the complete set of ids
$state = array('SolrReindexTest_Variant' => '1'); $state = array('SolrReindexTest_Variant' => '1');
for($i = 0; $i < 6; $i++) { for ($i = 0; $i < 6; $i++) {
// See testReindexSegmentsGroups for test that each of these states is invoked during a full reindex // See testReindexSegmentsGroups for test that each of these states is invoked during a full reindex
$this $this
->getHandler() ->getHandler()
->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, $i); ->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, $i);
} }
// Count all ids updated // Count all ids updated
$ids = array(); $ids = array();
foreach($logger->filterMessages('Updated ') as $message) { foreach ($logger->filterMessages('Updated ') as $message) {
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/', $message, $matches)); $this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/', $message, $matches));
$ids = array_unique(array_merge($ids, explode(',', $matches['ids']))); $ids = array_unique(array_merge($ids, explode(',', $matches['ids'])));
} }
// Check ids // Check ids
$this->assertEquals(120, count($ids)); $this->assertEquals(120, count($ids));
Phockito::verify($this->service, 6)->deleteByQuery(anything()); Phockito::verify($this->service, 6)->deleteByQuery(anything());
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=0 u=0}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=0 u=0}mod(ID, 6)" +(_testvariant:"1")'
); );
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=1 u=1}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=1 u=1}mod(ID, 6)" +(_testvariant:"1")'
); );
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")'
); );
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=3 u=3}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=3 u=3}mod(ID, 6)" +(_testvariant:"1")'
); );
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=4 u=4}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=4 u=4}mod(ID, 6)" +(_testvariant:"1")'
); );
Phockito::verify($this->service, 1)->deleteByQuery( Phockito::verify($this->service, 1)->deleteByQuery(
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=5 u=5}mod(ID, 6)" +(_testvariant:"1")' '+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=5 u=5}mod(ID, 6)" +(_testvariant:"1")'
); );
} }
} }
/** /**
* Provides a wrapper for testing SolrReindexBase * Provides a wrapper for testing SolrReindexBase
*/ */
class SolrReindexTest_TestHandler extends SolrReindexBase { class SolrReindexTest_TestHandler extends SolrReindexBase
{
public function processGroup( public function processGroup(
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName
) { ) {
$indexName = $indexInstance->getIndexName(); $indexName = $indexInstance->getIndexName();
$stateName = json_encode($state); $stateName = json_encode($state);
$logger->info("Called processGroup with {$indexName}, {$stateName}, {$class}, group {$group} of {$groups}"); $logger->info("Called processGroup with {$indexName}, {$stateName}, {$class}, group {$group} of {$groups}");
} }
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null) {
$logger->info("Called triggerReindex");
}
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null)
{
$logger->info("Called triggerReindex");
}
} }
class SolrReindexTest_Index extends SolrIndex implements TestOnly { class SolrReindexTest_Index extends SolrIndex implements TestOnly
public function init() { {
$this->addClass('SolrReindexTest_Item'); public function init()
$this->addAllFulltextFields(); {
} $this->addClass('SolrReindexTest_Item');
$this->addAllFulltextFields();
}
} }
/** /**
* Does not have any variant extensions * Does not have any variant extensions
*/ */
class SolrReindexTest_Item extends DataObject implements TestOnly { class SolrReindexTest_Item extends DataObject implements TestOnly
{
private static $extensions = array( private static $extensions = array(
'SolrReindexTest_ItemExtension' 'SolrReindexTest_ItemExtension'
); );
private static $db = array(
'Title' => 'Varchar(255)',
'Variant' => 'Int(0)'
);
private static $db = array(
'Title' => 'Varchar(255)',
'Variant' => 'Int(0)'
);
} }
/** /**
* Select only records in the current variant * Select only records in the current variant
*/ */
class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly { class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly
{
/** /**
* Filter records on the current variant * Filter records on the current variant
* *
* @param SQLQuery $query * @param SQLQuery $query
* @param DataQuery $dataQuery * @param DataQuery $dataQuery
*/ */
public function augmentSQL(SQLQuery &$query, DataQuery &$dataQuery = null) { public function augmentSQL(SQLQuery &$query, DataQuery &$dataQuery = null)
$variant = SolrReindexTest_Variant::get_current(); {
if($variant !== null && !$query->filtersOnID()) { $variant = SolrReindexTest_Variant::get_current();
$sqlVariant = Convert::raw2sql($variant); if ($variant !== null && !$query->filtersOnID()) {
$query->addWhere("\"Variant\" = '{$sqlVariant}'"); $sqlVariant = Convert::raw2sql($variant);
} $query->addWhere("\"Variant\" = '{$sqlVariant}'");
} }
}
} }
@ -342,190 +355,208 @@ class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly {
* *
* Variant states are 0 and 1, or null if disabled * Variant states are 0 and 1, or null if disabled
*/ */
class SolrReindexTest_Variant extends SearchVariant implements TestOnly { class SolrReindexTest_Variant extends SearchVariant implements TestOnly
{
/**
* Value of this variant (either null, 0, or 1)
*
* @var int|null
*/
protected static $current = null;
/** /**
* Value of this variant (either null, 0, or 1) * Activate this variant
* */
* @var int|null public static function enable()
*/ {
protected static $current = null; self::disable();
/** self::$current = 0;
* Activate this variant self::$variants = array(
*/ 'SolrReindexTest_Variant' => singleton('SolrReindexTest_Variant')
public static function enable() { );
self::disable(); }
self::$current = 0; /**
self::$variants = array( * Disable this variant and reset
'SolrReindexTest_Variant' => singleton('SolrReindexTest_Variant') */
); public static function disable()
} {
self::$current = null;
self::$variants = null;
self::$class_variants = array();
self::$call_instances = array();
}
/** public function activateState($state)
* Disable this variant and reset {
*/ self::set_current($state);
public static function disable() { }
self::$current = null;
self::$variants = null;
self::$class_variants = array();
self::$call_instances = array();
}
public function activateState($state) { /**
self::set_current($state); * Set the current variant to the given state
} *
* @param int $current 0, 1, 2, or null (disabled)
*/
public static function set_current($current)
{
self::$current = $current;
}
/** /**
* Set the current variant to the given state * Get the current state
* *
* @param int $current 0, 1, 2, or null (disabled) * @return string|null
*/ */
public static function set_current($current) { public static function get_current()
self::$current = $current; {
} // Always use string values for states for consistent json_encode value
if (isset(self::$current)) {
return (string)self::$current;
}
}
/** public function alterDefinition($base, $index)
* Get the current state {
* $self = get_class($this);
* @return string|null
*/
public static function get_current() {
// Always use string values for states for consistent json_encode value
if(isset(self::$current)) {
return (string)self::$current;
}
}
function alterDefinition($base, $index) { $index->filterFields['_testvariant'] = array(
$self = get_class($this); 'name' => '_testvariant',
'field' => '_testvariant',
'fullfield' => '_testvariant',
'base' => $base,
'origin' => $base,
'type' => 'Int',
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
$index->filterFields['_testvariant'] = array( public function alterQuery($query, $index)
'name' => '_testvariant', {
'field' => '_testvariant', // I guess just calling it _testvariant is ok?
'fullfield' => '_testvariant', $query->filter('_testvariant', $this->currentState());
'base' => $base, }
'origin' => $base,
'type' => 'Int',
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
);
}
public function alterQuery($query, $index) { public function appliesTo($class, $includeSubclasses)
// I guess just calling it _testvariant is ok? {
$query->filter('_testvariant', $this->currentState()); return $class === 'SolrReindexTest_Item' ||
} ($includeSubclasses && is_subclass_of($class, 'SolrReindexTest_Item', true));
}
public function appliesTo($class, $includeSubclasses) { public function appliesToEnvironment()
return $class === 'SolrReindexTest_Item' || {
($includeSubclasses && is_subclass_of($class, 'SolrReindexTest_Item', true)); // Set to null to disable
} return self::$current !== null;
}
public function appliesToEnvironment() { public function currentState()
// Set to null to disable {
return self::$current !== null; return self::get_current();
} }
public function currentState() {
return self::get_current();
}
public function reindexStates() {
// Always use string values for states for consistent json_encode value
return array('0', '1', '2');
}
public function reindexStates()
{
// Always use string values for states for consistent json_encode value
return array('0', '1', '2');
}
} }
/** /**
* Test logger for recording messages * Test logger for recording messages
*/ */
class SolrReindexTest_RecordingLogger extends Logger implements TestOnly { class SolrReindexTest_RecordingLogger extends Logger implements TestOnly
{
/**
* @var SolrReindexTest_Handler
*/
protected $testHandler = null;
/** public function __construct($name = 'testlogger', array $handlers = array(), array $processors = array())
* @var SolrReindexTest_Handler {
*/ parent::__construct($name, $handlers, $processors);
protected $testHandler = null;
public function __construct($name = 'testlogger', array $handlers = array(), array $processors = array()) { $this->testHandler = new SolrReindexTest_Handler();
parent::__construct($name, $handlers, $processors); $this->pushHandler($this->testHandler);
}
$this->testHandler = new SolrReindexTest_Handler(); /**
$this->pushHandler($this->testHandler); * @return array
} */
public function getMessages()
{
return $this->testHandler->getMessages();
}
/** /**
* @return array * Clear all messages
*/ */
public function getMessages() { public function clear()
return $this->testHandler->getMessages(); {
} $this->testHandler->clear();
}
/** /**
* Clear all messages * Get messages with the given filter
*/ *
public function clear() { * @param string $containing
$this->testHandler->clear(); * @return array Filtered array
} */
public function filterMessages($containing)
{
return array_values(array_filter(
$this->getMessages(),
function ($content) use ($containing) {
return stripos($content, $containing) !== false;
}
));
}
/** /**
* Get messages with the given filter * Count all messages containing the given substring
* *
* @param string $containing * @param string $containing Message to filter by
* @return array Filtered array * @return int
*/ */
public function filterMessages($containing) { public function countMessages($containing = null)
return array_values(array_filter( {
$this->getMessages(), if ($containing) {
function($content) use ($containing) { $messages = $this->filterMessages($containing);
return stripos($content, $containing) !== false; } else {
} $messages = $this->getMessages();
)); }
} return count($messages);
}
/**
* Count all messages containing the given substring
*
* @param string $containing Message to filter by
* @return int
*/
public function countMessages($containing = null) {
if($containing) {
$messages = $this->filterMessages($containing);
} else {
$messages = $this->getMessages();
}
return count($messages);
}
} }
/** /**
* Logger for recording messages for later retrieval * Logger for recording messages for later retrieval
*/ */
class SolrReindexTest_Handler extends AbstractProcessingHandler implements TestOnly { class SolrReindexTest_Handler extends AbstractProcessingHandler implements TestOnly
{
/**
* Messages
*
* @var array
*/
protected $messages = array();
/** /**
* Messages * Get all messages
* *
* @var array * @return array
*/ */
protected $messages = array(); public function getMessages()
{
return $this->messages;
}
/** public function clear()
* Get all messages {
* $this->messages = array();
* @return array }
*/
public function getMessages() {
return $this->messages;
}
public function clear() { protected function write(array $record)
$this->messages = array(); {
} $this->messages[] = $record['message'];
}
protected function write(array $record) { }
$this->messages[] = $record['message'];
}
}