API Solr_Reindex uses configured SearchUpdater instead of always doing a direct write

Emit errors on Solr_Reindex if verbose=1
This commit is contained in:
Damian Mooyman 2015-05-06 17:01:41 +12:00
parent 66a2353d33
commit 625d282df2
3 changed files with 83 additions and 22 deletions

View File

@ -1,3 +1,3 @@
DataObject: DataObject:
extensions: extensions:
- 'SearchUpdater_DeleteHandler' - 'SearchUpdater_ObjectHandler'

View File

@ -73,7 +73,7 @@ class SearchUpdater extends Object {
SearchVariant::call('extractManipulationState', $manipulation); SearchVariant::call('extractManipulationState', $manipulation);
// Then combine the manipulation back into object field sets // Then combine the manipulation back into object field sets
$writes = array(); $writes = array();
foreach ($manipulation as $table => $details) { foreach ($manipulation as $table => $details) {
@ -88,11 +88,21 @@ class SearchUpdater extends Object {
$key = "$id:$base:".serialize($state); $key = "$id:$base:".serialize($state);
$statefulids = array(array('id' => $id, 'state' => $state)); $statefulids = array(array('id' => $id, 'state' => $state));
// Is this the first table for this particular object? Then add an item to $writes // Is this the first table for this particular object? Then add an item to $writes
if (!isset($writes[$key])) $writes[$key] = array('base' => $base, 'class' => $class, 'id' => $id, 'statefulids' => $statefulids, 'fields' => array()); if (!isset($writes[$key])) {
$writes[$key] = array(
'base' => $base,
'class' => $class,
'id' => $id,
'statefulids' => $statefulids,
'fields' => array()
);
}
// Otherwise update the class label if it's more specific than the currently recorded one // Otherwise update the class label if it's more specific than the currently recorded one
else if (is_subclass_of($class, $writes[$key]['class'])) $writes[$key]['class'] = $class; else if (is_subclass_of($class, $writes[$key]['class'])) {
$writes[$key]['class'] = $class;
}
// Update the fields // Update the fields
foreach ($fields as $field => $value) { foreach ($fields as $field => $value) {
@ -104,8 +114,17 @@ class SearchUpdater extends Object {
SearchVariant::call('extractManipulationWriteState', $writes); SearchVariant::call('extractManipulationWriteState', $writes);
// Then for each write, figure out what objects need updating // Submit all of these writes to the search processor
static::process_writes($writes);
}
/**
* Send updates to the current search processor for execution
*
* @param array $writes
*/
public static function process_writes($writes) {
foreach ($writes as $write) { foreach ($writes as $write) {
// For every index // For every index
foreach (FullTextSearch::get_indexes() as $index => $instance) { foreach (FullTextSearch::get_indexes() as $index => $instance) {
@ -117,7 +136,9 @@ class SearchUpdater extends Object {
// Then add then then to the global list to deal with later // Then add then then to the global list to deal with later
foreach ($dirtyids as $dirtyclass => $ids) { foreach ($dirtyids as $dirtyclass => $ids) {
if ($ids) { if ($ids) {
if (!self::$processor) self::$processor = Injector::inst()->create('SearchUpdateProcessor'); if (!self::$processor) {
self::$processor = Injector::inst()->create('SearchUpdateProcessor');
}
self::$processor->addDirtyIDs($dirtyclass, $ids, $index); self::$processor->addDirtyIDs($dirtyclass, $ids, $index);
} }
} }
@ -125,11 +146,11 @@ class SearchUpdater extends Object {
} }
} }
// Finally, if we do have some work to do register the shutdown function to actually do the work // If we do have some work to do register the shutdown function to actually do the work
// Don't do it if we're testing - there's no database connection outside the test methods, so we'd // Don't do it if we're testing - there's no database connection outside the test methods, so we'd
// just get errors // just get errors
$runningTests = class_exists('SapphireTest',false) && SapphireTest::is_running_test(); $runningTests = class_exists('SapphireTest', false) && SapphireTest::is_running_test();
if (self::$processor && !self::$registered && !$runningTests) { if (self::$processor && !self::$registered && !$runningTests) {
register_shutdown_function(array("SearchUpdater", "flush_dirty_indexes")); register_shutdown_function(array("SearchUpdater", "flush_dirty_indexes"));
@ -168,17 +189,17 @@ class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter {
/** /**
* Delete operations do not use database manipulations. * Delete operations do not use database manipulations.
* *
* If a delete has been requested, force a write on objects that should be * If a delete has been requested, force a write on objects that should be
* indexed. This causes the object to be marked for deletion from the index. * indexed. This causes the object to be marked for deletion from the index.
*/ */
class SearchUpdater_DeleteHandler extends DataExtension { class SearchUpdater_ObjectHandler extends DataExtension {
public function onAfterDelete() { public function onAfterDelete() {
// Calling delete() on empty objects does nothing // Calling delete() on empty objects does nothing
if (!$this->owner->ID) return; if (!$this->owner->ID) return;
// Force SearchUpdater to mark this record as dirty // Force SearchUpdater to mark this record as dirty
$manipulation = array( $manipulation = array(
$this->owner->ClassName => array( $this->owner->ClassName => array(
@ -191,4 +212,36 @@ class SearchUpdater_DeleteHandler extends DataExtension {
SearchUpdater::handle_manipulation($manipulation); SearchUpdater::handle_manipulation($manipulation);
} }
/**
* Forces this object to trigger a re-index in the current state
*/
public function triggerReindex() {
if (!$this->owner->ID) {
return;
}
$id = $this->owner->ID;
$class = $this->owner->ClassName;
$state = SearchVariant::current_state();
$base = ClassInfo::baseDataClass($class);
$key = "$id:$base:".serialize($state);
$statefulids = array(array(
'id' => $id,
'state' => $state
));
$writes = array(
$key => array(
'base' => $base,
'class' => $class,
'id' => $id,
'statefulids' => $statefulids,
'fields' => array()
)
);
SearchUpdater::process_writes($writes);
}
} }

View File

@ -163,7 +163,7 @@ class Solr_Configure extends BuildTask {
} else { } else {
user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR); user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR);
} }
foreach ($indexes as $instance) { foreach ($indexes as $instance) {
$index = $instance->getIndexName(); $index = $instance->getIndexName();
echo "Configuring $index. \n"; flush(); echo "Configuring $index. \n"; flush();
@ -200,7 +200,14 @@ class Solr_Configure extends BuildTask {
class Solr_Reindex extends BuildTask { class Solr_Reindex extends BuildTask {
static $recordsPerRequest = 200;
/**
* Number of records to load and index per request
*
* @var int
* @config
*/
private static $recordsPerRequest = 200;
public function run($request) { public function run($request) {
increase_time_limit_to(); increase_time_limit_to();
@ -223,7 +230,7 @@ class Solr_Reindex extends BuildTask {
foreach (Solr::get_indexes() as $index => $instance) { foreach (Solr::get_indexes() as $index => $instance) {
echo "Rebuilding {$instance->getIndexName()}\n\n"; echo "Rebuilding {$instance->getIndexName()}\n\n";
$classes = $instance->getClasses(); $classes = $instance->getClasses();
if($request->getVar('class')) { if($request->getVar('class')) {
$limitClasses = explode(',', $request->getVar('class')); $limitClasses = explode(',', $request->getVar('class'));
@ -234,10 +241,10 @@ class Solr_Reindex extends BuildTask {
foreach ($classes as $class => $options) { foreach ($classes as $class => $options) {
$includeSubclasses = $options['include_children']; $includeSubclasses = $options['include_children'];
foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) { foreach (SearchVariant::reindex_states($class, $includeSubclasses) as $state) {
if ($instance->variantStateExcluded($state)) continue; if ($instance->variantStateExcluded($state)) continue;
SearchVariant::activate_state($state); SearchVariant::activate_state($state);
$filter = $includeSubclasses ? "" : '"ClassName" = \''.$class."'"; $filter = $includeSubclasses ? "" : '"ClassName" = \''.$class."'";
@ -259,12 +266,12 @@ class Solr_Reindex extends BuildTask {
echo "$offset.."; echo "$offset..";
$cmd = "php $script dev/tasks/$self index=$index class=$class start=$offset variantstate=$statevar"; $cmd = "php $script dev/tasks/$self index=$index class=$class start=$offset variantstate=$statevar";
if($verbose) { if($verbose) {
echo "\n Running '$cmd'\n"; echo "\n Running '$cmd'\n";
$cmd .= " verbose=1"; $cmd .= " verbose=1 2>&1";
} }
$res = $verbose ? passthru($cmd) : `$cmd`; $res = $verbose ? passthru($cmd) : `$cmd`;
if($verbose) echo " ".preg_replace('/\r\n|\n/', '$0 ', $res)."\n"; if($verbose) echo " ".preg_replace('/\r\n|\n/', '$0 ', $res)."\n";
@ -304,9 +311,10 @@ class Solr_Reindex extends BuildTask {
foreach ($items as $item) { foreach ($items as $item) {
if($verbose) echo $item->ID . ' '; if($verbose) echo $item->ID . ' ';
$index->add($item); // See SearchUpdater_ObjectHandler::triggerReindex
$item->triggerReindex();
$item->destroy(); $item->destroy();
} }
if($verbose) echo "Done "; if($verbose) echo "Done ";