mirror of
https://github.com/silverstripe/silverstripe-fulltextsearch
synced 2024-10-22 14:05:29 +02:00
563 lines
17 KiB
PHP
563 lines
17 KiB
PHP
<?php
|
|
|
|
use Monolog\Handler\AbstractProcessingHandler;
|
|
use Monolog\Handler\HandlerInterface;
|
|
use Monolog\Logger;
|
|
use Psr\Log\LoggerInterface;
|
|
|
|
if (class_exists('Phockito')) {
|
|
Phockito::include_hamcrest();
|
|
}
|
|
|
|
class SolrReindexTest extends SapphireTest
|
|
{
|
|
protected $usesDatabase = true;
|
|
|
|
protected $extraDataObjects = array(
|
|
'SolrReindexTest_Item'
|
|
);
|
|
|
|
/**
|
|
* Forced index for testing
|
|
*
|
|
* @var SolrReindexTest_Index
|
|
*/
|
|
protected $index = null;
|
|
|
|
/**
|
|
* Mock service
|
|
*
|
|
* @var SolrService
|
|
*/
|
|
protected $service = null;
|
|
|
|
public function setUp()
|
|
{
|
|
parent::setUp();
|
|
|
|
if (!class_exists('Phockito')) {
|
|
$this->skipTest = true;
|
|
return $this->markTestSkipped("These tests need the Phockito module installed to run");
|
|
}
|
|
|
|
// Set test handler for reindex
|
|
Config::inst()->update('Injector', 'SolrReindexHandler', array(
|
|
'class' => 'SolrReindexTest_TestHandler'
|
|
));
|
|
Injector::inst()->registerService(new SolrReindexTest_TestHandler(), 'SolrReindexHandler');
|
|
|
|
// Set test variant
|
|
SolrReindexTest_Variant::enable();
|
|
|
|
// Set index list
|
|
$this->service = $this->getServiceMock();
|
|
$this->index = singleton('SolrReindexTest_Index');
|
|
$this->index->setService($this->service);
|
|
FullTextSearch::force_index_list($this->index);
|
|
}
|
|
|
|
/**
|
|
* Populate database with dummy dataset
|
|
*
|
|
* @param int $number Number of records to create in each variant
|
|
*/
|
|
protected function createDummyData($number)
|
|
{
|
|
// Populate dataobjects. Use truncate to generate predictable IDs
|
|
DB::query('TRUNCATE "SolrReindexTest_Item"');
|
|
|
|
// Note that we don't create any records in variant = 2, to represent a variant
|
|
// that should be cleared without any re-indexes performed
|
|
foreach (array(0, 1) as $variant) {
|
|
for ($i = 1; $i <= $number; $i++) {
|
|
$item = new SolrReindexTest_Item();
|
|
$item->Variant = $variant;
|
|
$item->Title = "Item $variant / $i";
|
|
$item->write();
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Mock service
|
|
*
|
|
* @return SolrService
|
|
*/
|
|
protected function getServiceMock()
|
|
{
|
|
return Phockito::mock('Solr4Service');
|
|
}
|
|
|
|
public function tearDown()
|
|
{
|
|
FullTextSearch::force_index_list();
|
|
SolrReindexTest_Variant::disable();
|
|
parent::tearDown();
|
|
}
|
|
|
|
/**
|
|
* Get the reindex handler
|
|
*
|
|
* @return SolrReindexHandler
|
|
*/
|
|
protected function getHandler()
|
|
{
|
|
return Injector::inst()->get('SolrReindexHandler');
|
|
}
|
|
|
|
/**
|
|
* Ensure the test variant is up and running properly
|
|
*/
|
|
public function testVariant()
|
|
{
|
|
// State defaults to 0
|
|
$variant = SearchVariant::current_state();
|
|
$this->assertEquals(
|
|
array(
|
|
"SolrReindexTest_Variant" => "0"
|
|
),
|
|
$variant
|
|
);
|
|
|
|
// All states enumerated
|
|
$allStates = iterator_to_array(SearchVariant::reindex_states());
|
|
$this->assertEquals(
|
|
array(
|
|
array(
|
|
"SolrReindexTest_Variant" => "0"
|
|
),
|
|
array(
|
|
"SolrReindexTest_Variant" => "1"
|
|
),
|
|
array(
|
|
"SolrReindexTest_Variant" => "2"
|
|
)
|
|
),
|
|
$allStates
|
|
);
|
|
|
|
// Check correct items created and that filtering on variant works
|
|
$this->createDummyData(120);
|
|
SolrReindexTest_Variant::set_current(2);
|
|
$this->assertEquals(0, SolrReindexTest_Item::get()->count());
|
|
SolrReindexTest_Variant::set_current(1);
|
|
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
|
|
SolrReindexTest_Variant::set_current(0);
|
|
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
|
|
SolrReindexTest_Variant::disable();
|
|
$this->assertEquals(240, SolrReindexTest_Item::get()->count());
|
|
}
|
|
|
|
|
|
/**
|
|
* Given the invocation of a new re-index with a given set of data, ensure that the necessary
|
|
* list of groups are created and segmented for each state
|
|
*
|
|
* Test should work fine with any variants (versioned, subsites, etc) specified
|
|
*/
|
|
public function testReindexSegmentsGroups()
|
|
{
|
|
$this->createDummyData(120);
|
|
|
|
// Initiate re-index
|
|
$logger = new SolrReindexTest_RecordingLogger();
|
|
$this->getHandler()->runReindex($logger, 21, 'Solr_Reindex');
|
|
|
|
// Test that invalid classes are removed
|
|
$this->assertNotEmpty($logger->getMessages('Clearing obsolete classes from SolrReindexTest_Index'));
|
|
Phockito::verify($this->service, 1)
|
|
->deleteByQuery('-(ClassHierarchy:SolrReindexTest_Item)');
|
|
|
|
// Test that valid classes in invalid variants are removed
|
|
$this->assertNotEmpty($logger->getMessages(
|
|
'Clearing all records of type SolrReindexTest_Item in the current state: {"SolrReindexTest_Variant":"2"}'
|
|
));
|
|
Phockito::verify($this->service, 1)
|
|
->deleteByQuery('+(ClassHierarchy:SolrReindexTest_Item) +(_testvariant:"2")');
|
|
|
|
// 120x2 grouped into groups of 21 results in 12 groups
|
|
$this->assertEquals(12, $logger->countMessages('Called processGroup with '));
|
|
$this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"0"}'));
|
|
$this->assertEquals(6, $logger->countMessages('{"SolrReindexTest_Variant":"1"}'));
|
|
|
|
// Given that there are two variants, there should be two group ids of each number
|
|
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 0 of 6'));
|
|
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 1 of 6'));
|
|
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 2 of 6'));
|
|
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 3 of 6'));
|
|
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 4 of 6'));
|
|
$this->assertEquals(2, $logger->countMessages(' SolrReindexTest_Item, group 5 of 6'));
|
|
|
|
// Check various group sizes
|
|
$logger->clear();
|
|
$this->getHandler()->runReindex($logger, 120, 'Solr_Reindex');
|
|
$this->assertEquals(2, $logger->countMessages('Called processGroup with '));
|
|
$logger->clear();
|
|
$this->getHandler()->runReindex($logger, 119, 'Solr_Reindex');
|
|
$this->assertEquals(4, $logger->countMessages('Called processGroup with '));
|
|
$logger->clear();
|
|
$this->getHandler()->runReindex($logger, 121, 'Solr_Reindex');
|
|
$this->assertEquals(2, $logger->countMessages('Called processGroup with '));
|
|
$logger->clear();
|
|
$this->getHandler()->runReindex($logger, 2, 'Solr_Reindex');
|
|
$this->assertEquals(120, $logger->countMessages('Called processGroup with '));
|
|
}
|
|
|
|
/**
|
|
* Test index processing on individual groups
|
|
*/
|
|
public function testRunGroup()
|
|
{
|
|
$this->createDummyData(120);
|
|
$logger = new SolrReindexTest_RecordingLogger();
|
|
|
|
// Initiate re-index of third group (index 2 of 6)
|
|
$state = array('SolrReindexTest_Variant' => '1');
|
|
$this->getHandler()->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, 2);
|
|
$idMessage = $logger->filterMessages('Updated ');
|
|
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/i', $idMessage[0], $matches));
|
|
$ids = array_unique(explode(',', $matches['ids']));
|
|
|
|
// Test successful
|
|
$this->assertNotEmpty($logger->getMessages('Adding SolrReindexTest_Item'));
|
|
$this->assertNotEmpty($logger->getMessages('Done'));
|
|
|
|
// Test that items in this variant / group are cleared from solr
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
|
|
// Test that items in this variant / group are re-indexed
|
|
// 120 divided into 6 groups should be 20 at least (max 21)
|
|
$this->assertEquals(21, count($ids), 'Group size is about 20', 1);
|
|
foreach ($ids as $id) {
|
|
// Each id should be % 6 == 2
|
|
$this->assertEquals(2, $id % 6, "ID $id Should match pattern ID % 6 = 2");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Test that running all groups covers the entire range of dataobject IDs
|
|
*/
|
|
public function testRunAllGroups()
|
|
{
|
|
$this->createDummyData(120);
|
|
$logger = new SolrReindexTest_RecordingLogger();
|
|
|
|
// Test that running all groups covers the complete set of ids
|
|
$state = array('SolrReindexTest_Variant' => '1');
|
|
for ($i = 0; $i < 6; $i++) {
|
|
// See testReindexSegmentsGroups for test that each of these states is invoked during a full reindex
|
|
$this
|
|
->getHandler()
|
|
->runGroup($logger, $this->index, $state, 'SolrReindexTest_Item', 6, $i);
|
|
}
|
|
|
|
// Count all ids updated
|
|
$ids = array();
|
|
foreach ($logger->filterMessages('Updated ') as $message) {
|
|
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/', $message, $matches));
|
|
$ids = array_unique(array_merge($ids, explode(',', $matches['ids'])));
|
|
}
|
|
|
|
// Check ids
|
|
$this->assertEquals(120, count($ids));
|
|
Phockito::verify($this->service, 6)->deleteByQuery(anything());
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=0 u=0}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=1 u=1}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=3 u=3}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=4 u=4}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
Phockito::verify($this->service, 1)->deleteByQuery(
|
|
'+(ClassHierarchy:SolrReindexTest_Item) +_query_:"{!frange l=5 u=5}mod(ID, 6)" +(_testvariant:"1")'
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Provides a wrapper for testing SolrReindexBase
|
|
*/
|
|
class SolrReindexTest_TestHandler extends SolrReindexBase
|
|
{
|
|
public function processGroup(
|
|
LoggerInterface $logger, SolrIndex $indexInstance, $state, $class, $groups, $group, $taskName
|
|
) {
|
|
$indexName = $indexInstance->getIndexName();
|
|
$stateName = json_encode($state);
|
|
$logger->info("Called processGroup with {$indexName}, {$stateName}, {$class}, group {$group} of {$groups}");
|
|
}
|
|
|
|
public function triggerReindex(LoggerInterface $logger, $batchSize, $taskName, $classes = null)
|
|
{
|
|
$logger->info("Called triggerReindex");
|
|
}
|
|
}
|
|
|
|
|
|
class SolrReindexTest_Index extends SolrIndex implements TestOnly
|
|
{
|
|
public function init()
|
|
{
|
|
$this->addClass('SolrReindexTest_Item');
|
|
$this->addAllFulltextFields();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Does not have any variant extensions
|
|
*/
|
|
class SolrReindexTest_Item extends DataObject implements TestOnly
|
|
{
|
|
private static $extensions = array(
|
|
'SolrReindexTest_ItemExtension'
|
|
);
|
|
|
|
private static $db = array(
|
|
'Title' => 'Varchar(255)',
|
|
'Variant' => 'Int(0)'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Select only records in the current variant
|
|
*/
|
|
class SolrReindexTest_ItemExtension extends DataExtension implements TestOnly
|
|
{
|
|
/**
|
|
* Filter records on the current variant
|
|
*
|
|
* @param SQLQuery $query
|
|
* @param DataQuery $dataQuery
|
|
*/
|
|
public function augmentSQL(SQLQuery &$query, DataQuery &$dataQuery = null)
|
|
{
|
|
$variant = SolrReindexTest_Variant::get_current();
|
|
if ($variant !== null && !$query->filtersOnID()) {
|
|
$sqlVariant = Convert::raw2sql($variant);
|
|
$query->addWhere("\"Variant\" = '{$sqlVariant}'");
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Dummy variant that selects items with field Varient matching the current value
|
|
*
|
|
* Variant states are 0 and 1, or null if disabled
|
|
*/
|
|
class SolrReindexTest_Variant extends SearchVariant implements TestOnly
|
|
{
|
|
/**
|
|
* Value of this variant (either null, 0, or 1)
|
|
*
|
|
* @var int|null
|
|
*/
|
|
protected static $current = null;
|
|
|
|
/**
|
|
* Activate this variant
|
|
*/
|
|
public static function enable()
|
|
{
|
|
self::disable();
|
|
|
|
self::$current = 0;
|
|
self::$variants = array(
|
|
'SolrReindexTest_Variant' => singleton('SolrReindexTest_Variant')
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Disable this variant and reset
|
|
*/
|
|
public static function disable()
|
|
{
|
|
self::$current = null;
|
|
self::$variants = null;
|
|
self::$class_variants = array();
|
|
self::$call_instances = array();
|
|
}
|
|
|
|
public function activateState($state)
|
|
{
|
|
self::set_current($state);
|
|
}
|
|
|
|
/**
|
|
* Set the current variant to the given state
|
|
*
|
|
* @param int $current 0, 1, 2, or null (disabled)
|
|
*/
|
|
public static function set_current($current)
|
|
{
|
|
self::$current = $current;
|
|
}
|
|
|
|
/**
|
|
* Get the current state
|
|
*
|
|
* @return string|null
|
|
*/
|
|
public static function get_current()
|
|
{
|
|
// Always use string values for states for consistent json_encode value
|
|
if (isset(self::$current)) {
|
|
return (string)self::$current;
|
|
}
|
|
}
|
|
|
|
public function alterDefinition($base, $index)
|
|
{
|
|
$self = get_class($this);
|
|
|
|
$index->filterFields['_testvariant'] = array(
|
|
'name' => '_testvariant',
|
|
'field' => '_testvariant',
|
|
'fullfield' => '_testvariant',
|
|
'base' => $base,
|
|
'origin' => $base,
|
|
'type' => 'Int',
|
|
'lookup_chain' => array(array('call' => 'variant', 'variant' => $self, 'method' => 'currentState'))
|
|
);
|
|
}
|
|
|
|
public function alterQuery($query, $index)
|
|
{
|
|
// I guess just calling it _testvariant is ok?
|
|
$query->filter('_testvariant', $this->currentState());
|
|
}
|
|
|
|
public function appliesTo($class, $includeSubclasses)
|
|
{
|
|
return $class === 'SolrReindexTest_Item' ||
|
|
($includeSubclasses && is_subclass_of($class, 'SolrReindexTest_Item', true));
|
|
}
|
|
|
|
public function appliesToEnvironment()
|
|
{
|
|
// Set to null to disable
|
|
return self::$current !== null;
|
|
}
|
|
|
|
public function currentState()
|
|
{
|
|
return self::get_current();
|
|
}
|
|
|
|
public function reindexStates()
|
|
{
|
|
// Always use string values for states for consistent json_encode value
|
|
return array('0', '1', '2');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Test logger for recording messages
|
|
*/
|
|
class SolrReindexTest_RecordingLogger extends Logger implements TestOnly
|
|
{
|
|
/**
|
|
* @var SolrReindexTest_Handler
|
|
*/
|
|
protected $testHandler = null;
|
|
|
|
public function __construct($name = 'testlogger', array $handlers = array(), array $processors = array())
|
|
{
|
|
parent::__construct($name, $handlers, $processors);
|
|
|
|
$this->testHandler = new SolrReindexTest_Handler();
|
|
$this->pushHandler($this->testHandler);
|
|
}
|
|
|
|
/**
|
|
* @return array
|
|
*/
|
|
public function getMessages()
|
|
{
|
|
return $this->testHandler->getMessages();
|
|
}
|
|
|
|
/**
|
|
* Clear all messages
|
|
*/
|
|
public function clear()
|
|
{
|
|
$this->testHandler->clear();
|
|
}
|
|
|
|
/**
|
|
* Get messages with the given filter
|
|
*
|
|
* @param string $containing
|
|
* @return array Filtered array
|
|
*/
|
|
public function filterMessages($containing)
|
|
{
|
|
return array_values(array_filter(
|
|
$this->getMessages(),
|
|
function ($content) use ($containing) {
|
|
return stripos($content, $containing) !== false;
|
|
}
|
|
));
|
|
}
|
|
|
|
/**
|
|
* Count all messages containing the given substring
|
|
*
|
|
* @param string $containing Message to filter by
|
|
* @return int
|
|
*/
|
|
public function countMessages($containing = null)
|
|
{
|
|
if ($containing) {
|
|
$messages = $this->filterMessages($containing);
|
|
} else {
|
|
$messages = $this->getMessages();
|
|
}
|
|
return count($messages);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Logger for recording messages for later retrieval
|
|
*/
|
|
class SolrReindexTest_Handler extends AbstractProcessingHandler implements TestOnly
|
|
{
|
|
/**
|
|
* Messages
|
|
*
|
|
* @var array
|
|
*/
|
|
protected $messages = array();
|
|
|
|
/**
|
|
* Get all messages
|
|
*
|
|
* @return array
|
|
*/
|
|
public function getMessages()
|
|
{
|
|
return $this->messages;
|
|
}
|
|
|
|
public function clear()
|
|
{
|
|
$this->messages = array();
|
|
}
|
|
|
|
protected function write(array $record)
|
|
{
|
|
$this->messages[] = $record['message'];
|
|
}
|
|
}
|