2014-05-16 07:42:08 +02:00
|
|
|
<?php
|
|
|
|
|
2017-04-21 02:23:27 +02:00
|
|
|
namespace SilverStripe\FullTextSearch\Tests;
|
2014-05-16 07:42:08 +02:00
|
|
|
|
2017-04-21 02:23:27 +02:00
|
|
|
use SilverStripe\Dev\SapphireTest;
|
|
|
|
use SilverStripe\FullTextSearch\Search\FullTextSearch;
|
2017-04-26 12:52:20 +02:00
|
|
|
use SilverStripe\ORM\FieldType\DBDatetime;
|
|
|
|
use SilverStripe\Core\Config\Config;
|
|
|
|
use SilverStripe\Versioned\Versioned;
|
|
|
|
use SilverStripe\Core\Injector\Injector;
|
|
|
|
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessor_QueuedJobService;
|
|
|
|
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Index;
|
|
|
|
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Object;
|
|
|
|
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateCommitJobProcessor;
|
|
|
|
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateQueuedJobProcessor;
|
|
|
|
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateBatchedProcessor;
|
|
|
|
use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater;
|
|
|
|
use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned;
|
2017-11-29 03:31:17 +01:00
|
|
|
use Symbiote\QueuedJobs\Services\QueuedJobService;
|
2015-05-07 05:16:27 +02:00
|
|
|
|
2014-05-16 07:42:08 +02:00
|
|
|
/**
|
|
|
|
* Tests {@see SearchUpdateQueuedJobProcessor}
|
|
|
|
*/
|
2015-11-21 07:19:20 +01:00
|
|
|
class BatchedProcessorTest extends SapphireTest
|
|
|
|
{
|
|
|
|
protected $oldProcessor;
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2017-04-26 12:52:20 +02:00
|
|
|
protected static $extra_dataobjects = array(
|
|
|
|
BatchedProcessorTest_Object::class
|
2015-11-21 07:19:20 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
protected $illegalExtensions = array(
|
|
|
|
'SiteTree' => array(
|
|
|
|
'SiteTreeSubsites',
|
|
|
|
'Translatable'
|
|
|
|
)
|
|
|
|
);
|
|
|
|
|
|
|
|
public function setUpOnce()
|
|
|
|
{
|
|
|
|
// Disable illegal extensions if skipping this test
|
2017-11-29 03:31:17 +01:00
|
|
|
if (class_exists('Subsite') || !interface_exists('Symbiote\QueuedJobs\Services\QueuedJob')) {
|
2015-11-21 07:19:20 +01:00
|
|
|
$this->illegalExtensions = array();
|
|
|
|
}
|
|
|
|
parent::setUpOnce();
|
|
|
|
}
|
|
|
|
|
|
|
|
public function setUp()
|
|
|
|
{
|
|
|
|
parent::setUp();
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2017-11-29 03:31:17 +01:00
|
|
|
if (!interface_exists('Symbiote\QueuedJobs\Services\QueuedJob')) {
|
2015-11-21 07:19:20 +01:00
|
|
|
$this->skipTest = true;
|
|
|
|
$this->markTestSkipped("These tests need the QueuedJobs module installed to run");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (class_exists('Subsite')) {
|
|
|
|
$this->skipTest = true;
|
|
|
|
$this->markTestSkipped(get_class() . ' skipped when running with subsites');
|
|
|
|
}
|
|
|
|
|
2017-04-26 12:52:20 +02:00
|
|
|
DBDatetime::set_mock_now('2015-05-07 06:00:00');
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2017-04-26 12:52:20 +02:00
|
|
|
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_size', 5);
|
|
|
|
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 0);
|
|
|
|
Config::modify()->set(SearchUpdateCommitJobProcessor::class, 'cooldown', 600);
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2017-04-26 12:52:20 +02:00
|
|
|
Versioned::set_stage("Stage");
|
2015-11-21 07:19:20 +01:00
|
|
|
|
2017-04-26 12:52:20 +02:00
|
|
|
Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), QueuedJobService::class);
|
2015-11-21 07:19:20 +01:00
|
|
|
|
2017-04-26 12:52:20 +02:00
|
|
|
FullTextSearch::force_index_list(BatchedProcessorTest_Index::class);
|
2015-11-21 07:19:20 +01:00
|
|
|
|
|
|
|
SearchUpdateCommitJobProcessor::$dirty_indexes = array();
|
|
|
|
SearchUpdateCommitJobProcessor::$has_run = false;
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
$this->oldProcessor = SearchUpdater::$processor;
|
|
|
|
SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor();
|
|
|
|
}
|
|
|
|
|
|
|
|
public function tearDown()
|
|
|
|
{
|
|
|
|
if ($this->oldProcessor) {
|
|
|
|
SearchUpdater::$processor = $this->oldProcessor;
|
|
|
|
}
|
|
|
|
FullTextSearch::force_index_list();
|
|
|
|
parent::tearDown();
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @return SearchUpdateQueuedJobProcessor
|
|
|
|
*/
|
|
|
|
protected function generateDirtyIds()
|
|
|
|
{
|
|
|
|
$processor = SearchUpdater::$processor;
|
|
|
|
for ($id = 1; $id <= 42; $id++) {
|
|
|
|
// Save to db
|
|
|
|
$object = new BatchedProcessorTest_Object();
|
|
|
|
$object->TestText = 'Object ' . $id;
|
|
|
|
$object->write();
|
|
|
|
// Add to index manually
|
|
|
|
$processor->addDirtyIDs(
|
2017-04-26 12:52:20 +02:00
|
|
|
BatchedProcessorTest_Object::class,
|
2015-11-21 07:19:20 +01:00
|
|
|
array(array(
|
|
|
|
'id' => $id,
|
2017-04-26 12:52:20 +02:00
|
|
|
'state' => array(SearchVariantVersioned::class => 'Stage')
|
2015-11-21 07:19:20 +01:00
|
|
|
)),
|
2017-04-26 12:52:20 +02:00
|
|
|
BatchedProcessorTest_Index::class
|
2015-11-21 07:19:20 +01:00
|
|
|
);
|
|
|
|
}
|
|
|
|
$processor->batchData();
|
|
|
|
return $processor;
|
|
|
|
}
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
/**
|
|
|
|
* Tests that large jobs are broken up into a suitable number of batches
|
|
|
|
*/
|
|
|
|
public function testBatching()
|
|
|
|
{
|
2017-04-26 12:52:20 +02:00
|
|
|
$index = singleton(BatchedProcessorTest_Index::class);
|
2015-11-21 07:19:20 +01:00
|
|
|
$index->reset();
|
|
|
|
$processor = $this->generateDirtyIds();
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// Check initial state
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
$this->assertEquals(9, $data->totalSteps);
|
|
|
|
$this->assertEquals(0, $data->currentStep);
|
|
|
|
$this->assertEmpty($data->isComplete);
|
|
|
|
$this->assertEquals(0, count($index->getAdded()));
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// Advance state
|
|
|
|
for ($pass = 1; $pass <= 8; $pass++) {
|
|
|
|
$processor->process();
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
$this->assertEquals($pass, $data->currentStep);
|
|
|
|
$this->assertEquals($pass * 5, count($index->getAdded()));
|
|
|
|
}
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// Last run should have two hanging items
|
|
|
|
$processor->process();
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
$this->assertEquals(9, $data->currentStep);
|
|
|
|
$this->assertEquals(42, count($index->getAdded()));
|
|
|
|
$this->assertTrue($data->isComplete);
|
|
|
|
|
|
|
|
// Check any additional queued jobs
|
|
|
|
$processor->afterComplete();
|
2017-04-26 12:52:20 +02:00
|
|
|
$service = singleton(QueuedJobService::class);
|
2015-11-21 07:19:20 +01:00
|
|
|
$jobs = $service->getJobs();
|
|
|
|
$this->assertEquals(1, count($jobs));
|
2017-04-26 12:52:20 +02:00
|
|
|
$this->assertInstanceOf(SearchUpdateCommitJobProcessor::class, $jobs[0]['job']);
|
2015-11-21 07:19:20 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Test creation of multiple commit jobs
|
|
|
|
*/
|
|
|
|
public function testMultipleCommits()
|
|
|
|
{
|
2017-04-26 12:52:20 +02:00
|
|
|
$index = singleton(BatchedProcessorTest_Index::class);
|
2015-11-21 07:19:20 +01:00
|
|
|
$index->reset();
|
|
|
|
|
|
|
|
// Test that running a commit immediately after submitting to the indexes
|
|
|
|
// correctly commits
|
|
|
|
$first = SearchUpdateCommitJobProcessor::queue();
|
|
|
|
$second = SearchUpdateCommitJobProcessor::queue();
|
|
|
|
|
|
|
|
$this->assertFalse($index->getIsCommitted());
|
|
|
|
|
|
|
|
// First process will cause the commit
|
|
|
|
$this->assertFalse($first->jobFinished());
|
|
|
|
$first->process();
|
|
|
|
$allMessages = $first->getMessages();
|
|
|
|
$this->assertTrue($index->getIsCommitted());
|
|
|
|
$this->assertTrue($first->jobFinished());
|
|
|
|
$this->assertStringEndsWith('All indexes committed', $allMessages[2]);
|
|
|
|
|
|
|
|
// Executing the subsequent processor should not re-trigger a commit
|
|
|
|
$index->reset();
|
|
|
|
$this->assertFalse($second->jobFinished());
|
|
|
|
$second->process();
|
|
|
|
$allMessages = $second->getMessages();
|
|
|
|
$this->assertFalse($index->getIsCommitted());
|
|
|
|
$this->assertTrue($second->jobFinished());
|
|
|
|
$this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]);
|
|
|
|
|
|
|
|
// Given that a third job is created, and the indexes are dirtied, attempting to run this job
|
|
|
|
// should result in a delay
|
|
|
|
$index->reset();
|
|
|
|
$third = SearchUpdateCommitJobProcessor::queue();
|
|
|
|
$this->assertFalse($third->jobFinished());
|
|
|
|
$third->process();
|
|
|
|
$this->assertTrue($third->jobFinished());
|
|
|
|
$allMessages = $third->getMessages();
|
|
|
|
$this->assertStringEndsWith(
|
|
|
|
'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00',
|
|
|
|
$allMessages[0]
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
/**
|
|
|
|
* Tests that the batch_soft_cap setting is properly respected
|
|
|
|
*/
|
|
|
|
public function testSoftCap()
|
|
|
|
{
|
2017-04-26 12:52:20 +02:00
|
|
|
$index = singleton(BatchedProcessorTest_Index::class);
|
2015-11-21 07:19:20 +01:00
|
|
|
$index->reset();
|
|
|
|
$processor = $this->generateDirtyIds();
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// Test that increasing the soft cap to 2 will reduce the number of batches
|
2017-04-26 12:52:20 +02:00
|
|
|
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 2);
|
2015-11-21 07:19:20 +01:00
|
|
|
$processor->batchData();
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
//Debug::dump($data);die;
|
|
|
|
$this->assertEquals(8, $data->totalSteps);
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// A soft cap of 1 should not fit in the hanging two items
|
2017-04-26 12:52:20 +02:00
|
|
|
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 1);
|
2015-11-21 07:19:20 +01:00
|
|
|
$processor->batchData();
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
$this->assertEquals(9, $data->totalSteps);
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// Extra large soft cap should fit both items
|
2017-04-26 12:52:20 +02:00
|
|
|
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 4);
|
2015-11-21 07:19:20 +01:00
|
|
|
$processor->batchData();
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
$this->assertEquals(8, $data->totalSteps);
|
2016-04-15 05:46:19 +02:00
|
|
|
|
2015-11-21 07:19:20 +01:00
|
|
|
// Process all data and ensure that all are processed adequately
|
|
|
|
for ($pass = 1; $pass <= 8; $pass++) {
|
|
|
|
$processor->process();
|
|
|
|
}
|
|
|
|
$data = $processor->getJobData();
|
|
|
|
$this->assertEquals(8, $data->currentStep);
|
|
|
|
$this->assertEquals(42, count($index->getAdded()));
|
|
|
|
$this->assertTrue($data->isComplete);
|
|
|
|
}
|
2014-05-16 07:42:08 +02:00
|
|
|
}
|