silverstripe-fulltextsearch/tests/BatchedProcessorTest.php

249 lines
9.1 KiB
PHP
Raw Normal View History

<?php
2017-04-21 02:23:27 +02:00
namespace SilverStripe\FullTextSearch\Tests;
use SilverStripe\CMS\Model\SiteTree;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\SapphireTest;
use SilverStripe\FullTextSearch\Search\FullTextSearch;
use SilverStripe\FullTextSearch\Search\Services\SearchableService;
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessor_QueuedJobService;
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Index;
use SilverStripe\FullTextSearch\Tests\BatchedProcessorTest\BatchedProcessorTest_Object;
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateCommitJobProcessor;
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateQueuedJobProcessor;
use SilverStripe\FullTextSearch\Search\Processors\SearchUpdateBatchedProcessor;
use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater;
use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned;
use SilverStripe\Subsites\Extensions\SiteTreeSubsites;
use SilverStripe\Subsites\Model\Subsite;
use SilverStripe\ORM\FieldType\DBDatetime;
use SilverStripe\Versioned\Versioned;
use Symbiote\QueuedJobs\Services\QueuedJobService;
use Symbiote\QueuedJobs\Services\QueuedJob;
/**
* Tests {@see SearchUpdateQueuedJobProcessor}
*/
2015-11-21 07:19:20 +01:00
class BatchedProcessorTest extends SapphireTest
{
protected $usesDatabase = true;
2015-11-21 07:19:20 +01:00
protected $oldProcessor;
protected static $extra_dataobjects = [
BatchedProcessorTest_Object::class,
];
2015-11-21 07:19:20 +01:00
protected static $illegal_extensions = [
SiteTree::class => [
SiteTreeSubsites::class,
],
];
2015-11-21 07:19:20 +01:00
2021-11-02 02:48:12 +01:00
public static function setUpBeforeClass(): void
2015-11-21 07:19:20 +01:00
{
// Disable illegal extensions if skipping this test
if (class_exists(Subsite::class) || !interface_exists(QueuedJob::class)) {
static::$illegal_extensions = [];
2015-11-21 07:19:20 +01:00
}
parent::setUpBeforeClass();
2015-11-21 07:19:20 +01:00
}
2021-11-02 02:48:12 +01:00
protected function setUp(): void
2015-11-21 07:19:20 +01:00
{
parent::setUp();
if (!interface_exists(QueuedJob::class)) {
2015-11-21 07:19:20 +01:00
$this->markTestSkipped("These tests need the QueuedJobs module installed to run");
}
if (class_exists(Subsite::class)) {
2015-11-21 07:19:20 +01:00
$this->markTestSkipped(get_class() . ' skipped when running with subsites');
}
DBDatetime::set_mock_now('2015-05-07 06:00:00');
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_size', 5);
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 0);
Config::modify()->set(SearchUpdateCommitJobProcessor::class, 'cooldown', 600);
Versioned::set_stage(Versioned::DRAFT);
2015-11-21 07:19:20 +01:00
Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), QueuedJobService::class);
2015-11-21 07:19:20 +01:00
FullTextSearch::force_index_list(BatchedProcessorTest_Index::class);
2015-11-21 07:19:20 +01:00
SearchUpdateCommitJobProcessor::$dirty_indexes = array();
SearchUpdateCommitJobProcessor::$has_run = false;
2015-11-21 07:19:20 +01:00
$this->oldProcessor = SearchUpdater::$processor;
SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor();
}
2021-11-02 02:48:12 +01:00
protected function tearDown(): void
2015-11-21 07:19:20 +01:00
{
if ($this->oldProcessor) {
SearchUpdater::$processor = $this->oldProcessor;
}
FullTextSearch::force_index_list();
parent::tearDown();
}
/**
* @return SearchUpdateQueuedJobProcessor
*/
protected function generateDirtyIds()
{
$processor = SearchUpdater::$processor;
for ($id = 1; $id <= 42; $id++) {
// Save to db
$object = new BatchedProcessorTest_Object();
$object->TestText = 'Object ' . $id;
$object->write();
// Add to index manually
$processor->addDirtyIDs(
BatchedProcessorTest_Object::class,
2015-11-21 07:19:20 +01:00
array(array(
'id' => $id,
'state' => array(SearchVariantVersioned::class => 'Stage')
2015-11-21 07:19:20 +01:00
)),
BatchedProcessorTest_Index::class
2015-11-21 07:19:20 +01:00
);
}
$processor->batchData();
return $processor;
}
2015-11-21 07:19:20 +01:00
/**
* Tests that large jobs are broken up into a suitable number of batches
*/
public function testBatching()
{
Config::modify()->set(SearchableService::class, 'indexing_canview_exclude_classes', [SiteTree::class]);
Config::modify()->set(SearchableService::class, 'variant_state_draft_excluded', false);
$index = singleton(BatchedProcessorTest_Index::class);
2015-11-21 07:19:20 +01:00
$index->reset();
$processor = $this->generateDirtyIds();
2015-11-21 07:19:20 +01:00
// Check initial state
$data = $processor->getJobData();
$this->assertEquals(9, $data->totalSteps);
$this->assertEquals(0, $data->currentStep);
$this->assertEmpty($data->isComplete);
$this->assertEquals(0, count($index->getAdded()));
2015-11-21 07:19:20 +01:00
// Advance state
for ($pass = 1; $pass <= 8; $pass++) {
$processor->process();
$data = $processor->getJobData();
$this->assertEquals($pass, $data->currentStep);
$this->assertEquals($pass * 5, count($index->getAdded()));
}
2015-11-21 07:19:20 +01:00
// Last run should have two hanging items
$processor->process();
$data = $processor->getJobData();
$this->assertEquals(9, $data->currentStep);
$this->assertEquals(42, count($index->getAdded()));
$this->assertTrue($data->isComplete);
// Check any additional queued jobs
$processor->afterComplete();
$service = singleton(QueuedJobService::class);
2015-11-21 07:19:20 +01:00
$jobs = $service->getJobs();
$this->assertEquals(1, count($jobs));
$this->assertInstanceOf(SearchUpdateCommitJobProcessor::class, $jobs[0]['job']);
2015-11-21 07:19:20 +01:00
}
/**
* Test creation of multiple commit jobs
*/
public function testMultipleCommits()
{
$index = singleton(BatchedProcessorTest_Index::class);
2015-11-21 07:19:20 +01:00
$index->reset();
// Test that running a commit immediately after submitting to the indexes
// correctly commits
$first = SearchUpdateCommitJobProcessor::queue();
$second = SearchUpdateCommitJobProcessor::queue();
$this->assertFalse($index->getIsCommitted());
// First process will cause the commit
$this->assertFalse($first->jobFinished());
$first->process();
$allMessages = $first->getMessages();
$this->assertTrue($index->getIsCommitted());
$this->assertTrue($first->jobFinished());
$this->assertStringEndsWith('All indexes committed', $allMessages[2]);
// Executing the subsequent processor should not re-trigger a commit
$index->reset();
$this->assertFalse($second->jobFinished());
$second->process();
$allMessages = $second->getMessages();
$this->assertFalse($index->getIsCommitted());
$this->assertTrue($second->jobFinished());
$this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]);
// Given that a third job is created, and the indexes are dirtied, attempting to run this job
// should result in a delay
$index->reset();
$third = SearchUpdateCommitJobProcessor::queue();
$this->assertFalse($third->jobFinished());
$third->process();
$this->assertTrue($third->jobFinished());
$allMessages = $third->getMessages();
$this->assertStringEndsWith(
'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00',
$allMessages[0]
);
}
/**
* Tests that the batch_soft_cap setting is properly respected
*/
public function testSoftCap()
{
$this->markTestIncomplete(
'@todo PostgreSQL: This test passes in isolation, but not in conjunction with the previous test'
);
$index = singleton(BatchedProcessorTest_Index::class);
2015-11-21 07:19:20 +01:00
$index->reset();
2015-11-21 07:19:20 +01:00
$processor = $this->generateDirtyIds();
2015-11-21 07:19:20 +01:00
// Test that increasing the soft cap to 2 will reduce the number of batches
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 2);
2015-11-21 07:19:20 +01:00
$processor->batchData();
$data = $processor->getJobData();
$this->assertEquals(8, $data->totalSteps);
2015-11-21 07:19:20 +01:00
// A soft cap of 1 should not fit in the hanging two items
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 1);
2015-11-21 07:19:20 +01:00
$processor->batchData();
$data = $processor->getJobData();
$this->assertEquals(9, $data->totalSteps);
2015-11-21 07:19:20 +01:00
// Extra large soft cap should fit both items
Config::modify()->set(SearchUpdateBatchedProcessor::class, 'batch_soft_cap', 4);
2015-11-21 07:19:20 +01:00
$processor->batchData();
$data = $processor->getJobData();
$this->assertEquals(8, $data->totalSteps);
2015-11-21 07:19:20 +01:00
// Process all data and ensure that all are processed adequately
for ($pass = 1; $pass <= 8; $pass++) {
$processor->process();
}
$data = $processor->getJobData();
$this->assertEquals(8, $data->currentStep);
$this->assertEquals(42, count($index->getAdded()));
$this->assertTrue($data->isComplete);
}
}