silverstripe-fulltextsearch/tests/BatchedProcessorTest.php

<?php


class BatchedProcessorTest_Object extends SiteTree implements TestOnly
{
    private static $db = array(
        'TestText' => 'Varchar'
    );
}

class BatchedProcessorTest_Index extends SearchIndex_Recording implements TestOnly
{
    public function init()
    {
        $this->addClass('BatchedProcessorTest_Object');
        $this->addFilterField('TestText');
    }
}

class BatchedProcessor_QueuedJobService
{
    protected $jobs = array();

    public function queueJob(QueuedJob $job, $startAfter = null, $userId = null, $queueName = null)
    {
        $this->jobs[] = array(
            'job' => $job,
            'startAfter' => $startAfter
        );
        return $job;
    }

    public function getJobs()
    {
        return $this->jobs;
    }
}

/**
 * Tests {@see SearchUpdateQueuedJobProcessor}
 */
class BatchedProcessorTest extends SapphireTest
{
    protected $oldProcessor;

    protected $extraDataObjects = array(
        'BatchedProcessorTest_Object'
    );

    protected $illegalExtensions = array(
        'SiteTree' => array(
            'SiteTreeSubsites',
            'Translatable'
        )
    );

    public function setUpOnce()
    {
        // Disable illegal extensions if skipping this test
        if (class_exists('Subsite') || !interface_exists('QueuedJob')) {
            $this->illegalExtensions = array();
        }
        parent::setUpOnce();
    }

    public function setUp()
    {
        parent::setUp();

        if (!interface_exists('QueuedJob')) {
            $this->skipTest = true;
            $this->markTestSkipped("These tests need the QueuedJobs module installed to run");
        }

        if (class_exists('Subsite')) {
            $this->skipTest = true;
            $this->markTestSkipped(get_class() . ' skipped when running with subsites');
        }

        SS_Datetime::set_mock_now('2015-05-07 06:00:00');

        Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_size', 5);
        Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 0);
        Config::inst()->update('SearchUpdateCommitJobProcessor', 'cooldown', 600);

        Versioned::reading_stage("Stage");

        Injector::inst()->registerService(new BatchedProcessor_QueuedJobService(), 'QueuedJobService');

        FullTextSearch::force_index_list('BatchedProcessorTest_Index');

        SearchUpdateCommitJobProcessor::$dirty_indexes = array();
        SearchUpdateCommitJobProcessor::$has_run = false;

        $this->oldProcessor = SearchUpdater::$processor;
        SearchUpdater::$processor = new SearchUpdateQueuedJobProcessor();
    }

    public function tearDown()
    {
        if ($this->oldProcessor) {
            SearchUpdater::$processor = $this->oldProcessor;
        }
        FullTextSearch::force_index_list();
        parent::tearDown();
    }

    /**
     * @return SearchUpdateQueuedJobProcessor
     */
    protected function generateDirtyIds()
    {
        $processor = SearchUpdater::$processor;
        for ($id = 1; $id <= 42; $id++) {
            // Save to db
            $object = new BatchedProcessorTest_Object();
            $object->TestText = 'Object ' . $id;
            $object->write();
            // Add to index manually
            $processor->addDirtyIDs(
                'BatchedProcessorTest_Object',
                array(array(
                    'id' => $id,
                    'state' => array('SearchVariantVersioned' => 'Stage')
                )),
                'BatchedProcessorTest_Index'
            );
        }
        $processor->batchData();
        return $processor;
    }

    /**
     * Tests that large jobs are broken up into a suitable number of batches
     */
    public function testBatching()
    {
        $index = singleton('BatchedProcessorTest_Index');
        $index->reset();
        $processor = $this->generateDirtyIds();

        // Check initial state
        $data = $processor->getJobData();
        $this->assertEquals(9, $data->totalSteps);
        $this->assertEquals(0, $data->currentStep);
        $this->assertEmpty($data->isComplete);
        $this->assertEquals(0, count($index->getAdded()));

        // Advance state
        for ($pass = 1; $pass <= 8; $pass++) {
            $processor->process();
            $data = $processor->getJobData();
            $this->assertEquals($pass, $data->currentStep);
            $this->assertEquals($pass * 5, count($index->getAdded()));
        }

        // Last run should have two hanging items
        $processor->process();
        $data = $processor->getJobData();
        $this->assertEquals(9, $data->currentStep);
        $this->assertEquals(42, count($index->getAdded()));
        $this->assertTrue($data->isComplete);

        // Check any additional queued jobs
        $processor->afterComplete();
        $service = singleton('QueuedJobService');
        $jobs = $service->getJobs();
        $this->assertEquals(1, count($jobs));
        $this->assertInstanceOf('SearchUpdateCommitJobProcessor', $jobs[0]['job']);
    }

    /**
     * Test creation of multiple commit jobs
     */
    public function testMultipleCommits()
    {
        $index = singleton('BatchedProcessorTest_Index');
        $index->reset();

        // Test that running a commit immediately after submitting to the indexes
        // correctly commits
        $first = SearchUpdateCommitJobProcessor::queue();
        $second = SearchUpdateCommitJobProcessor::queue();

        $this->assertFalse($index->getIsCommitted());

        // First process will cause the commit
        $this->assertFalse($first->jobFinished());
        $first->process();
        $allMessages = $first->getMessages();
        $this->assertTrue($index->getIsCommitted());
        $this->assertTrue($first->jobFinished());
        $this->assertStringEndsWith('All indexes committed', $allMessages[2]);

        // Executing the subsequent processor should not re-trigger a commit
        $index->reset();
        $this->assertFalse($second->jobFinished());
        $second->process();
        $allMessages = $second->getMessages();
        $this->assertFalse($index->getIsCommitted());
        $this->assertTrue($second->jobFinished());
        $this->assertStringEndsWith('Indexing already completed this request: Discarding this job', $allMessages[0]);

        // Given that a third job is created, and the indexes are dirtied, attempting to run this job
        // should result in a delay
        $index->reset();
        $third = SearchUpdateCommitJobProcessor::queue();
        $this->assertFalse($third->jobFinished());
        $third->process();
        $this->assertTrue($third->jobFinished());
        $allMessages = $third->getMessages();
        $this->assertStringEndsWith(
            'Indexing already run this request, but incomplete. Re-scheduling for 2015-05-07 06:10:00',
            $allMessages[0]
        );
    }


    /**
     * Tests that the batch_soft_cap setting is properly respected
     */
    public function testSoftCap()
    {
        $index = singleton('BatchedProcessorTest_Index');
        $index->reset();
        $processor = $this->generateDirtyIds();

        // Test that increasing the soft cap to 2 will reduce the number of batches
        Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 2);
        $processor->batchData();
        $data = $processor->getJobData();
        //Debug::dump($data);die;
        $this->assertEquals(8, $data->totalSteps);

        // A soft cap of 1 should not fit in the hanging two items
        Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 1);
        $processor->batchData();
        $data = $processor->getJobData();
        $this->assertEquals(9, $data->totalSteps);

        // Extra large soft cap should fit both items
        Config::inst()->update('SearchUpdateBatchedProcessor', 'batch_soft_cap', 4);
        $processor->batchData();
        $data = $processor->getJobData();
        $this->assertEquals(8, $data->totalSteps);

        // Process all data and ensure that all are processed adequately
        for ($pass = 1; $pass <= 8; $pass++) {
            $processor->process();
        }
        $data = $processor->getJobData();
        $this->assertEquals(8, $data->currentStep);
        $this->assertEquals(42, count($index->getAdded()));
        $this->assertTrue($data->isComplete);
    }
}