silverstripe-fulltextsearch/tests/SolrReindexTest.php
2022-04-13 11:24:03 +12:00

498 lines
20 KiB
PHP

<?php
namespace SilverStripe\FullTextSearch\Tests;
use Apache_Solr_Document;
use Page;
use SilverStripe\Assets\File;
use SilverStripe\CMS\Model\SiteTree;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Dev\SapphireTest;
use SilverStripe\FullTextSearch\Search\FullTextSearch;
use SilverStripe\FullTextSearch\Search\Services\SearchableService;
use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater;
use SilverStripe\FullTextSearch\Search\Variants\SearchVariant;
use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned;
use SilverStripe\FullTextSearch\Solr\Reindex\Handlers\SolrReindexHandler;
use SilverStripe\FullTextSearch\Solr\Reindex\Handlers\SolrReindexImmediateHandler;
use SilverStripe\FullTextSearch\Solr\Services\Solr4Service;
use SilverStripe\FullTextSearch\Solr\Services\SolrService;
use SilverStripe\FullTextSearch\Solr\Tasks\Solr_Reindex;
use SilverStripe\FullTextSearch\Tests\SearchVariantVersionedTest\SearchVariantVersionedTest_Item;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_MyDataObjectOne;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_MyDataObjectTwo;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_MyPage;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_ShowInSearchIndex;
use SilverStripe\FullTextSearch\Tests\SolrReindexTest\SolrReindexTest_Index;
use SilverStripe\FullTextSearch\Tests\SolrReindexTest\SolrReindexTest_Item;
use SilverStripe\FullTextSearch\Tests\SolrReindexTest\SolrReindexTest_RecordingLogger;
use SilverStripe\FullTextSearch\Tests\SolrReindexTest\SolrReindexTest_TestHandler;
use SilverStripe\FullTextSearch\Tests\SolrReindexTest\SolrReindexTest_Variant;
use SilverStripe\Versioned\Versioned;
class SolrReindexTest extends SapphireTest
{
protected $usesDatabase = true;
protected static $extra_dataobjects = array(
SolrReindexTest_Item::class,
SolrIndexTest_MyPage::class,
SolrIndexTest_MyDataObjectOne::class,
SolrIndexTest_MyDataObjectTwo::class,
);
/**
* Forced index for testing
*
* @var SolrReindexTest_Index
*/
protected $index = null;
/**
* Mock service
*
* @var SolrService
*/
protected $service = null;
protected function setUp(): void
{
parent::setUp();
// Set test handler for reindex
Config::modify()->set(Injector::class, SolrReindexHandler::class, array(
'class' => SolrReindexTest_TestHandler::class
));
Injector::inst()->registerService(new SolrReindexTest_TestHandler(), SolrReindexHandler::class);
// Set test variant
SolrReindexTest_Variant::enable();
// Set index list
$this->service = $this->getServiceMock();
$this->index = singleton(SolrReindexTest_Index::class);
$this->index->setService($this->service);
FullTextSearch::force_index_list($this->index);
}
/**
* Populate database with dummy dataset
*
* @param int $number Number of records to create in each variant
*/
protected function createDummyData($number)
{
// Note that we don't create any records in variant = 2, to represent a variant
// that should be cleared without any re-indexes performed
foreach ([0, 1] as $variant) {
for ($i = 1; $i <= $number; $i++) {
$item = new SolrReindexTest_Item();
$item->Variant = $variant;
$item->Title = "Item $variant / $i";
$item->write();
}
}
}
/**
* Mock service
*
* @return SolrService
*/
protected function getServiceMock()
{
$serviceMock = $this->getMockBuilder(Solr4Service::class)
->setMethods(['deleteByQuery', 'addDocument']);
return $serviceMock->getMock();
}
protected function tearDown(): void
{
FullTextSearch::force_index_list();
SolrReindexTest_Variant::disable();
parent::tearDown();
}
/**
* Get the reindex handler
*
* @return SolrReindexHandler
*/
protected function getHandler()
{
return Injector::inst()->get(SolrReindexHandler::class);
}
/**
* Ensure the test variant is up and running properly
*/
public function testVariant()
{
// State defaults to 0
$variant = SearchVariant::current_state();
$this->assertEquals(
array(
SolrReindexTest_Variant::class => "0"
),
$variant
);
// All states enumerated
$allStates = iterator_to_array(SearchVariant::reindex_states());
$this->assertEquals(
array(
array(
SolrReindexTest_Variant::class => "0"
),
array(
SolrReindexTest_Variant::class => "1"
),
array(
SolrReindexTest_Variant::class => "2"
)
),
$allStates
);
// Check correct items created and that filtering on variant works
$this->createDummyData(120);
SolrReindexTest_Variant::set_current(2);
$this->assertEquals(0, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(1);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::set_current(0);
$this->assertEquals(120, SolrReindexTest_Item::get()->count());
SolrReindexTest_Variant::disable();
$this->assertEquals(240, SolrReindexTest_Item::get()->count());
}
/**
* Given the invocation of a new re-index with a given set of data, ensure that the necessary
* list of groups are created and segmented for each state
*
* Test should work fine with any variants (versioned, subsites, etc) specified
*/
public function testReindexSegmentsGroups()
{
$this->service->method('deleteByQuery')
->withConsecutive(
['-(ClassHierarchy:' . SolrReindexTest_Item::class . ')'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +(_testvariant:"2")']
);
$this->createDummyData(120);
// Initiate re-index
$logger = new SolrReindexTest_RecordingLogger();
$this->getHandler()->runReindex($logger, 21, Solr_Reindex::class);
// Test that invalid classes are removed
$this->assertContains(
'Clearing obsolete classes from ' . str_replace('\\', '-', SolrReindexTest_Index::class),
$logger->getMessages()
);
// Test that valid classes in invalid variants are removed
$this->assertContains(
'Clearing all records of type ' . SolrReindexTest_Item::class . ' in the current state: {'
. json_encode(SolrReindexTest_Variant::class) . ':"2"}',
$logger->getMessages()
);
// 120x2 grouped into groups of 21 results in 12 groups
$this->assertEquals(12, $logger->countMessages('Called processGroup with '));
$this->assertEquals(6, $logger->countMessages('{' . json_encode(SolrReindexTest_Variant::class) . ':"0"}'));
$this->assertEquals(6, $logger->countMessages('{' . json_encode(SolrReindexTest_Variant::class) . ':"1"}'));
// Given that there are two variants, there should be two group ids of each number
$this->assertEquals(2, $logger->countMessages(' ' . SolrReindexTest_Item::class . ', group 0 of 6'));
$this->assertEquals(2, $logger->countMessages(' ' . SolrReindexTest_Item::class . ', group 1 of 6'));
$this->assertEquals(2, $logger->countMessages(' ' . SolrReindexTest_Item::class . ', group 2 of 6'));
$this->assertEquals(2, $logger->countMessages(' ' . SolrReindexTest_Item::class . ', group 3 of 6'));
$this->assertEquals(2, $logger->countMessages(' ' . SolrReindexTest_Item::class . ', group 4 of 6'));
$this->assertEquals(2, $logger->countMessages(' ' . SolrReindexTest_Item::class . ', group 5 of 6'));
// Check various group sizes
$logger->clear();
$this->getHandler()->runReindex($logger, 120, 'Solr_Reindex');
$this->assertEquals(2, $logger->countMessages('Called processGroup with '));
$logger->clear();
$this->getHandler()->runReindex($logger, 119, 'Solr_Reindex');
$this->assertEquals(4, $logger->countMessages('Called processGroup with '));
$logger->clear();
$this->getHandler()->runReindex($logger, 121, 'Solr_Reindex');
$this->assertEquals(2, $logger->countMessages('Called processGroup with '));
$logger->clear();
$this->getHandler()->runReindex($logger, 2, 'Solr_Reindex');
$this->assertEquals(120, $logger->countMessages('Called processGroup with '));
}
/**
* Test index processing on individual groups
*/
public function testRunGroup()
{
$classesToSkip = [SolrReindexTest_Item::class];
Config::modify()->set(SearchableService::class, 'indexing_canview_exclude_classes', $classesToSkip);
$this->service->method('deleteByQuery')
->with('+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")');
$this->createDummyData(120);
$logger = new SolrReindexTest_RecordingLogger();
// Initiate re-index of third group (index 2 of 6)
$state = array(SolrReindexTest_Variant::class => '1');
$this->getHandler()->runGroup($logger, $this->index, $state, SolrReindexTest_Item::class, 6, 2);
$idMessage = $logger->filterMessages('Updated ');
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/i', $idMessage[0] ?? '', $matches));
$ids = array_unique(explode(',', $matches['ids'] ?? ''));
// Test successful
$this->assertNotEmpty($logger->getMessages('Adding ' . SolrReindexTest_Item::class));
$this->assertNotEmpty($logger->getMessages('Done'));
// Test that items in this variant / group are re-indexed
// 120 divided into 6 groups should be 20 at least (max 21)
$c = count($ids ?? []);
$this->assertTrue($c === 20 || $c === 21, 'Group size is about 20');
foreach ($ids as $id) {
// Each id should be % 6 == 2
$this->assertEquals(2, $id % 6, "ID $id Should match pattern ID % 6 = 2");
}
}
/**
* Test that running all groups covers the entire range of dataobject IDs
*/
public function testRunAllGroups()
{
$classesToSkip = [SolrReindexTest_Item::class];
Config::modify()->set(SearchableService::class, 'indexing_canview_exclude_classes', $classesToSkip);
$this->service->method('deleteByQuery')
->withConsecutive(
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=0 u=0}mod(ID, 6)" +(_testvariant:"1")'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=1 u=1}mod(ID, 6)" +(_testvariant:"1")'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=2 u=2}mod(ID, 6)" +(_testvariant:"1")'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=3 u=3}mod(ID, 6)" +(_testvariant:"1")'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=4 u=4}mod(ID, 6)" +(_testvariant:"1")'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=5 u=5}mod(ID, 6)" +(_testvariant:"1")'],
['+(ClassHierarchy:' . SolrReindexTest_Item::class . ') +_query_:"{!frange l=6 u=6}mod(ID, 6)" +(_testvariant:"1")']
);
$this->createDummyData(120);
$logger = new SolrReindexTest_RecordingLogger();
// Test that running all groups covers the complete set of ids
$state = array(SolrReindexTest_Variant::class => '1');
for ($i = 0; $i < 6; $i++) {
// See testReindexSegmentsGroups for test that each of these states is invoked during a full reindex
$this
->getHandler()
->runGroup($logger, $this->index, $state, SolrReindexTest_Item::class, 6, $i);
}
// Count all ids updated
$ids = array();
foreach ($logger->filterMessages('Updated ') as $message) {
$this->assertNotEmpty(preg_match('/^Updated (?<ids>[,\d]+)/', $message ?? '', $matches));
$ids = array_unique(array_merge($ids, explode(',', $matches['ids'] ?? '')));
}
// Check ids
$this->assertEquals(120, count($ids ?? []));
}
/**
* Test that ShowInSearch filtering is working correctly
*/
public function testShowInSearch()
{
// allow anonymous users to assess draft-only content to pass canView() check (will auto-reset for next test)
Versioned::set_draft_site_secured(false);
Versioned::set_reading_mode('Stage.' . Versioned::DRAFT);
Config::modify()->set(SearchableService::class, 'variant_state_draft_excluded', false);
// will get added
$pageA = new Page();
$pageA->Title = 'Test Page true';
$pageA->ShowInSearch = true;
$pageA->write();
// will get filtered out
$page = new Page();
$page->Title = 'Test Page false';
$page->ShowInSearch = false;
$page->write();
// will get added
$fileA = new File();
$fileA->Title = 'Test File true';
$fileA->ShowInSearch = true;
$fileA->write();
// will get filtered out
$file = new File();
$file->Title = 'Test File false';
$file->ShowInSearch = false;
$file->write();
// will get added
$objOneA = new SolrIndexTest_MyDataObjectOne();
$objOneA->Title = 'Test MyDataObjectOne true';
$objOneA->ShowInSearch = true;
$objOneA->write();
// will get filtered out
$objOne = new SolrIndexTest_MyDataObjectOne();
$objOne->Title = 'Test MyDataObjectOne false';
$objOne->ShowInSearch = false;
$objOne->write();
// will get added
// this class has a getShowInSearch() == true, which will override $mypage->ShowInSearch = false
$objTwoA = new SolrIndexTest_MyDataObjectTwo();
$objTwoA->Title = 'Test MyDataObjectTwo false';
$objTwoA->ShowInSearch = false;
$objTwoA->write();
// will get added
// this class has a getShowInSearch() == true, which will override $mypage->ShowInSearch = false
$myPageA = new SolrIndexTest_MyPage();
$myPageA->Title = 'Test MyPage false';
$myPageA->ShowInSearch = false;
$myPageA->write();
$serviceMock = $this->getMockBuilder(Solr4Service::class)
->setMethods(['addDocument', 'deleteByQuery'])
->getMock();
$index = new SolrIndexTest_ShowInSearchIndex();
$index->setService($serviceMock);
FullTextSearch::force_index_list($index);
$callback = function (Apache_Solr_Document $doc) use ($pageA, $myPageA, $fileA, $objOneA, $objTwoA): bool {
$validKeys = [
Page::class . $pageA->ID,
SolrIndexTest_MyPage::class . $myPageA->ID,
File::class . $fileA->ID,
SolrIndexTest_MyDataObjectOne::class . $objOneA->ID,
SolrIndexTest_MyDataObjectTwo::class . $objTwoA->ID
];
return in_array($this->createSolrDocKey($doc), $validKeys ?? []);
};
$serviceMock
->expects($this->exactly(5))
->method('addDocument')
->withConsecutive(
[$this->callback($callback)],
[$this->callback($callback)],
[$this->callback($callback)],
[$this->callback($callback)],
[$this->callback($callback)]
);
$logger = new SolrReindexTest_RecordingLogger();
$state = [SearchVariantVersioned::class => Versioned::DRAFT];
$handler = Injector::inst()->get(SolrReindexImmediateHandler::class);
$handler->runGroup($logger, $index, $state, SiteTree::class, 1, 0);
$handler->runGroup($logger, $index, $state, File::class, 1, 0);
$handler->runGroup($logger, $index, $state, SolrIndexTest_MyDataObjectOne::class, 1, 0);
}
/**
* Test that CanView filtering is working correctly
*/
public function testCanView()
{
// allow anonymous users to assess draft-only content to pass canView() check (will auto-reset for next test)
Versioned::set_draft_site_secured(false);
Versioned::set_reading_mode('Stage.' . Versioned::DRAFT);
Config::modify()->set(SearchableService::class, 'variant_state_draft_excluded', false);
// will get added
$pageA = new Page();
$pageA->Title = 'Test Page Anyone';
$pageA->CanViewType = 'Anyone';
$pageA->write();
// will get filtered out
$page = new Page();
$page->Title = 'Test Page LoggedInUsers';
$page->CanViewType = 'LoggedInUsers';
$page->write();
// will get added
$fileA = new File();
$fileA->Title = 'Test File Anyone';
$fileA->CanViewType = 'Anyone';
$fileA->write();
// will get filtered out
$file = new File();
$file->Title = 'Test File LoggedInUsers';
$file->CanViewType = 'LoggedInUsers';
$file->write();
// will get added
$objOneA = new SolrIndexTest_MyDataObjectOne();
$objOneA->Title = 'Test MyDataObjectOne true';
$objOneA->CanViewValue = true;
$objOneA->ShowInSearch = true;
$objOneA->write();
// will get filtered out
$objOne = new SolrIndexTest_MyDataObjectOne();
$objOne->Title = 'Test MyDataObjectOne false';
$objOne->CanViewValue = false;
$objOneA->ShowInSearch = true;
$objOne->write();
$serviceMock = $this->getMockBuilder(Solr4Service::class)
->setMethods(['addDocument', 'deleteByQuery'])
->getMock();
$index = new SolrIndexTest_ShowInSearchIndex();
$index->setService($serviceMock);
FullTextSearch::force_index_list($index);
$callback = function (Apache_Solr_Document $doc) use ($pageA, $fileA, $objOneA): bool {
$validKeys = [
Page::class . $pageA->ID,
File::class . $fileA->ID,
SolrIndexTest_MyDataObjectOne::class . $objOneA->ID,
];
$solrDocKey = $this->createSolrDocKey($doc);
return in_array($this->createSolrDocKey($doc), $validKeys ?? []);
};
$serviceMock
->expects($this->exactly(3))
->method('addDocument')
->withConsecutive(
[$this->callback($callback)],
[$this->callback($callback)],
[$this->callback($callback)]
);
$logger = new SolrReindexTest_RecordingLogger();
$state = [SearchVariantVersioned::class => Versioned::DRAFT];
$handler = Injector::inst()->get(SolrReindexImmediateHandler::class);
$handler->runGroup($logger, $index, $state, SiteTree::class, 1, 0);
$handler->runGroup($logger, $index, $state, File::class, 1, 0);
$handler->runGroup($logger, $index, $state, SolrIndexTest_MyDataObjectOne::class, 1, 0);
}
protected function createSolrDocKey(Apache_Solr_Document $doc)
{
return $doc->getField('ClassName')['value'] . $doc->getField('ID')['value'];
}
}