silverstripe-fulltextsearch/tests/SolrIndexTest.php

<?php

namespace SilverStripe\FullTextSearch\Tests;

use Apache_Solr_Document;
use Page;
use SebastianBergmann\Version;
use SilverStripe\Assets\File;
use SilverStripe\CMS\Model\SiteTree;
use SilverStripe\Core\Config\Config;
use SilverStripe\Core\Environment;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Core\Kernel;
use SilverStripe\Dev\SapphireTest;
use SilverStripe\FullTextSearch\Search\FullTextSearch;
use SilverStripe\FullTextSearch\Search\Queries\SearchQuery;
use SilverStripe\FullTextSearch\Search\Services\IndexableService;
use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater;
use SilverStripe\FullTextSearch\Search\Variants\SearchVariantSubsites;
use SilverStripe\FullTextSearch\Solr\SolrIndex;
use SilverStripe\FullTextSearch\Solr\Services\Solr3Service;
use SilverStripe\FullTextSearch\Solr\Services\Solr4Service;
use SilverStripe\FullTextSearch\Tests\SearchUpdaterTest\SearchUpdaterTest_Container;
use SilverStripe\FullTextSearch\Tests\SearchUpdaterTest\SearchUpdaterTest_HasOne;
use SilverStripe\FullTextSearch\Tests\SearchUpdaterTest\SearchUpdaterTest_HasMany;
use SilverStripe\FullTextSearch\Tests\SearchUpdaterTest\SearchUpdaterTest_ManyMany;
use SilverStripe\FullTextSearch\Tests\SearchUpdaterTest\SearchUpdaterTest_OtherContainer;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_AmbiguousRelationIndex;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_AmbiguousRelationInheritedIndex;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_BoostedIndex;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_FakeIndex;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_FakeIndex2;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_ShowInSearchIndex;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_MyPage;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_MyDataObjectOne;
use SilverStripe\FullTextSearch\Tests\SolrIndexTest\SolrIndexTest_MyDataObjectTwo;
use SilverStripe\ORM\DataObject;
use SilverStripe\Subsites\Model\Subsite;
use SilverStripe\Versioned\Versioned;

class SolrIndexTest extends SapphireTest
{

    protected $usesDatabase = true;

    protected static $extra_dataobjects = [
        SolrIndexTest_MyPage::class,
        SolrIndexTest_MyDataObjectOne::class,
        SolrIndexTest_MyDataObjectTwo::class,
    ];

    public function testFieldDataHasOne()
    {
        $index = new SolrIndexTest_FakeIndex();
        $data = $index->fieldData('HasOneObject.Field1');

        $data = $data[SearchUpdaterTest_Container::class . '_HasOneObject_Field1'];

        $this->assertEquals(SearchUpdaterTest_Container::class, $data['origin']);
        $this->assertEquals(SearchUpdaterTest_Container::class, $data['base']);
        $this->assertEquals(SearchUpdaterTest_HasOne::class, $data['class']);
    }

    public function testFieldDataHasMany()
    {
        $index = new SolrIndexTest_FakeIndex();
        $data = $index->fieldData('HasManyObjects.Field1');
        $data = $data[SearchUpdaterTest_Container::class . '_HasManyObjects_Field1'];

        $this->assertEquals(SearchUpdaterTest_Container::class, $data['origin']);
        $this->assertEquals(SearchUpdaterTest_Container::class, $data['base']);
        $this->assertEquals(SearchUpdaterTest_HasMany::class, $data['class']);
    }

    public function testFieldDataManyMany()
    {
        $index = new SolrIndexTest_FakeIndex();
        $data = $index->fieldData('ManyManyObjects.Field1');
        $data = $data[SearchUpdaterTest_Container::class . '_ManyManyObjects_Field1'];

        $this->assertEquals(SearchUpdaterTest_Container::class, $data['origin']);
        $this->assertEquals(SearchUpdaterTest_Container::class, $data['base']);
        $this->assertEquals(SearchUpdaterTest_ManyMany::class, $data['class']);
    }

    public function testFieldDataAmbiguousHasMany()
    {
        $index = new SolrIndexTest_AmbiguousRelationIndex();
        $data = $index->fieldData('HasManyObjects.Field1');

        $this->assertArrayHasKey(SearchUpdaterTest_Container::class . '_HasManyObjects_Field1', $data);
        $this->assertArrayHasKey(SearchUpdaterTest_OtherContainer::class . '_HasManyObjects_Field1', $data);

        $dataContainer = $data[SearchUpdaterTest_Container::class . '_HasManyObjects_Field1'];
        $this->assertEquals(SearchUpdaterTest_Container::class, $dataContainer['origin']);
        $this->assertEquals(SearchUpdaterTest_Container::class, $dataContainer['base']);
        $this->assertEquals(SearchUpdaterTest_HasMany::class, $dataContainer['class']);

        $dataOtherContainer = $data[SearchUpdaterTest_OtherContainer::class . '_HasManyObjects_Field1'];
        $this->assertEquals(SearchUpdaterTest_OtherContainer::class, $dataOtherContainer['origin']);
        $this->assertEquals(SearchUpdaterTest_OtherContainer::class, $dataOtherContainer['base']);
        $this->assertEquals(SearchUpdaterTest_HasMany::class, $dataOtherContainer['class']);
    }

    public function testFieldDataAmbiguousManyMany()
    {
        $index = new SolrIndexTest_AmbiguousRelationIndex();
        $data = $index->fieldData('ManyManyObjects.Field1');

        $this->assertArrayHasKey(SearchUpdaterTest_Container::class . '_ManyManyObjects_Field1', $data);
        $this->assertArrayHasKey(SearchUpdaterTest_OtherContainer::class . '_ManyManyObjects_Field1', $data);

        $dataContainer = $data[SearchUpdaterTest_Container::class . '_ManyManyObjects_Field1'];
        $this->assertEquals(SearchUpdaterTest_Container::class, $dataContainer['origin']);
        $this->assertEquals(SearchUpdaterTest_Container::class, $dataContainer['base']);
        $this->assertEquals(SearchUpdaterTest_ManyMany::class, $dataContainer['class']);

        $dataOtherContainer = $data[SearchUpdaterTest_OtherContainer::class . '_ManyManyObjects_Field1'];
        $this->assertEquals(SearchUpdaterTest_OtherContainer::class, $dataOtherContainer['origin']);
        $this->assertEquals(SearchUpdaterTest_OtherContainer::class, $dataOtherContainer['base']);
        $this->assertEquals(SearchUpdaterTest_ManyMany::class, $dataOtherContainer['class']);
    }

    public function testFieldDataAmbiguousManyManyInherited()
    {
        $index = new SolrIndexTest_AmbiguousRelationInheritedIndex();
        $data = $index->fieldData('ManyManyObjects.Field1');

        $this->assertArrayHasKey(SearchUpdaterTest_Container::class . '_ManyManyObjects_Field1', $data);
        $this->assertArrayHasKey(SearchUpdaterTest_OtherContainer::class . '_ManyManyObjects_Field1', $data);
        $this->assertArrayNotHasKey(SearchUpdaterTest_ExtendedContainer::class . '_ManyManyObjects_Field1', $data);

        $dataContainer = $data[SearchUpdaterTest_Container::class . '_ManyManyObjects_Field1'];
        $this->assertEquals(SearchUpdaterTest_Container::class, $dataContainer['origin']);
        $this->assertEquals(SearchUpdaterTest_Container::class, $dataContainer['base']);
        $this->assertEquals(SearchUpdaterTest_ManyMany::class, $dataContainer['class']);

        $dataOtherContainer = $data[SearchUpdaterTest_OtherContainer::class . '_ManyManyObjects_Field1'];
        $this->assertEquals(SearchUpdaterTest_OtherContainer::class, $dataOtherContainer['origin']);
        $this->assertEquals(SearchUpdaterTest_OtherContainer::class, $dataOtherContainer['base']);
        $this->assertEquals(SearchUpdaterTest_ManyMany::class, $dataOtherContainer['class']);
    }

    /**
     * Test boosting on SearchQuery
     */
    public function testBoostedQuery()
    {
        /** @var Solr3Service|PHPUnit_Framework_MockObject_MockObject $serviceMock */
        $serviceMock = $this->getMockBuilder(Solr3Service::class)
            ->setMethods(['search'])
            ->getMock();

        $serviceMock->expects($this->once())
            ->method('search')
            ->with(
                $this->equalTo('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)'),
                $this->anything(),
                $this->anything(),
                $this->anything(),
                $this->anything()
            )->willReturn($this->getFakeRawSolrResponse());

        $index = new SolrIndexTest_FakeIndex();
        $index->setService($serviceMock);

        $query = new SearchQuery();
        $query->addSearchTerm(
            'term',
            null,
            array('Field1' => 1.5, 'HasOneObject_Field1' => 3)
        );
        $index->search($query);
    }

    /**
     * Test boosting on field schema (via queried fields parameter)
     */
    public function testBoostedField()
    {
        if (class_exists(Subsite::class)) {
            Config::modify()->set(SearchVariantSubsites::class, 'enabled', false);
        }

        /** @var Solr3Service|PHPUnit_Framework_MockObject_MockObject $serviceMock */
        $serviceMock = $this->getMockBuilder(Solr3Service::class)
            ->setMethods(['search'])
            ->getMock();

        $serviceMock->expects($this->once())
            ->method('search')
            ->with(
                $this->equalTo('+term'),
                $this->anything(),
                $this->anything(),
                $this->equalTo([
                    'qf' => SearchUpdaterTest_Container::class . '_Field1^1.5 '
                        . SearchUpdaterTest_Container::class . '_Field2^2.1 _text',
                    'fq' => '',
                ]),
                $this->anything()
            )->willReturn($this->getFakeRawSolrResponse());

        $index = new SolrIndexTest_BoostedIndex();
        $index->setService($serviceMock);

        $query = new SearchQuery();
        $query->addSearchTerm('term');
        $index->search($query);
    }

    public function testHighlightQueryOnBoost()
    {
        /** @var SilverStripe\FullTextSearch\Solr\Services\Solr3Service|ObjectProphecy $serviceMock */
        $serviceMock = $this->getMockBuilder(Solr3Service::class)
            ->setMethods(['search'])
            ->getMock();

        $serviceMock->expects($this->exactly(2))
            ->method('search')
            ->withConsecutive(
                [
                    $this->equalTo('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)'),
                    $this->anything(),
                    $this->anything(),
                    $this->logicalNot(
                        $this->arrayHasKey('hl.q')
                    ),
                    $this->anything()
                ],
                [
                    $this->equalTo('+(Field1:term^1.5 OR HasOneObject_Field1:term^3)'),
                    $this->anything(),
                    $this->anything(),
                    $this->arrayHasKey('hl.q'),
                    $this->anything()
                ]
            )->willReturn($this->getFakeRawSolrResponse());

        $index = new SolrIndexTest_FakeIndex();
        $index->setService($serviceMock);

        // Search without highlighting
        $query = new SearchQuery();
        $query->addSearchTerm(
            'term',
            null,
            array('Field1' => 1.5, 'HasOneObject_Field1' => 3)
        );
        $index->search($query);

        // Search with highlighting
        $query = new SearchQuery();
        $query->addSearchTerm(
            'term',
            null,
            array('Field1' => 1.5, 'HasOneObject_Field1' => 3)
        );
        $index->search($query, -1, -1, array('hl' => true));
    }

    public function testIndexExcludesNullValues()
    {
        /** @var Solr3Service|ObjectProphecy $serviceMock */
        $serviceMock = $this->createMock(Solr3Service::class);
        $index = new SolrIndexTest_FakeIndex();
        $index->setService($serviceMock);
        $obj = new SearchUpdaterTest_Container();

        $obj->Field1 = 'Field1 val';
        $obj->Field2 = null;
        $obj->MyDate = null;
        $docs = $index->add($obj);
        $value = $docs[0]->getField(SearchUpdaterTest_Container::class . '_Field1');
        $this->assertEquals('Field1 val', $value['value'], 'Writes non-NULL string fields');
        $value = $docs[0]->getField(SearchUpdaterTest_Container::class . '_Field2');
        $this->assertFalse($value, 'Ignores string fields if they are NULL');
        $value = $docs[0]->getField(SearchUpdaterTest_Container::class . '_MyDate');
        $this->assertFalse($value, 'Ignores date fields if they are NULL');

        $obj->MyDate = '2010-12-30';
        $docs = $index->add($obj);
        $value = $docs[0]->getField(SearchUpdaterTest_Container::class . '_MyDate');
        $this->assertEquals('2010-12-30T00:00:00Z', $value['value'], 'Writes non-NULL dates');
    }

    public function testAddFieldExtraOptions()
    {
        Injector::inst()->get(Kernel::class)->setEnvironment('live');

        $index = new SolrIndexTest_FakeIndex();

        $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
        $defField1 = $defs->xpath('field[@name="' . SearchUpdaterTest_Container::class . '_Field1"]');
        $this->assertEquals((string)$defField1[0]['stored'], 'false');

        $index->addFilterField('Field1', null, array('stored' => 'true'));
        $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
        $defField1 = $defs->xpath('field[@name="' . SearchUpdaterTest_Container::class . '_Field1"]');
        $this->assertEquals((string)$defField1[0]['stored'], 'true');
    }

    public function testAddAnalyzer()
    {
        $index = new SolrIndexTest_FakeIndex();

        $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
        $defField1 = $defs->xpath('field[@name="' . SearchUpdaterTest_Container::class . '_Field1"]');
        $analyzers = $defField1[0]->analyzer;
        $this->assertFalse((bool)$analyzers);

        $index->addAnalyzer('Field1', 'charFilter', array('class' => 'solr.HTMLStripCharFilterFactory'));
        $defs = simplexml_load_string('<fields>' . $index->getFieldDefinitions() . '</fields>');
        $defField1 = $defs->xpath('field[@name="' . SearchUpdaterTest_Container::class . '_Field1"]');
        $analyzers = $defField1[0]->analyzer;
        $this->assertTrue((bool)$analyzers);
        $this->assertEquals('solr.HTMLStripCharFilterFactory', $analyzers[0]->charFilter[0]['class']);
    }

    public function testAddCopyField()
    {
        $index = new SolrIndexTest_FakeIndex();
        $index->addCopyField('sourceField', 'destField');

        $defs = simplexml_load_string('<fields>' . $index->getCopyFieldDefinitions() . '</fields>');
        $copyField = $defs->xpath('copyField');

        $this->assertEquals('sourceField', $copyField[0]['source']);
        $this->assertEquals('destField', $copyField[0]['dest']);
    }

    /**
     * Tests the setting of the 'stored' flag
     */
    public function testStoredFields()
    {
        // Test two fields
        $index = new SolrIndexTest_FakeIndex2();
        $index->addStoredField('Field1');
        $index->addFulltextField('Field2');
        $schema = $index->getFieldDefinitions();
        $this->assertContains(
            "<field name='" . SearchUpdaterTest_Container::class . "_Field1' type='text' indexed='true' stored='true'",
            $schema
        );
        $this->assertContains(
            "<field name='" . SearchUpdaterTest_Container::class . "_Field2' type='text' indexed='true' stored='false'",
            $schema
        );

        // Test with addAllFulltextFields
        $index2 = new SolrIndexTest_FakeIndex2();
        $index2->addAllFulltextFields();
        $index2->addStoredField('Field2');
        $schema2 = $index2->getFieldDefinitions();
        $this->assertContains(
            "<field name='" . SearchUpdaterTest_Container::class . "_Field1' type='text' indexed='true' stored='false'",
            $schema2
        );
        $this->assertContains(
            "<field name='" . SearchUpdaterTest_Container::class . "_Field2' type='text' indexed='true' stored='true'",
            $schema2
        );
    }

    public function testSanitiseClassName()
    {
        $index = new SolrIndexTest_FakeIndex2;

        $this->assertSame(
            'SilverStripe\\\\FullTextSearch\\\\Tests\\\\SolrIndexTest',
            $index->sanitiseClassName(static::class)
        );

        $this->assertSame(
            'SilverStripe-FullTextSearch-Tests-SolrIndexTest',
            $index->sanitiseClassName(static::class, '-')
        );
    }

    public function testGetIndexName()
    {
        $index = new SolrIndexTest_FakeIndex2;
        $this->assertSame(
            'SilverStripe-FullTextSearch-Tests-SolrIndexTest-SolrIndexTest_FakeIndex2',
            $index->getIndexName()
        );
    }

    public function testGetIndexNameWithPrefixAndSuffixFromEnvironment()
    {
        $index = new SolrIndexTest_FakeIndex2;

        Environment::putEnv('SS_SOLR_INDEX_PREFIX="foo_"');
        Environment::putEnv('SS_SOLR_INDEX_SUFFIX="_bar"');

        $this->assertSame(
            'foo_SilverStripe-FullTextSearch-Tests-SolrIndexTest-SolrIndexTest_FakeIndex2_bar',
            $index->getIndexName()
        );
    }

    /**
     * Test that ShowInSearch and getShowInSearch() exclude DataObjects from being added to the index
     *
     * Note: this code path that really being tested here is SearchUpdateProcessor->prepareIndexes()
     * This code path is used for 'inlet' filtering on CMS->save()
     * The results of this will show-up in SolrIndex->_addAs()
     */
    public function testShowInSearch()
    {
        $defaultMode = Versioned::get_reading_mode();
        Versioned::set_reading_mode('Stage.' . Versioned::DRAFT);

        $serviceMock = $this->getMockBuilder(Solr4Service::class)
            ->setMethods(['addDocument', 'deleteById'])
            ->getMock();

        $index = new SolrIndexTest_ShowInSearchIndex();
        $index->setService($serviceMock);
        FullTextSearch::force_index_list($index);

        // will get added
        $pageA = new Page();
        $pageA->Title = 'Test Page true';
        $pageA->ShowInSearch = true;
        $pageA->write();

        // will get filtered out
        $page = new Page();
        $page->Title = 'Test Page false';
        $page->ShowInSearch = false;
        $page->write();

        // will get added
        $fileA = new File();
        $fileA->Title = 'Test File true';
        $fileA->ShowInSearch = true;
        $fileA->write();

        // will get filtered out
        $file = new File();
        $file->Title = 'Test File false';
        $file->ShowInSearch = false;
        $file->write();

        // will get added
        $objOneA = new SolrIndexTest_MyDataObjectOne();
        $objOneA->Title = 'Test MyDataObjectOne true';
        $objOneA->ShowInSearch = true;
        $objOneA->write();

        // will get filtered out
        $objOne = new SolrIndexTest_MyDataObjectOne();
        $objOne->Title = 'Test MyDataObjectOne false';
        $objOne->ShowInSearch = false;
        $objOne->write();

        // will get added
        // this class has a getShowInSearch() == true, which will override $mypage->ShowInSearch = false
        $objTwoA = new SolrIndexTest_MyDataObjectTwo();
        $objTwoA->Title = 'Test MyDataObjectTwo false';
        $objTwoA->ShowInSearch = false;
        $objTwoA->write();

        // will get added
        // this class has a getShowInSearch() == true, which will override $mypage->ShowInSearch = false
        $myPageA = new SolrIndexTest_MyPage();
        $myPageA->Title = 'Test MyPage false';
        $myPageA->ShowInSearch = false;
        $myPageA->write();

        $callback = function (Apache_Solr_Document $doc) use ($pageA, $myPageA, $fileA, $objOneA, $objTwoA): bool {
            $validKeys = [
                Page::class . $pageA->ID,
                SolrIndexTest_MyPage::class . $myPageA->ID,
                File::class . $fileA->ID,
                SolrIndexTest_MyDataObjectOne::class . $objOneA->ID,
                SolrIndexTest_MyDataObjectTwo::class . $objTwoA->ID
            ];
            return in_array($this->createSolrDocKey($doc), $validKeys);
        };

        $serviceMock
            ->expects($this->exactly(5))
            ->method('addDocument')
            ->withConsecutive(
                [$this->callback($callback)],
                [$this->callback($callback)],
                [$this->callback($callback)],
                [$this->callback($callback)],
                [$this->callback($callback)]
            );

        // This is what actually triggers all the solr stuff
        SearchUpdater::flush_dirty_indexes();

        // delete a solr doc by setting ShowInSearch to false
        $pageA->ShowInSearch = false;
        $pageA->write();

        $serviceMock
            ->expects($this->exactly(1))
            ->method('deleteById')
            ->withConsecutive(
                [$this->callback(function (string $docID) use ($pageA): bool {
                    return strpos($docID, $pageA->ID . '-' . SiteTree::class) !== false;
                })]
            );

        IndexableService::singleton()->clearCache();
        SearchUpdater::flush_dirty_indexes();

        Versioned::set_reading_mode($defaultMode);
    }

    protected function createSolrDocKey(Apache_Solr_Document $doc)
    {
        return $doc->getField('ClassName')['value'] . $doc->getField('ID')['value'];
    }

    protected function getFakeRawSolrResponse()
    {
        return new \Apache_Solr_Response(
            new \Apache_Solr_HttpTransport_Response(
                null,
                null,
                '{}'
            )
        );
    }
}