From 51749c603ae3b884b280e4cb9100871c1ca52bf5 Mon Sep 17 00:00:00 2001 From: Ingo Schommer Date: Tue, 7 Feb 2017 08:58:21 +1300 Subject: [PATCH 1/7] Fixed Travis URL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a6bf49..66ab25f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FullTextSearch module -[![Build Status](https://secure.travis-ci.org/silverstripe-labs/silverstripe-fulltextsearch.png?branch=master)](http://travis-ci.org/silverstripe-labs/silverstripe-fulltextsearch) +[![Build Status](https://secure.travis-ci.org/silverstripe/silverstripe-fulltextsearch.png?branch=master)](http://travis-ci.org/silverstripe/silverstripe-fulltextsearch) Adds support for fulltext search engines like Sphinx and Solr to SilverStripe CMS. From bbaf4276af015e2f32d63256f1401bd3e7123faa Mon Sep 17 00:00:00 2001 From: Damian Mooyman Date: Thu, 9 Mar 2017 11:40:13 +1300 Subject: [PATCH 2/7] BUG Fix delete / unpublish Split SearchUpdater classes into separate files Fixes #134 --- code/search/SearchUpdater.php | 83 ++----------------- ...hUpdater_BindManipulationCaptureFilter.php | 15 ++++ code/search/SearchUpdater_ObjectHandler.php | 71 ++++++++++++++++ tests/SearchVariantVersionedTest.php | 22 +++++ 4 files changed, 113 insertions(+), 78 deletions(-) create mode 100644 code/search/SearchUpdater_BindManipulationCaptureFilter.php create mode 100644 code/search/SearchUpdater_ObjectHandler.php diff --git a/code/search/SearchUpdater.php b/code/search/SearchUpdater.php index 0c836f0..12e0740 100644 --- a/code/search/SearchUpdater.php +++ b/code/search/SearchUpdater.php @@ -97,6 +97,7 @@ class SearchUpdater extends Object $id = $details['id']; $state = $details['state']; $class = $details['class']; + $command = $details['command']; $fields = isset($details['fields']) ? $details['fields'] : array(); $base = ClassInfo::baseDataClass($class); @@ -111,6 +112,7 @@ class SearchUpdater extends Object 'class' => $class, 'id' => $id, 'statefulids' => $statefulids, + 'command' => $command, 'fields' => array() ); } @@ -125,9 +127,9 @@ class SearchUpdater extends Object } } - // Trim records without fields - foreach(array_keys($writes) as $key) { - if(empty($writes[$key]['fields'])) { + // Trim non-delete records without fields + foreach (array_keys($writes) as $key) { + if ($writes[$key]['command'] !== 'delete' && empty($writes[$key]['fields'])) { unset($writes[$key]); } } @@ -203,78 +205,3 @@ class SearchUpdater extends Object self::$processor = null; } } - -class SearchUpdater_BindManipulationCaptureFilter implements RequestFilter -{ - public function preRequest(SS_HTTPRequest $request, Session $session, DataModel $model) - { - SearchUpdater::bind_manipulation_capture(); - } - - public function postRequest(SS_HTTPRequest $request, SS_HTTPResponse $response, DataModel $model) - { - /* NOP */ - } -} - -/** - * Delete operations do not use database manipulations. - * - * If a delete has been requested, force a write on objects that should be - * indexed. This causes the object to be marked for deletion from the index. - */ - -class SearchUpdater_ObjectHandler extends DataExtension -{ - public function onAfterDelete() - { - // Calling delete() on empty objects does nothing - if (!$this->owner->ID) { - return; - } - - // Force SearchUpdater to mark this record as dirty - $manipulation = array( - $this->owner->ClassName => array( - 'fields' => array(), - 'id' => $this->owner->ID, - 'command' => 'update' - ) - ); - $this->owner->extend('augmentWrite', $manipulation); - SearchUpdater::handle_manipulation($manipulation); - } - - /** - * Forces this object to trigger a re-index in the current state - */ - public function triggerReindex() - { - if (!$this->owner->ID) { - return; - } - - $id = $this->owner->ID; - $class = $this->owner->ClassName; - $state = SearchVariant::current_state($class); - $base = ClassInfo::baseDataClass($class); - $key = "$id:$base:".serialize($state); - - $statefulids = array(array( - 'id' => $id, - 'state' => $state - )); - - $writes = array( - $key => array( - 'base' => $base, - 'class' => $class, - 'id' => $id, - 'statefulids' => $statefulids, - 'fields' => array() - ) - ); - - SearchUpdater::process_writes($writes); - } -} diff --git a/code/search/SearchUpdater_BindManipulationCaptureFilter.php b/code/search/SearchUpdater_BindManipulationCaptureFilter.php new file mode 100644 index 0000000..24cef93 --- /dev/null +++ b/code/search/SearchUpdater_BindManipulationCaptureFilter.php @@ -0,0 +1,15 @@ +owner->ID) { + return; + } + + // Force SearchUpdater to mark this record as dirty + // Note: Some extensions require entire hierarchy passed to augmentWrite() + $manipulation = array(); + foreach (ClassInfo::ancestry($this->owner) as $class) { + if (!is_subclass_of($class, 'DataObject')) { + continue; + } + $manipulation[$class] = array( + 'fields' => array(), + 'id' => $this->owner->ID, + // Note: 'delete' command not actually handled by manipulations, + // but added so that SearchUpdater can detect the deletion + 'command' => 'delete' + ); + } + $this->owner->extend('augmentWrite', $manipulation); + SearchUpdater::handle_manipulation($manipulation); + } + + /** + * Forces this object to trigger a re-index in the current state + */ + public function triggerReindex() + { + if (!$this->owner->ID) { + return; + } + + $id = $this->owner->ID; + $class = $this->owner->ClassName; + $state = SearchVariant::current_state($class); + $base = ClassInfo::baseDataClass($class); + $key = "$id:$base:" . serialize($state); + + $statefulids = array( + array( + 'id' => $id, + 'state' => $state + ) + ); + + $writes = array( + $key => array( + 'base' => $base, + 'class' => $class, + 'id' => $id, + 'statefulids' => $statefulids, + 'fields' => array() + ) + ); + + SearchUpdater::process_writes($writes); + } +} diff --git a/tests/SearchVariantVersionedTest.php b/tests/SearchVariantVersionedTest.php index 76747f5..0f490a7 100644 --- a/tests/SearchVariantVersionedTest.php +++ b/tests/SearchVariantVersionedTest.php @@ -72,6 +72,28 @@ class SearchVariantVersionedTest extends SapphireTest )); $added = self::$index->getAdded(array('ID', '_versionedstage')); $this->assertEquals($expected, $added); + + // Test unpublish + + self::$index->reset(); + + $item->deleteFromStage('Live'); + + SearchUpdater::flush_dirty_indexes(); + + $this->assertEquals(1, count(self::$index->deleted)); + $this->assertEquals( + 'SiteTree', + self::$index->deleted[0]['base'] + ); + $this->assertEquals( + $item->ID, + self::$index->deleted[0]['id'] + ); + $this->assertEquals( + 'Live', + self::$index->deleted[0]['state']['SearchVariantVersioned'] + ); } public function testExcludeVariantState() From f5392b630f10a3f4813872787b8c618242265eac Mon Sep 17 00:00:00 2001 From: Elliot Sawyer Date: Tue, 28 Mar 2017 16:20:46 +1300 Subject: [PATCH 3/7] Update Solr.md Update documentation explaining how to search for numeric terms by overloading types.ss --- docs/en/Solr.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/en/Solr.md b/docs/en/Solr.md index 89e22bb..c84a0d3 100644 --- a/docs/en/Solr.md +++ b/docs/en/Solr.md @@ -191,6 +191,17 @@ from a new file `mysite/solr/templates/types.ss` instead: return $this->renderWith(Director::baseFolder() . '/mysite/solr/templates/types.ss'); } } + +#### Searching for words containing numbers + +By default, the fulltextmodule is configured to split words containing numbers into multiple tokens. For example, the word “A1” would be interpreted as “A” “1”; since “a” is a common stopword, the term “A1” will be excluded from search. + +To allow searches on words containing numeric tokens, you'll need to update your overloaded template to change the behaviour of the WordDelimiterFilterFactory. Each instance of `` needs to include the following attributes and values: + +* add splitOnNumerics="0" on all WordDelimiterFilterFactory fields +* change catenateOnNumbers="1" on all WordDelimiterFilterFactory fields + +Update your index to point to your overloaded template using the method described above. ### Spell Checking ("Did you mean...") From 2246fa9ab8a23a1c948b7d20483ba056a4c4dfa9 Mon Sep 17 00:00:00 2001 From: Elliot Sawyer Date: Thu, 30 Mar 2017 23:17:46 +1300 Subject: [PATCH 4/7] Obtain search results for a particular subsite when the request comes from a query (#136) * Allow a user to obtain search results for a particular subsite when the request comes from a query When attempting to do this in project code, SearchVariantSubsites kicks in and replaces any filter you've applied with the current subsite ID, which prevents you from searching for filtered results in another subsite. This fix prevents the module from doing this if a filter is applied on the query TODO write unit tests for the filter() and exclude() behaviours --- README.md | 1 + code/search/SearchVariantSubsites.php | 29 ++++++++- tests/SearchVariantSubsitesTest.php | 91 +++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 tests/SearchVariantSubsitesTest.php diff --git a/README.md b/README.md index 66ab25f..a5389d3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # FullTextSearch module [![Build Status](https://secure.travis-ci.org/silverstripe/silverstripe-fulltextsearch.png?branch=master)](http://travis-ci.org/silverstripe/silverstripe-fulltextsearch) diff --git a/code/search/SearchVariantSubsites.php b/code/search/SearchVariantSubsites.php index 8f52ac2..63ad366 100644 --- a/code/search/SearchVariantSubsites.php +++ b/code/search/SearchVariantSubsites.php @@ -60,9 +60,21 @@ class SearchVariantSubsites extends SearchVariant )); } - + /** + * This field has been altered to allow a user to obtain search results for a particular subsite + * When attempting to do this in project code, SearchVariantSubsites kicks and overwrites any filter you've applied + * This fix prevents the module from doing this if a filter is applied on the index or the query, or if a field is + * being excluded specifically before being executed. + * + * A pull request has been raised for this issue. Once accepted this forked module can be deleted and the parent + * project should be used instead. + */ public function alterQuery($query, $index) { + if ($this->isFieldFiltered('_subsite', $query)) { + return; + } + $subsite = Subsite::currentSubsiteID(); $query->filter('_subsite', array($subsite, SearchQuery::$missing)); } @@ -109,4 +121,19 @@ class SearchVariantSubsites extends SearchVariant $writes[$key]['statefulids'] = $next; } } + + /** + * Determine if a field with a certain name is filtered by the search query or on the index + * This is the equivalent of saying "show me the results that do ONLY contain this value" + * @param $field string name of the field being filtered + * @param $query SearchQuery currently being executed + * @param $index SearchIndex which specifies a filter field + * @return bool true if $field is being filtered, false if it is not being filtered + */ + protected function isFieldFiltered($field, $query) + { + $queryHasFilter = !empty($query->require[$field]); + + return $queryHasFilter; + } } diff --git a/tests/SearchVariantSubsitesTest.php b/tests/SearchVariantSubsitesTest.php new file mode 100644 index 0000000..eb29375 --- /dev/null +++ b/tests/SearchVariantSubsitesTest.php @@ -0,0 +1,91 @@ +markTestSkipped('The subsites module is not installed'); + } + + if (self::$index === null) { + self::$index = singleton('SearchVariantSubsiteTest'); + } + + SearchUpdater::bind_manipulation_capture(); + + Config::inst()->update('Injector', 'SearchUpdateProcessor', array( + 'class' => 'SearchUpdateImmediateProcessor' + )); + + FullTextSearch::force_index_list(self::$index); + SearchUpdater::clear_dirty_indexes(); + } + + public function testQueryIsAlteredWhenSubsiteNotSet() + { + $index = new SolrIndexTest_FakeIndex(); + $query = new SearchQuery(); + + //typical behaviour: nobody is explicitly filtering on subsite, so the search variant adds a filter to the query + $this->assertArrayNotHasKey('_subsite', $query->require); + $variant = new SearchVariantSubsites(); + $variant->alterDefinition('SearchUpdaterTest_Container', $index); + $variant->alterQuery($query, $index); + + //check that the "default" query has been put in place: it's not empty, and we're searching on Subsite ID:0 and + // an object of SearchQuery::missing + $this->assertNotEmpty($query->require['_subsite']); + $this->assertEquals(0, $query->require['_subsite'][0]); + + //check that SearchQuery::missing is set (by default, it is an object of stdClass) + $this->assertInstanceOf('stdClass', $query->require['_subsite'][1]); + } + + + public function testQueryIsAlteredWhenSubsiteIsSet() + { + //now we want to test if somebody has already applied the _subsite filter to the query + $index = new SolrIndexTest_FakeIndex(); + $query = new SearchQuery(); + + //check that _subsite is not applied yet + //this key should not be exist until the SearchVariant applies it later + $this->assertArrayNotHasKey('_subsite', $query->require); + + //apply the subsite filter on the query (for example, if it's passed into a controller and set before searching) + //we've chosen an arbirary value of 2 here, to check if it is changed later + $query->filter('_subsite', 2); + $this->assertNotEmpty($query->require['_subsite']); + + //apply the search variant's definition and query + $variant = new SearchVariantSubsites(); + $variant->alterDefinition('SearchUpdaterTest_Container', $index); + + //the protected function isFieldFiltered is implicitly tested here + $variant->alterQuery($query, $index); + + //confirm that the query has been altered, but NOT with default values + //first check that _subsite filter is not empty + $this->assertNotEmpty($query->require['_subsite']); + //subsite filter first value is not 0 + $this->assertNotEquals(0, $query->require['_subsite'][0]); + + //subsite filter SearchQuery::missing should not be set so its expected location is empty + $this->assertArrayNotHasKey(1, $query->require['_subsite']); + + //subsite filter has been modified with our arbitrary test value. The second value is not set + //this proves that the query has not been altered by the variant + $this->assertEquals(2, $query->require['_subsite'][0]); + + } + +} + From 4c44d1c29e6b50da420b6fe592fdc18838babfeb Mon Sep 17 00:00:00 2001 From: Brett Tasker Date: Mon, 1 May 2017 20:18:59 +1200 Subject: [PATCH 5/7] FIX: Broken merge as files have been moved + Fixed additional tests --- ...hUpdater_BindManipulationCaptureFilter.php | 15 ---- code/search/SearchUpdater_ObjectHandler.php | 71 ------------------- .../updaters/SearchUpdater_ObjectHandler.php | 24 +++++-- tests/SearchVariantVersionedTest.php | 6 +- 4 files changed, 23 insertions(+), 93 deletions(-) delete mode 100644 code/search/SearchUpdater_BindManipulationCaptureFilter.php delete mode 100644 code/search/SearchUpdater_ObjectHandler.php diff --git a/code/search/SearchUpdater_BindManipulationCaptureFilter.php b/code/search/SearchUpdater_BindManipulationCaptureFilter.php deleted file mode 100644 index 24cef93..0000000 --- a/code/search/SearchUpdater_BindManipulationCaptureFilter.php +++ /dev/null @@ -1,15 +0,0 @@ -owner->ID) { - return; - } - - // Force SearchUpdater to mark this record as dirty - // Note: Some extensions require entire hierarchy passed to augmentWrite() - $manipulation = array(); - foreach (ClassInfo::ancestry($this->owner) as $class) { - if (!is_subclass_of($class, 'DataObject')) { - continue; - } - $manipulation[$class] = array( - 'fields' => array(), - 'id' => $this->owner->ID, - // Note: 'delete' command not actually handled by manipulations, - // but added so that SearchUpdater can detect the deletion - 'command' => 'delete' - ); - } - $this->owner->extend('augmentWrite', $manipulation); - SearchUpdater::handle_manipulation($manipulation); - } - - /** - * Forces this object to trigger a re-index in the current state - */ - public function triggerReindex() - { - if (!$this->owner->ID) { - return; - } - - $id = $this->owner->ID; - $class = $this->owner->ClassName; - $state = SearchVariant::current_state($class); - $base = ClassInfo::baseDataClass($class); - $key = "$id:$base:" . serialize($state); - - $statefulids = array( - array( - 'id' => $id, - 'state' => $state - ) - ); - - $writes = array( - $key => array( - 'base' => $base, - 'class' => $class, - 'id' => $id, - 'statefulids' => $statefulids, - 'fields' => array() - ) - ); - - SearchUpdater::process_writes($writes); - } -} diff --git a/code/search/updaters/SearchUpdater_ObjectHandler.php b/code/search/updaters/SearchUpdater_ObjectHandler.php index f469584..c75b8df 100644 --- a/code/search/updaters/SearchUpdater_ObjectHandler.php +++ b/code/search/updaters/SearchUpdater_ObjectHandler.php @@ -4,7 +4,9 @@ namespace SilverStripe\FullTextSearch\Search\Updaters; use SilverStripe\ORM\DataExtension; use SilverStripe\FullTextSearch\Search\Variants\SearchVariant; +use SilverStripe\FullTextSearch\Search\Updaters\SearchUpdater; use SilverStripe\ORM\DataObject; +use SilverStripe\Core\ClassInfo; /** * Delete operations do not use database manipulations. @@ -23,14 +25,26 @@ class SearchUpdater_ObjectHandler extends DataExtension } // Force SearchUpdater to mark this record as dirty - $manipulation = array( - $this->owner->ClassName => array( + // Note: Some extensions require entire hierarchy passed to augmentWrite() + $manipulation = array(); + foreach (ClassInfo::ancestry($this->owner) as $class) { + if (!is_subclass_of($class, DataObject::class)) { + continue; + } + + $tableName = DataObject::getSchema()->tableName($class); + $manipulation[$tableName] = array( 'fields' => array(), 'id' => $this->owner->ID, - 'command' => 'update' - ) - ); + 'class' => $class, + // Note: 'delete' command not actually handled by manipulations, + // but added so that SearchUpdater can detect the deletion + 'command' => 'delete' + ); + } + $this->owner->extend('augmentWrite', $manipulation); + SearchUpdater::handle_manipulation($manipulation); } diff --git a/tests/SearchVariantVersionedTest.php b/tests/SearchVariantVersionedTest.php index bd6a97e..f0a7695 100644 --- a/tests/SearchVariantVersionedTest.php +++ b/tests/SearchVariantVersionedTest.php @@ -2,8 +2,10 @@ use SilverStripe\Dev\SapphireTest; use SilverStripe\Core\Config\Config; +use SilverStripe\CMS\Model\SiteTree; use SilverStripe\FullTextSearch\Search\FullTextSearch; use SilverStripe\FullTextSearch\Search\Indexes\SearchIndex_Recording; +use SilverStripe\FullTextSearch\Search\Variants\SearchVariantVersioned; use SilverStripe\FullTextSearch\Tests\SearchVariantVersionedTest\SearchVariantVersionedTest_Index; use SilverStripe\FullTextSearch\Tests\SearchVariantVersionedTest\SearchVariantVersionedTest_Item; use SilverStripe\FullTextSearch\Tests\SearchVariantVersionedTest\SearchVariantVersionedTest_IndexNoStage; @@ -90,7 +92,7 @@ class SearchVariantVersionedTest extends SapphireTest $this->assertCount(1, self::$index->deleted); $this->assertEquals( - 'SiteTree', + SiteTree::class, self::$index->deleted[0]['base'] ); $this->assertEquals( @@ -99,7 +101,7 @@ class SearchVariantVersionedTest extends SapphireTest ); $this->assertEquals( 'Live', - self::$index->deleted[0]['state']['SearchVariantVersioned'] + self::$index->deleted[0]['state'][SearchVariantVersioned::class] ); } From 8adb3171f05814f1c59db641750c1fa6719f26d8 Mon Sep 17 00:00:00 2001 From: Brett Tasker Date: Mon, 1 May 2017 20:46:19 +1200 Subject: [PATCH 6/7] FIX: Update travis to remove php 5.5 + Fix SearchVariantSubsiteTest reference to SapphireTest --- .travis.yml | 1 - tests/SearchVariantSubsitesTest.php | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bdce2be..10647a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ language: php sudo: false php: - - 5.5 - 5.6 - 7.0 - 7.1 diff --git a/tests/SearchVariantSubsitesTest.php b/tests/SearchVariantSubsitesTest.php index eb29375..f5bee06 100644 --- a/tests/SearchVariantSubsitesTest.php +++ b/tests/SearchVariantSubsitesTest.php @@ -1,5 +1,7 @@ Date: Mon, 1 May 2017 21:19:22 +1200 Subject: [PATCH 7/7] SS 4.0 - Include PHPUnit in composer.json --- composer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index b2b4fd3..bb19a82 100644 --- a/composer.json +++ b/composer.json @@ -24,7 +24,8 @@ "monolog/monolog": "~1.15" }, "require-dev": { - "silverstripe/cms": "4.0.x-dev" + "silverstripe/cms": "4.0.x-dev", + "phpunit/phpunit": "*@stable" }, "extra": { "branch-alias": {