From e4fd283d11ac37e5699e21db1256f78946383227 Mon Sep 17 00:00:00 2001 From: Dylan Wagstaff Date: Wed, 17 Oct 2018 12:21:42 +1300 Subject: [PATCH 1/7] remove composer branch alias --- composer.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/composer.json b/composer.json index 6d5555e..5eed708 100644 --- a/composer.json +++ b/composer.json @@ -41,11 +41,6 @@ "bin": [ "bin/fulltextsearch_quickstart" ], - "extra": { - "branch-alias": { - "dev-master": "3.x-dev" - } - }, "suggest": { "symbiote/silverstripe-queuedjobs": "Add background execution of indexing tasks", "silverstripe/fulltextsearch-localsolr": "Adds a ready-to-use local Solr server for initial development" From 9834b94f97e4373c06b9c517fd36fee444f4976d Mon Sep 17 00:00:00 2001 From: Scott Hutchinson Date: Tue, 23 Oct 2018 18:30:34 +1300 Subject: [PATCH 2/7] Enable macrons in search by default --- conf/solr/4/templates/types.ss | 26 ++++++++++++--------- docs/en/05_advanced_configuration.md | 34 ++++++++++++++-------------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/conf/solr/4/templates/types.ss b/conf/solr/4/templates/types.ss index 974ee2d..85ab507 100644 --- a/conf/solr/4/templates/types.ss +++ b/conf/solr/4/templates/types.ss @@ -8,7 +8,7 @@ + leading wildcard queries. --> @@ -320,10 +324,10 @@ a token of "foo|1.4" would be indexed as "foo" with a payload of 1.4f Attributes of the DelimitedPayloadTokenFilterFactory : "delimiter" - a one character delimiter. Default is | (pipe) - "encoder" - how to encode the following value into a playload - float -> org.apache.lucene.analysis.payloads.FloatEncoder, - integer -> o.a.l.a.p.IntegerEncoder - identity -> o.a.l.a.p.IdentityEncoder + "encoder" - how to encode the following value into a playload + float -> org.apache.lucene.analysis.payloads.FloatEncoder, + integer -> o.a.l.a.p.IntegerEncoder + identity -> o.a.l.a.p.IdentityEncoder Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor. --> @@ -364,8 +368,8 @@ - - \ No newline at end of file + --> + diff --git a/docs/en/05_advanced_configuration.md b/docs/en/05_advanced_configuration.md index 942af4c..19ffe8e 100644 --- a/docs/en/05_advanced_configuration.md +++ b/docs/en/05_advanced_configuration.md @@ -11,7 +11,7 @@ use SilverStripe\FullTextSearch\Search\Queries\SearchQuery; $index = MyIndex::singleton(); $query = SearchQuery::create() ->addSearchTerm('My Term'); -$params = [ +$params = [ 'facet' => 'true', 'facet.field' => 'SiteTree_ClassName', ]; @@ -159,7 +159,7 @@ substituted, which will include the original token. * Two comma-separated lists of words with the symbol "=>" between them. If the token matches any word on the left, then the list on the right is substituted. The original token will not be included unless it is also in the -list on the right. +list on the right. For example: @@ -207,7 +207,7 @@ $results = $index->search($query, -1, -1, $params); $results->spellcheck; ``` -The built-in `_text` data is better than nothing, but also has some problems: it's heavily processed, for example by +The built-in `_text` data is better than nothing, but also has some problems: it's heavily processed, for example by stemming filters which butcher words. So misspelling "Govnernance" will suggest "govern" rather than "Governance". This can be fixed by aggregating spell checking data in a separate field. @@ -289,22 +289,22 @@ Each result will automatically contain an `Excerpt` property which you can use i to avoid matching HTML attributes, and cluttering highlighted content with unparsed HTML. ## Boosting/Weighting - + Results aren't all created equal. Matches in some fields are more important than others; for example, a page `Title` might be considered more relevant to the user than terms in the `Content` field. - + To account for this, a "weighting" (or "boosting") factor can be applied to each searched field. The default value is `1.0`, anything below that will decrease the relevance, anything above increases it. You can get more information on relevancy at the [Solr wiki](http://wiki.apache.org/solr/SolrRelevancyFAQ). - + You can manage the boosting in two ways: - + ### Boosting on query - + To adjust the relative values at the time of querying, pass them in as the third argument to your `addSearchTerm()` call: - + ```php use My\Namespace\Index\MyIndex; use SilverStripe\FullTextSearch\Search\Queries\SearchQuery; use Page; - + $query = SearchQuery::create() ->addSearchTerm( 'fire', @@ -317,9 +317,9 @@ You can manage the boosting in two ways: ); $results = MyIndex::singleton()->search($query); ``` - + This will ensure that `Title` is given higher priority for matches than `Content`, which is well above `SecretParagraph`. - + ### Boosting on index Boost values for specific can also be specified directly on the `SolrIndex` class directly. @@ -357,14 +357,14 @@ class SolrSearchIndex extends SolrIndex ## Indexing related objects -To add a related object to your index. +To add a related object to your index. ## Subsites When you are utilising the [subsites module](https://github.com/silverstripe/silverstripe-subsites) you may want to add [boosting](#boosting/weighting) to results from the current subsite. To do so, you'll need to use [eDisMax](https://lucene.apache.org/solr/guide/6_6/the-extended-dismax-query-parser.html) -and the supporting parameters `bq` and `bf`. You should add the following to your `SolrIndex` +and the supporting parameters `bq` and `bf`. You should add the following to your `SolrIndex` extension: ```php @@ -385,7 +385,7 @@ public function search(SearchQuery $query, $offset = -1, $limit = -1, $params = ## Custom field types Solr supports custom field type definitions which are written to its XML schema. Many standard ones are already included - in the default schema. As the XML file is generated dynamically, we can add our own types by overloading the template + in the default schema. As the XML file is generated dynamically, we can add our own types by overloading the template responsible for it: `types.ss`. In the following example, we read our type definitions from a new file `mysite/solr/templates/types.ss` instead: @@ -428,7 +428,7 @@ To allow searches on words containing numeric tokens, you'll need to change the The `ASCIIFoldingFilterFactory` filter converts alphabetic, numeric, and symbolic Unicode characters which are not in the Basic Latin Unicode block (the first 127 ASCII characters) to their ASCII equivalents, if one exists. -Find the fields in your overloaded `types.ss` that you want to enable this behaviour in, for example inside the `` block, add the following to both its index analyzer and query analyzer records. +By default, this functionality is enabled on the `htmltext` and `text` fieldTypes. If you want it enabled for any other fieldTypes simply find the fields in your overloaded `types.ss` that you want to enable this behaviour in, for example inside the `` block, add the following to both its index analyzer and query analyzer records. ```xml @@ -436,7 +436,7 @@ Find the fields in your overloaded `types.ss` that you want to enable this behav ## Text extraction -Solr provides built-in text extraction capabilities for PDF and Office documents, and numerous other formats, through +Solr provides built-in text extraction capabilities for PDF and Office documents, and numerous other formats, through the `ExtractingRequestHandler` API (see [the Solr wiki entry](http://wiki.apache.org/solr/ExtractingRequestHandler). If you're using a default Solr installation, it's most likely already bundled and set up. But if you plan on running the Solr server integrated into this module, you'll need to download the libraries and link them first. Run the following From ed2c135b38d72a4cbc9a3e3fc3931132da135b38 Mon Sep 17 00:00:00 2001 From: Michal Kleiner Date: Tue, 4 Dec 2018 15:43:53 +1300 Subject: [PATCH 3/7] FIX #241 - Support numerically indexed array of classes --- src/Search/Variants/SearchVariant.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Search/Variants/SearchVariant.php b/src/Search/Variants/SearchVariant.php index a7c5756..4af5c53 100644 --- a/src/Search/Variants/SearchVariant.php +++ b/src/Search/Variants/SearchVariant.php @@ -186,6 +186,11 @@ abstract class SearchVariant // Construct new array of variants applicable to at least one class in the list $commonVariants = []; foreach ($classes as $class => $options) { + // BC for numerically indexed list of classes + if (is_numeric($class) && isset($options['class'])) { + $class = $options['class']; + } + // Extract relevant class options $includeSubclasses = isset($options['include_children']) ? $options['include_children'] : true; From fe5b8b768124e3e6021d2b207dc73389f22ab18e Mon Sep 17 00:00:00 2001 From: Michal Kleiner Date: Tue, 4 Dec 2018 16:51:40 +1300 Subject: [PATCH 4/7] FIX #241 - Minor edits --- src/Search/Variants/SearchVariant.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Search/Variants/SearchVariant.php b/src/Search/Variants/SearchVariant.php index 4af5c53..f1b4463 100644 --- a/src/Search/Variants/SearchVariant.php +++ b/src/Search/Variants/SearchVariant.php @@ -187,8 +187,8 @@ abstract class SearchVariant $commonVariants = []; foreach ($classes as $class => $options) { // BC for numerically indexed list of classes - if (is_numeric($class) && isset($options['class'])) { - $class = $options['class']; + if (is_numeric($class) && !empty($options['class'])) { + $class = $options['class']; // $options['class'] is assumed to exist throughout the code base } // Extract relevant class options From 8600c244f9ce735364448f0e03757ab689a525f8 Mon Sep 17 00:00:00 2001 From: Robbie Averill Date: Sat, 26 Jan 2019 16:03:26 +0200 Subject: [PATCH 5/7] Update namespace references in broken PHP docs and simplify condition logic --- src/Solr/Solr.php | 9 +++++---- src/Solr/SolrIndex.php | 1 + src/Solr/Tasks/Solr_BuildTask.php | 3 ++- src/Solr/Tasks/Solr_Configure.php | 27 ++++++++++++++------------ src/Utils/Logging/SearchLogFactory.php | 5 +++-- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/Solr/Solr.php b/src/Solr/Solr.php index 3cdf321..3fd977e 100644 --- a/src/Solr/Solr.php +++ b/src/Solr/Solr.php @@ -9,6 +9,8 @@ use SilverStripe\Core\Manifest\ModuleLoader; use SilverStripe\FullTextSearch\Search\FullTextSearch; use SilverStripe\FullTextSearch\Solr\Services\Solr4Service; use SilverStripe\FullTextSearch\Solr\Services\Solr3Service; +use SilverStripe\FullTextSearch\Solr\Services\SolrService; +use SilverStripe\FullTextSearch\Solr\Services\SolrService_Core; class Solr { @@ -112,14 +114,14 @@ class Solr /** @var SolrService | null - The instance of SolrService for core management */ protected static $service_singleton = null; - /** @var [SolrService_Core] - The instances of SolrService_Core for each core */ + /** @var SolrService_Core[] - The instances of SolrService_Core for each core */ protected static $service_core_singletons = array(); /** * Get a SolrService * * @param string $core Optional name of index class - * @return SolrService_Core + * @return SolrService|SolrService_Core */ public static function service($core = null) { @@ -142,9 +144,8 @@ class Solr } return self::$service_core_singletons[$core]; - } else { - return self::$service_singleton; } + return self::$service_singleton; } public static function get_indexes() diff --git a/src/Solr/SolrIndex.php b/src/Solr/SolrIndex.php index ad175a2..00f9202 100644 --- a/src/Solr/SolrIndex.php +++ b/src/Solr/SolrIndex.php @@ -12,6 +12,7 @@ use SilverStripe\FullTextSearch\Search\Queries\SearchQuery; use SilverStripe\FullTextSearch\Search\Queries\SearchQuery_Range; use SilverStripe\FullTextSearch\Search\Variants\SearchVariant; use SilverStripe\FullTextSearch\Search\SearchIntrospection; +use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore; use SilverStripe\ORM\ArrayList; use SilverStripe\ORM\DataObject; use SilverStripe\ORM\FieldType\DBField; diff --git a/src/Solr/Tasks/Solr_BuildTask.php b/src/Solr/Tasks/Solr_BuildTask.php index f9c24ec..ad3952e 100644 --- a/src/Solr/Tasks/Solr_BuildTask.php +++ b/src/Solr/Tasks/Solr_BuildTask.php @@ -3,6 +3,7 @@ namespace SilverStripe\FullTextSearch\Solr\Tasks; use Monolog\Handler\StreamHandler; use Psr\Log\LoggerInterface; +use SilverStripe\Control\HTTPRequest; use SilverStripe\Core\Injector\Injector; use SilverStripe\Dev\BuildTask; use SilverStripe\FullTextSearch\Utils\Logging\SearchLogFactory; @@ -52,7 +53,7 @@ class Solr_BuildTask extends BuildTask /** * Setup task * - * @param SS_HTTPReqest $request + * @param HTTPRequest $request */ public function run($request) { diff --git a/src/Solr/Tasks/Solr_Configure.php b/src/Solr/Tasks/Solr_Configure.php index 28afc98..22bdb5c 100644 --- a/src/Solr/Tasks/Solr_Configure.php +++ b/src/Solr/Tasks/Solr_Configure.php @@ -4,6 +4,7 @@ namespace SilverStripe\FullTextSearch\Solr\Tasks; use Exception; use SilverStripe\Core\ClassInfo; use SilverStripe\FullTextSearch\Solr\Solr; +use SilverStripe\FullTextSearch\Solr\SolrIndex; use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_File; use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_WebDAV; use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore; @@ -50,29 +51,30 @@ class Solr_Configure extends Solr_BuildTask protected function updateIndex($instance, $store) { $index = $instance->getIndexName(); - $this->getLogger()->addInfo("Configuring $index."); + $this->getLogger()->info("Configuring $index."); // Upload the config files for this index - $this->getLogger()->addInfo("Uploading configuration ..."); + $this->getLogger()->info("Uploading configuration ..."); $instance->uploadConfig($store); // Then tell Solr to use those config files $service = Solr::service(); if ($service->coreIsActive($index)) { - $this->getLogger()->addInfo("Reloading core ..."); + $this->getLogger()->info("Reloading core ..."); $service->coreReload($index); } else { - $this->getLogger()->addInfo("Creating core ..."); + $this->getLogger()->info("Creating core ..."); $service->coreCreate($index, $store->instanceDir($index)); } - $this->getLogger()->addInfo("Done"); + $this->getLogger()->info("Done"); } /** * Get config store * * @return SolrConfigStore + * @throws Exception */ protected function getSolrConfigStore() { @@ -85,14 +87,15 @@ class Solr_Configure extends Solr_BuildTask // Find the IndexStore handler, which will handle uploading config files to Solr $mode = $indexstore['mode']; - if ($mode == 'file') { + if ($mode === 'file') { return new SolrConfigStore_File($indexstore); - } elseif ($mode == 'webdav') { - return new SolrConfigStore_WebDAV($indexstore); - } elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, SolrConfigStore::class)) { - return new $mode($indexstore); - } else { - user_error('Unknown Solr index mode ' . $indexstore['mode'], E_USER_ERROR); } + if ($mode === 'webdav') { + return new SolrConfigStore_WebDAV($indexstore); + } + if (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, SolrConfigStore::class)) { + return new $mode($indexstore); + } + user_error('Unknown Solr index mode ' . $indexstore['mode'], E_USER_ERROR); } } diff --git a/src/Utils/Logging/SearchLogFactory.php b/src/Utils/Logging/SearchLogFactory.php index 230ee62..5e98e5a 100644 --- a/src/Utils/Logging/SearchLogFactory.php +++ b/src/Utils/Logging/SearchLogFactory.php @@ -3,6 +3,7 @@ namespace SilverStripe\FullTextSearch\Utils\Logging; use Psr\Log; +use Symbiote\QueuedJobs\Services\QueuedJob; interface SearchLogFactory { @@ -10,7 +11,7 @@ interface SearchLogFactory * Make a logger for a queuedjob * * @param QueuedJob $job - * @return Log + * @return Log\LoggerInterface */ public function getQueuedJobLogger($job); @@ -19,7 +20,7 @@ interface SearchLogFactory * * @param string $name * @param bool $verbose - * @return Log + * @return Log\LoggerInterface */ public function getOutputLogger($name, $verbose); } From ea91cf5a3c795f09318b8332d1730769a153a889 Mon Sep 17 00:00:00 2001 From: jovenden Date: Mon, 25 Mar 2019 09:59:25 +1300 Subject: [PATCH 6/7] NEW Adds a 'post' option for solrconfigstore --- src/Solr/Solr.php | 12 +++- src/Solr/Stores/SolrConfigStore_Post.php | 74 ++++++++++++++++++++++++ src/Solr/Tasks/Solr_Configure.php | 8 ++- 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 src/Solr/Stores/SolrConfigStore_Post.php diff --git a/src/Solr/Solr.php b/src/Solr/Solr.php index 3fd977e..905eb78 100644 --- a/src/Solr/Solr.php +++ b/src/Solr/Solr.php @@ -34,13 +34,19 @@ class Solr * * indexstore => an array with * - * mode - a classname which implements SolrConfigStore, or 'file' or 'webdav' + * mode - a classname which implements SolrConfigStore, or 'file', 'webdav' or 'post' * - * When mode == SolrConfigStore_File or file (indexes should be written on a local filesystem) + * When mode == SolrConfigStore_File or 'file' (indexes should be written on a local filesystem) * path - The (locally accessible) path to write the index configurations to. * remotepath (default: the same as indexpath) - The path that the Solr server will read the index configurations from * - * When mode == SolrConfigStore_WebDAV or webdav (indexes should stored on a remote Solr server via webdav) + * When mode == SolrConfigStore_Post or 'post' (indexes should stored on a remote Solr server via post) + * This mode will require custom software on the remote solr server which handles receiving the post and + * passing on that information to solr. It is up to the user of this mode to write such software. + * path (default: /solrindex) - The suburl on the solr host that is set up to accept index configurations + * port (default: none) - The port on the remote server which is set up to receive the post information + * + * When mode == SolrConfigStore_WebDAV or 'webdav' (indexes should stored on a remote Solr server via webdav) * auth (default: none) - A username:password pair string to use to auth against the webdav server * path (default: /solrindex) - The suburl on the solr host that is set up to accept index configurations via webdav * port (default: none) - The port for WebDAV if different from the Solr port diff --git a/src/Solr/Stores/SolrConfigStore_Post.php b/src/Solr/Stores/SolrConfigStore_Post.php new file mode 100644 index 0000000..7d40f68 --- /dev/null +++ b/src/Solr/Stores/SolrConfigStore_Post.php @@ -0,0 +1,74 @@ +url = implode('', [ + 'http://', + isset($config['auth']) ? $config['auth'] . '@' : '', + $options['host'] . ':' . $options['port'], + $config['path'] + ]); + $this->remote = $config['remotepath']; + } + + /** + * @param string $index + * @param string $file + * @return void + */ + public function uploadFile($index, $file) + { + $this->uploadString($index, basename($file), file_get_contents($file)); + } + + /** + * + * @param string $index + * @param string $filename + * @param string $string + * @return void + */ + public function uploadString($index, $filename, $string) + { + $targetDir = "{$this->url}/config/$index"; + + file_get_contents($targetDir . '/' . $filename, false, stream_context_create([ + 'http' => [ + 'method' => 'POST', + 'header' => 'Content-type: application/octet-stream', + 'content' => (string) $string + ] + ])); + } + + /** + * @param string $index + * @return string + */ + public function instanceDir($index) + { + return $this->remote ? "{$this->remote}/$index" : $index; + } +} diff --git a/src/Solr/Tasks/Solr_Configure.php b/src/Solr/Tasks/Solr_Configure.php index 22bdb5c..05e1634 100644 --- a/src/Solr/Tasks/Solr_Configure.php +++ b/src/Solr/Tasks/Solr_Configure.php @@ -5,9 +5,10 @@ use Exception; use SilverStripe\Core\ClassInfo; use SilverStripe\FullTextSearch\Solr\Solr; use SilverStripe\FullTextSearch\Solr\SolrIndex; -use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_File; -use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_WebDAV; use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore; +use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_File; +use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_Post; +use SilverStripe\FullTextSearch\Solr\Stores\SolrConfigStore_WebDAV; class Solr_Configure extends Solr_BuildTask { @@ -93,6 +94,9 @@ class Solr_Configure extends Solr_BuildTask if ($mode === 'webdav') { return new SolrConfigStore_WebDAV($indexstore); } + if ($mode == 'post') { + return new SolrConfigStore_Post($indexstore); + } if (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, SolrConfigStore::class)) { return new $mode($indexstore); } From cf784f5c0a3cd25af5b77a571ad7a36a97d49fa5 Mon Sep 17 00:00:00 2001 From: Robbie Averill Date: Mon, 25 Mar 2019 13:22:36 +1300 Subject: [PATCH 7/7] Use strict comparison --- src/Solr/Tasks/Solr_Configure.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Solr/Tasks/Solr_Configure.php b/src/Solr/Tasks/Solr_Configure.php index 05e1634..4ee6690 100644 --- a/src/Solr/Tasks/Solr_Configure.php +++ b/src/Solr/Tasks/Solr_Configure.php @@ -94,7 +94,7 @@ class Solr_Configure extends Solr_BuildTask if ($mode === 'webdav') { return new SolrConfigStore_WebDAV($indexstore); } - if ($mode == 'post') { + if ($mode === 'post') { return new SolrConfigStore_Post($indexstore); } if (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, SolrConfigStore::class)) {