WIP split Solr files with multiple classes into single file / single class. Adjust namespaces

This commit is contained in:
elliot sawyer 2017-04-21 13:08:14 +12:00
parent abdfe1947f
commit 424107fcac
15 changed files with 482 additions and 456 deletions

View File

@ -1,13 +1,7 @@
<?php
namespace SilverStripe\FullTextSearch\Solr;
use Monolog\Formatter\LineFormatter;
use Monolog\Handler\StreamHandler;
use Monolog\Logger;
use Psr\Log\LoggerInterface;
use SilverStripe\Dev\BuildTask;
use SilverStripe\Control\Director;
use SilverStripe\Core\Injector\Injector;
use SilverStripe\Core\ClassInfo;
use SilverStripe\Core\Object;
class Solr
@ -162,293 +156,4 @@ class Solr
$included = true;
}
}
}
/**
* Abstract class for build tasks
*/
class Solr_BuildTask extends BuildTask
{
protected $enabled = false;
/**
* Logger
*
* @var LoggerInterface
*/
protected $logger = null;
/**
* Get the current logger
*
* @return LoggerInterface
*/
public function getLogger()
{
return Injector::inst()->get('Logger');
}
/**
* Assign a new logger
*
* @param LoggerInterface $logger
*/
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
}
/**
* @return SearchLogFactory
*/
protected function getLoggerFactory()
{
// return Injector::inst()->get('SearchLogFactory');
}
/**
* Setup task
*
* @param SS_HTTPReqest $request
*/
public function run($request)
{
$name = get_class($this);
$verbose = $request->getVar('verbose');
// Set new logger
$logger = $this
->getLoggerFactory();
//@todo: Cannot instantiate interface SearchLogFactory
// ->getOutputLogger($name, $verbose);
$this->setLogger($logger);
}
}
class Solr_Configure extends Solr_BuildTask
{
protected $enabled = true;
public function run($request)
{
parent::run($request);
// Find the IndexStore handler, which will handle uploading config files to Solr
$store = $this->getSolrConfigStore();
$indexes = Solr::get_indexes();
foreach ($indexes as $instance) {
try {
$this->updateIndex($instance, $store);
} catch (Exception $e) {
// We got an exception. Warn, but continue to next index.
$this
->getLogger()
->error("Failure: " . $e->getMessage());
}
}
}
/**
* Update the index on the given store
*
* @param SolrIndex $instance Instance
* @param SolrConfigStore $store
*/
protected function updateIndex($instance, $store)
{
$index = $instance->getIndexName();
$this->getLogger()->addInfo("Configuring $index.");
// Upload the config files for this index
$this->getLogger()->addInfo("Uploading configuration ...");
$instance->uploadConfig($store);
// Then tell Solr to use those config files
$service = Solr::service();
if ($service->coreIsActive($index)) {
$this->getLogger()->addInfo("Reloading core ...");
$service->coreReload($index);
} else {
$this->getLogger()->addInfo("Creating core ...");
$service->coreCreate($index, $store->instanceDir($index));
}
$this->getLogger()->addInfo("Done");
}
/**
* Get config store
*
* @return SolrConfigStore
*/
protected function getSolrConfigStore()
{
$options = Solr::solr_options();
if (!isset($options['indexstore']) || !($indexstore = $options['indexstore'])) {
throw new Exception('No index configuration for Solr provided', E_USER_ERROR);
}
// Find the IndexStore handler, which will handle uploading config files to Solr
$mode = $indexstore['mode'];
if ($mode == 'file') {
return new SolrConfigStore_File($indexstore);
} elseif ($mode == 'webdav') {
return new SolrConfigStore_WebDAV($indexstore);
} elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, 'SolrConfigStore')) {
return new $mode($indexstore);
} else {
user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR);
}
}
}
/**
* Task used for both initiating a new reindex, as well as for processing incremental batches
* within a reindex.
*
* When running a complete reindex you can provide any of the following
* - class (to limit to a single class)
* - verbose (optional)
*
* When running with a single batch, provide the following querystring arguments:
* - start
* - index
* - class
* - variantstate
* - verbose (optional)
*/
class Solr_Reindex extends Solr_BuildTask
{
protected $enabled = true;
/**
* Number of records to load and index per request
*
* @var int
* @config
*/
private static $recordsPerRequest = 200;
/**
* Get the reindex handler
*
* @return SolrReindexHandler
*/
protected function getHandler()
{
//@todo: this needs to determine the best class from a Factory implementation
//@todo: it was 'SolrReindexHandler' but that doesn't work on 4.0
return Injector::inst()->get('SolrReindexImmediateHandler');
}
/**
* @param SS_HTTPRequest $request
*/
public function run($request)
{
parent::run($request);
// Reset state
$originalState = SearchVariant::current_state();
$this->doReindex($request);
SearchVariant::activate_state($originalState);
}
/**
* @param SS_HTTPRequest $request
*/
protected function doReindex($request)
{
$class = $request->getVar('class');
$index = $request->getVar('index');
//find the index classname by IndexName
// this is for when index names do not match the class name (this can be done by overloading getIndexName() on
// indexes
if ($index && !ClassInfo::exists($index)) {
foreach(ClassInfo::subclassesFor('SolrIndex') as $solrIndexClass) {
$reflection = new ReflectionClass($solrIndexClass);
//skip over abstract classes
if (!$reflection->isInstantiable()) {
continue;
}
//check the indexname matches the index passed to the request
if (!strcasecmp(singleton($solrIndexClass)->getIndexName(), $index)) {
//if we match, set the correct index name and move on
$index = $solrIndexClass;
break;
}
}
}
// Deprecated reindex mechanism
$start = $request->getVar('start');
if ($start !== null) {
// Run single batch directly
$indexInstance = singleton($index);
$state = json_decode($request->getVar('variantstate'), true);
$this->runFrom($indexInstance, $class, $start, $state);
return;
}
// Check if we are re-indexing a single group
// If not using queuedjobs, we need to invoke Solr_Reindex as a separate process
// Otherwise each group is processed via a SolrReindexGroupJob
$groups = $request->getVar('groups');
$handler = $this->getHandler();
if ($groups) {
// Run grouped batches (id % groups = group)
$group = $request->getVar('group');
$indexInstance = singleton($index);
$state = json_decode($request->getVar('variantstate'), true);
$handler->runGroup($this->getLogger(), $indexInstance, $state, $class, $groups, $group);
return;
}
// If run at the top level, delegate to appropriate handler
$self = get_class($this);
$handler->triggerReindex($this->getLogger(), $this->config()->recordsPerRequest, $self, $class);
}
/**
* @deprecated since version 2.0.0
*/
protected function runFrom($index, $class, $start, $variantstate)
{
DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism');
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($variantstate);
// Generate filtered list
$items = DataList::create($class)
->limit($this->config()->recordsPerRequest, $start);
// Add child filter
$classes = $index->getClasses();
$options = $classes[$class];
if (!$options['include_children']) {
$items = $items->filter('ClassName', $class);
}
// Process selected records in this class
$this->getLogger()->info("Adding $class");
foreach ($items->sort("ID") as $item) {
$this->getLogger()->debug($item->ID);
// See SearchUpdater_ObjectHandler::triggerReindex
$item->triggerReindex();
$item->destroy();
}
$this->getLogger()->info("Done");
}
}
}

View File

@ -1,134 +0,0 @@
<?php
namespace SilverStripe\FullTextSearch\Solr;
/**
* Class SolrConfigStore
*
* The interface Solr_Configure uses to upload configuration files to Solr
*/
interface SolrConfigStore
{
/**
* Upload a file to Solr for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
* @param $file string - A path to a file to upload. The base name of the file will be used on the remote side
* @return null
*/
public function uploadFile($index, $file);
/**
* Upload a file to Solr from a string for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
* @param $filename string - The base name of the file to use on the remote side
* @param $strong string - The contents of the file
* @return null
*/
public function uploadString($index, $filename, $string);
/**
* Get the instanceDir to tell Solr to use for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
*/
public function instanceDir($index);
}
/**
* Class SolrConfigStore_File
*
* A ConfigStore that uploads files to a Solr instance on a locally accessible filesystem
* by just using file copies
*/
class SolrConfigStore_File implements SolrConfigStore
{
public function __construct($config)
{
$this->local = $config['path'];
$this->remote = isset($config['remotepath']) ? $config['remotepath'] : $config['path'];
}
public function getTargetDir($index)
{
$targetDir = "{$this->local}/{$index}/conf";
if (!is_dir($targetDir)) {
$worked = @mkdir($targetDir, 0770, true);
if (!$worked) {
throw new RuntimeException(
sprintf('Failed creating target directory %s, please check permissions', $targetDir)
);
}
}
return $targetDir;
}
public function uploadFile($index, $file)
{
$targetDir = $this->getTargetDir($index);
copy($file, $targetDir.'/'.basename($file));
}
public function uploadString($index, $filename, $string)
{
$targetDir = $this->getTargetDir($index);
file_put_contents("$targetDir/$filename", $string);
}
public function instanceDir($index)
{
return $this->remote.'/'.$index;
}
}
/**
* Class SolrConfigStore_WebDAV
*
* A ConfigStore that uploads files to a Solr instance via a WebDAV server
*/
class SolrConfigStore_WebDAV implements SolrConfigStore
{
public function __construct($config)
{
$options = Solr::solr_options();
$this->url = implode('', array(
'http://',
isset($config['auth']) ? $config['auth'].'@' : '',
$options['host'].':'.(isset($config['port']) ? $config['port'] : $options['port']),
$config['path']
));
$this->remote = $config['remotepath'];
}
public function getTargetDir($index)
{
$indexdir = "{$this->url}/$index";
if (!WebDAV::exists($indexdir)) {
WebDAV::mkdir($indexdir);
}
$targetDir = "{$this->url}/$index/conf";
if (!WebDAV::exists($targetDir)) {
WebDAV::mkdir($targetDir);
}
return $targetDir;
}
public function uploadFile($index, $file)
{
$targetDir = $this->getTargetDir($index);
WebDAV::upload_from_file($file, $targetDir.'/'.basename($file));
}
public function uploadString($index, $filename, $string)
{
$targetDir = $this->getTargetDir($index);
WebDAV::upload_from_string($string, "$targetDir/$filename");
}
public function instanceDir($index)
{
return $this->remote ? "{$this->remote}/$index" : $index;
}
}

View File

@ -2,7 +2,6 @@
namespace SilverStripe\FullTextSearch\Solr;
Solr::include_client_api();
use SilverStripe\Dev\BuildTask;
use SilverStripe\Control\Director;
use SilverStripe\FulltextSearch\SearchIndex;

View File

@ -1,8 +1,5 @@
<?php
namespace SilverStripe\FullTextSearch\Solr;
class Solr3Service_Core extends SolrService_Core
{
}
namespace SilverStripe\FullTextSearch\Solr\Services;
class Solr3Service extends SolrService
{

View File

@ -0,0 +1,8 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Services;
class Solr3Service_Core extends SolrService_Core
{
}

View File

@ -0,0 +1,7 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Services;
class Solr4Service extends SolrService
{
private static $core_class = 'Solr4Service_Core';
}

View File

@ -1,5 +1,7 @@
<?php
namespace SilverStripe\FullTextSearch\Solr;
namespace SilverStripe\FullTextSearch\Solr\Services;
class Solr4Service_Core extends SolrService_Core
{
/**
@ -18,9 +20,9 @@ class Solr4Service_Core extends SolrService_Core
$rawPost = '<commit expungeDeletes="' . $expungeValue . '" waitSearcher="' . $searcherValue . '" />';
return $this->_sendRawPost($this->_updateUrl, $rawPost, $timeout);
}
/**
* @inheritdoc
* @inheritdoc
* @see Solr4Service_Core::addDocuments
*/
public function addDocument(Apache_Solr_Document $document, $allowDups = false,
@ -50,9 +52,4 @@ class Solr4Service_Core extends SolrService_Core
return $this->add($rawPost);
}
}
class Solr4Service extends SolrService
{
private static $core_class = 'Solr4Service_Core';
}
}

View File

@ -1,16 +1,8 @@
<?php
namespace SilverStripe\FullTextSearch\Solr;
Solr::include_client_api();
namespace SilverStripe\FullTextSearch\Solr\Services;
use SilverStripe\Core\Config\Config;
use \Apache_Solr_Service;
/**
* The API for accessing a specific core of a Solr server. Exactly the same as Apache_Solr_Service for now.
*/
class SolrService_Core extends Apache_Solr_Service
{
}
use SilverStripe\FullTextSearch\Solr\Solr;
Solr::include_client_api();
/**
* The API for accessing the primary Solr installation, which includes both SolrService_Core,
* plus extra methods for interrogating, creating, reloading and getting SolrService_Core instances

View File

@ -0,0 +1,18 @@
<?php
/**
* Created by PhpStorm.
* User: elliot
* Date: 21/04/17
* Time: 12:45 PM
*/
namespace SilverStripe\FullTextSearch\Solr\Services;
use \Apache_Solr_Service;
/**
* The API for accessing a specific core of a Solr server. Exactly the same as Apache_Solr_Service for now.
*/
class SolrService_Core extends Apache_Solr_Service
{
}

View File

@ -0,0 +1,36 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Stores;
/**
* Class SolrConfigStore
*
* The interface Solr_Configure uses to upload configuration files to Solr
*/
interface SolrConfigStore
{
/**
* Upload a file to Solr for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
* @param $file string - A path to a file to upload. The base name of the file will be used on the remote side
* @return null
*/
public function uploadFile($index, $file);
/**
* Upload a file to Solr from a string for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
* @param $filename string - The base name of the file to use on the remote side
* @param $strong string - The contents of the file
* @return null
*/
public function uploadString($index, $filename, $string);
/**
* Get the instanceDir to tell Solr to use for index $index
* @param $index string - The name of an index (which is also used as the name of the Solr core for the index)
*/
public function instanceDir($index);
}

View File

@ -0,0 +1,53 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Stores;
/**
* Class SolrConfigStore_File
*
* A ConfigStore that uploads files to a Solr instance on a locally accessible filesystem
* by just using file copies
*/
class SolrConfigStore_File implements SolrConfigStore
{
public function __construct($config)
{
$this->local = $config['path'];
$this->remote = isset($config['remotepath']) ? $config['remotepath'] : $config['path'];
}
public function getTargetDir($index)
{
$targetDir = "{$this->local}/{$index}/conf";
if (!is_dir($targetDir)) {
$worked = @mkdir($targetDir, 0770, true);
if (!$worked) {
throw new RuntimeException(
sprintf('Failed creating target directory %s, please check permissions', $targetDir)
);
}
}
return $targetDir;
}
public function uploadFile($index, $file)
{
$targetDir = $this->getTargetDir($index);
copy($file, $targetDir.'/'.basename($file));
}
public function uploadString($index, $filename, $string)
{
$targetDir = $this->getTargetDir($index);
file_put_contents("$targetDir/$filename", $string);
}
public function instanceDir($index)
{
return $this->remote.'/'.$index;
}
}

View File

@ -0,0 +1,56 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Stores;
/**
* Class SolrConfigStore_WebDAV
*
* A ConfigStore that uploads files to a Solr instance via a WebDAV server
*/
class SolrConfigStore_WebDAV implements SolrConfigStore
{
public function __construct($config)
{
$options = Solr::solr_options();
$this->url = implode('', array(
'http://',
isset($config['auth']) ? $config['auth'].'@' : '',
$options['host'].':'.(isset($config['port']) ? $config['port'] : $options['port']),
$config['path']
));
$this->remote = $config['remotepath'];
}
public function getTargetDir($index)
{
$indexdir = "{$this->url}/$index";
if (!WebDAV::exists($indexdir)) {
WebDAV::mkdir($indexdir);
}
$targetDir = "{$this->url}/$index/conf";
if (!WebDAV::exists($targetDir)) {
WebDAV::mkdir($targetDir);
}
return $targetDir;
}
public function uploadFile($index, $file)
{
$targetDir = $this->getTargetDir($index);
WebDAV::upload_from_file($file, $targetDir.'/'.basename($file));
}
public function uploadString($index, $filename, $string)
{
$targetDir = $this->getTargetDir($index);
WebDAV::upload_from_string($string, "$targetDir/$filename");
}
public function instanceDir($index)
{
return $this->remote ? "{$this->remote}/$index" : $index;
}
}

View File

@ -0,0 +1,63 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Tasks;
use SilverStripe\Dev\BuildTask;
/**
* Abstract class for build tasks
*/
class Solr_BuildTask extends BuildTask
{
protected $enabled = false;
/**
* Logger
*
* @var LoggerInterface
*/
protected $logger = null;
/**
* Get the current logger
*
* @return LoggerInterface
*/
public function getLogger()
{
return Injector::inst()->get('Logger');
}
/**
* Assign a new logger
*
* @param LoggerInterface $logger
*/
public function setLogger(LoggerInterface $logger)
{
$this->logger = $logger;
}
/**
* @return SearchLogFactory
*/
protected function getLoggerFactory()
{
// return Injector::inst()->get('SearchLogFactory');
}
/**
* Setup task
*
* @param SS_HTTPReqest $request
*/
public function run($request)
{
$name = get_class($this);
$verbose = $request->getVar('verbose');
// Set new logger
$logger = $this
->getLoggerFactory();
//@todo: Cannot instantiate interface SearchLogFactory
// ->getOutputLogger($name, $verbose);
$this->setLogger($logger);
}
}

View File

@ -0,0 +1,81 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Tasks;
class Solr_Configure extends Solr_BuildTask
{
protected $enabled = true;
public function run($request)
{
parent::run($request);
// Find the IndexStore handler, which will handle uploading config files to Solr
$store = $this->getSolrConfigStore();
$indexes = Solr::get_indexes();
foreach ($indexes as $instance) {
try {
$this->updateIndex($instance, $store);
} catch (Exception $e) {
// We got an exception. Warn, but continue to next index.
$this
->getLogger()
->error("Failure: " . $e->getMessage());
}
}
}
/**
* Update the index on the given store
*
* @param SolrIndex $instance Instance
* @param SolrConfigStore $store
*/
protected function updateIndex($instance, $store)
{
$index = $instance->getIndexName();
$this->getLogger()->addInfo("Configuring $index.");
// Upload the config files for this index
$this->getLogger()->addInfo("Uploading configuration ...");
$instance->uploadConfig($store);
// Then tell Solr to use those config files
$service = Solr::service();
if ($service->coreIsActive($index)) {
$this->getLogger()->addInfo("Reloading core ...");
$service->coreReload($index);
} else {
$this->getLogger()->addInfo("Creating core ...");
$service->coreCreate($index, $store->instanceDir($index));
}
$this->getLogger()->addInfo("Done");
}
/**
* Get config store
*
* @return SolrConfigStore
*/
protected function getSolrConfigStore()
{
$options = Solr::solr_options();
if (!isset($options['indexstore']) || !($indexstore = $options['indexstore'])) {
throw new Exception('No index configuration for Solr provided', E_USER_ERROR);
}
// Find the IndexStore handler, which will handle uploading config files to Solr
$mode = $indexstore['mode'];
if ($mode == 'file') {
return new SolrConfigStore_File($indexstore);
} elseif ($mode == 'webdav') {
return new SolrConfigStore_WebDAV($indexstore);
} elseif (ClassInfo::exists($mode) && ClassInfo::classImplements($mode, 'SolrConfigStore')) {
return new $mode($indexstore);
} else {
user_error('Unknown Solr index mode '.$indexstore['mode'], E_USER_ERROR);
}
}
}

View File

@ -0,0 +1,148 @@
<?php
namespace SilverStripe\FullTextSearch\Solr\Tasks;
/**
* Task used for both initiating a new reindex, as well as for processing incremental batches
* within a reindex.
*
* When running a complete reindex you can provide any of the following
* - class (to limit to a single class)
* - verbose (optional)
*
* When running with a single batch, provide the following querystring arguments:
* - start
* - index
* - class
* - variantstate
* - verbose (optional)
*/
class Solr_Reindex extends Solr_BuildTask
{
protected $enabled = true;
/**
* Number of records to load and index per request
*
* @var int
* @config
*/
private static $recordsPerRequest = 200;
/**
* Get the reindex handler
*
* @return SolrReindexHandler
*/
protected function getHandler()
{
//@todo: this needs to determine the best class from a Factory implementation
//@todo: it was 'SolrReindexHandler' but that doesn't work on 4.0
return Injector::inst()->get('SolrReindexImmediateHandler');
}
/**
* @param SS_HTTPRequest $request
*/
public function run($request)
{
parent::run($request);
// Reset state
$originalState = SearchVariant::current_state();
$this->doReindex($request);
SearchVariant::activate_state($originalState);
}
/**
* @param SS_HTTPRequest $request
*/
protected function doReindex($request)
{
$class = $request->getVar('class');
$index = $request->getVar('index');
//find the index classname by IndexName
// this is for when index names do not match the class name (this can be done by overloading getIndexName() on
// indexes
if ($index && !ClassInfo::exists($index)) {
foreach(ClassInfo::subclassesFor('SolrIndex') as $solrIndexClass) {
$reflection = new ReflectionClass($solrIndexClass);
//skip over abstract classes
if (!$reflection->isInstantiable()) {
continue;
}
//check the indexname matches the index passed to the request
if (!strcasecmp(singleton($solrIndexClass)->getIndexName(), $index)) {
//if we match, set the correct index name and move on
$index = $solrIndexClass;
break;
}
}
}
// Deprecated reindex mechanism
$start = $request->getVar('start');
if ($start !== null) {
// Run single batch directly
$indexInstance = singleton($index);
$state = json_decode($request->getVar('variantstate'), true);
$this->runFrom($indexInstance, $class, $start, $state);
return;
}
// Check if we are re-indexing a single group
// If not using queuedjobs, we need to invoke Solr_Reindex as a separate process
// Otherwise each group is processed via a SolrReindexGroupJob
$groups = $request->getVar('groups');
$handler = $this->getHandler();
if ($groups) {
// Run grouped batches (id % groups = group)
$group = $request->getVar('group');
$indexInstance = singleton($index);
$state = json_decode($request->getVar('variantstate'), true);
$handler->runGroup($this->getLogger(), $indexInstance, $state, $class, $groups, $group);
return;
}
// If run at the top level, delegate to appropriate handler
$self = get_class($this);
$handler->triggerReindex($this->getLogger(), $this->config()->recordsPerRequest, $self, $class);
}
/**
* @deprecated since version 2.0.0
*/
protected function runFrom($index, $class, $start, $variantstate)
{
DeprecationTest_Deprecation::notice('2.0.0', 'Solr_Reindex now uses a new grouping mechanism');
// Set time limit and state
increase_time_limit_to();
SearchVariant::activate_state($variantstate);
// Generate filtered list
$items = DataList::create($class)
->limit($this->config()->recordsPerRequest, $start);
// Add child filter
$classes = $index->getClasses();
$options = $classes[$class];
if (!$options['include_children']) {
$items = $items->filter('ClassName', $class);
}
// Process selected records in this class
$this->getLogger()->info("Adding $class");
foreach ($items->sort("ID") as $item) {
$this->getLogger()->debug($item->ID);
// See SearchUpdater_ObjectHandler::triggerReindex
$item->triggerReindex();
$item->destroy();
}
$this->getLogger()->info("Done");
}
}