Merge 2e20b4338d into c39c4b4983

_config/binding.yml

@@ -1,3 +1,6 @@
+---
+Name: fulltextbinding
+---
 Injector:
   RequestProcessor:
     properties:

_config/solr.yml

@@ -0,0 +1,12 @@
+---
+Name: solrconfig
+---
+Injector:
+  SolrService_Factory:
+    class: Solr_CacheFactory
+    constructor:
+      - '%$Solr'
+SolrIndex:
+  dependencies:
+    ServiceFactory: %$SolrService_Factory
+

code/search/SearchIndex.php

@@ -30,6 +30,7 @@
 abstract class SearchIndex extends ViewableData {
 
 	function __construct() {
+		parent::__construct();
 		$this->init();
 
 		foreach ($this->getClasses() as $class => $options) {

code/search/processors/SearchUpdateMessageQueueProcessor.php

@@ -1,5 +1,7 @@
 <?php
 
+if(!class_exists('MessageQueue')) return;
+
 class SearchUpdateMessageQueueProcessor extends SearchUpdateProcessor {
 	/**
 	 * The MessageQueue to use when processing updates

code/search/processors/SearchUpdateQueuedJobProcessor.php

@@ -1,5 +1,6 @@
 <?php
 
+if(!interface_exists('QueuedJob')) return;
 
 class SearchUpdateQueuedJobProcessor extends SearchUpdateProcessor implements QueuedJob {
 

code/solr/Solr.php

@@ -1,6 +1,23 @@
 <?php
 
-class Solr  {
+/**
+ * Interface for a factory for SolrService instances
+ */
+interface SolrService_Factory {
+	
+	/**
+	 * Generates a SolrServiceEngine for a given core
+	 * 
+	 * @param string $core
+	 * @return SolrService_Engine
+	 */
+	public function getService($core = null);
+}
+
+/**
+ * Main Solr controller and SolrService factory
+ */
+class Solr implements SolrService_Factory {
 
 	/**
 	 * Configuration on where to find the solr server and how to get new index configurations into it.
@@ -137,6 +154,35 @@ class Solr  {
 			$included = true;
 		}
 	}
+
+	public function getService($core = null) {
+		return self::service($core);
+	}
+
+}
+
+
+/**
+ * Provides caching of services
+ */
+class Solr_CacheFactory implements SolrService_Factory {
+	
+	/**
+	 * Parent factory to generate cached results for
+	 *
+	 * @var SolrService_Factory
+	 */
+	protected $parent = null;
+	
+	public function __construct(SolrService_Factory $parent) {
+		$this->parent = $parent;
+	}
+
+	public function getService($core = null) {
+		$service = $this->parent->getService($core);
+		return Injector::inst()->createWithArgs('SolrService_Cache', array($service));
+	}
+
 }
 
 

code/solr/SolrIndex.php

@@ -31,6 +31,32 @@ abstract class SolrIndex extends SearchIndex {
 	protected $extrasPath = null;
 
 	protected $templatesPath = null;
+	
+	/**
+	 * Factory for generating services
+	 *
+	 * @var SolrService_Factory
+	 */
+	protected $serviceFactory = null;
+	
+	/*
+	 * Assign an active factory
+	 * 
+	 * @param SolrService_Factory $factory
+	 */
+	public function setServiceFactory(SolrService_Factory $factory) {
+		$this->serviceFactory = $factory;
+	}
+	
+	/**
+	 * Retrieves the current factory
+	 * 
+	 * @return SolrService_Factory
+	 */
+	public function getServiceFactory() {
+		return $this->serviceFactory;
+	}
+	
 	/**
 	 * @return String Absolute path to the folder containing
 	 * templates which are used for generating the schema and field definitions.
@@ -478,13 +504,20 @@ abstract class SolrIndex extends SearchIndex {
 		return new ArrayData($ret);
 	}
 
+	/**
+	 * Currently active service
+	 *
+	 * @var SolrService_Engine
+	 */
 	protected $service;
 
 	/**
-	 * @return SolrService
+	 * @return SolrService_Engine
 	 */
 	public function getService() {
-		if(!$this->service) $this->service = Solr::service(get_class($this));
+		if(!$this->service) {
+			$this->service = $this->serviceFactory->getService(get_class($this));
+		}
 		return $this->service;
 	}
 

code/solr/SolrService.php

@@ -2,10 +2,93 @@
 
 Solr::include_client_api();
 
+
+/**
+ * Interface to a SolrSearch instance
+ */
+interface SolrService_Engine {
+	
+	/**
+	 * Add a Solr Document to the index
+	 *
+	 * @param Apache_Solr_Document $document
+	 * @param boolean $allowDups
+	 * @param boolean $overwritePending
+	 * @param boolean $overwriteCommitted
+	 * @param integer $commitWithin The number of milliseconds that a document must be committed within, see @{link http://wiki.apache.org/solr/UpdateXmlMessages#The_Update_Schema} for details.  If left empty this property will not be set in the request.
+	 * @return Apache_Solr_Response
+	 *
+	 * @throws Apache_Solr_HttpTransportException If an error occurs during the service call
+	 */
+	public function addDocument(Apache_Solr_Document $document, $allowDups = false, $overwritePending = true, $overwriteCommitted = true, $commitWithin = 0);
+	
+	/**
+	 * Create a delete document based on document ID
+	 *
+	 * @param string $id Expected to be utf-8 encoded
+	 * @param boolean $fromPending
+	 * @param boolean $fromCommitted
+	 * @param float $timeout Maximum expected duration of the delete operation on the server (otherwise, will throw a communication exception)
+	 * @return Apache_Solr_Response
+	 *
+	 * @throws Apache_Solr_HttpTransportException If an error occurs during the service call
+	 */
+	public function deleteById($id, $fromPending = true, $fromCommitted = true, $timeout = 3600);
+	
+	/**
+	 * Send a commit command.  Will be synchronous unless both wait parameters are set to false.
+	 *
+	 * @param boolean $expungeDeletes Defaults to false, merge segments with deletes away
+	 * @param boolean $waitFlush Defaults to true,  block until index changes are flushed to disk
+	 * @param boolean $waitSearcher Defaults to true, block until a new searcher is opened and registered as the main query searcher, making the changes visible
+	 * @param float $timeout Maximum expected duration (in seconds) of the commit operation on the server (otherwise, will throw a communication exception). Defaults to 1 hour
+	 * @return Apache_Solr_Response
+	 *
+	 * @throws Apache_Solr_HttpTransportException If an error occurs during the service call
+	 */
+	public function commit($expungeDeletes = false, $waitFlush = true, $waitSearcher = true, $timeout = 3600);
+	
+	/**
+	 * Simple Search interface
+	 *
+	 * @param string $query The raw query string
+	 * @param int $offset The starting offset for result documents
+	 * @param int $limit The maximum number of result documents to return
+	 * @param array $params key / value pairs for other query parameters (see Solr documentation), use arrays for parameter keys used more than once (e.g. facet.field)
+	 * @param string $method The HTTP method (Apache_Solr_Service::METHOD_GET or Apache_Solr_Service::METHOD::POST)
+	 * @return Apache_Solr_Response
+	 *
+	 * @throws Apache_Solr_HttpTransportException If an error occurs during the service call
+	 * @throws Apache_Solr_InvalidArgumentException If an invalid HTTP method is used
+	 */
+	public function search($query, $offset = 0, $limit = 10, $params = array(), $method = self::METHOD_GET);
+	
+	/**
+	 * Get the set path.
+	 *
+	 * @return string
+	 */
+	public function getPath();
+}
+
+/**
+ * Interface to a SolrService_Engine which has additional reporting capabilities
+ */
+interface SolrService_Engine_Reportable extends SolrService_Engine {
+	
+	/**
+	 * Return an array of data describing the last search
+	 * 
+	 * @return array
+	 */
+	public function getReport();
+}
+
 /**
  * The API for accessing a specific core of a Solr server. Exactly the same as Apache_Solr_Service for now.
  */
-class SolrService_Core extends Apache_Solr_Service {
+class SolrService_Core extends Apache_Solr_Service implements SolrService_Engine {
+
 }
 
 /**
@@ -75,3 +158,178 @@ class SolrService extends SolrService_Core {
 		return new $klass($this->_host, $this->_port, $this->_path.$core, $this->_httpTransport);
 	}
 }
+
+
+/**
+ * Provides caching and rate limiting optimisations to {@see SolrIndex}
+ */
+class SolrService_Cache implements SolrService_Engine_Reportable {
+	
+	/**
+	 * Number of seconds to cache results for.
+	 * 
+	 * @config
+	 * @var int
+	 */
+	private static $cache_lifetime = 300;
+	
+	/**
+	 * True if caching should be enabled
+	 * 
+	 * @config
+	 * @var bool
+	 */
+	private static $cache_enabled = true;
+	
+	/**
+	 * Parent search service to cache
+	 *
+	 * @var SolrService_Engine
+	 */
+	protected $parent = null;
+	
+	/**
+	 * Indicate whether the last attempt to call search was a cache hit
+	 *
+	 * @var bool
+	 */
+	protected $cacheHit = false;
+	
+	public function __construct(SolrService_Engine $parent) {
+		$this->setParent($parent);
+		$this->cacheHit = false;
+	}
+	
+	/**
+	 * Gets the parent service from this caching service
+	 * 
+	 * @return SolrService_Engine
+	 */
+	public function getParent() {
+		return $this->parent;
+	}
+	
+	/**
+	 * Assign a new parent service
+	 * 
+	 * @param SolrService_Engine $parent
+	 */
+	public function setParent(SolrService_Engine $parent) {
+		$this->parent = $parent;
+	}
+	
+	/**
+	 * Gets the cache to use
+	 * 
+	 * @return Zend_Cache_Frontend
+	 */
+	protected function getFilterCache() {
+		$cache = SS_Cache::factory('SolrService_Cache');
+		$cache->setOption('automatic_serialization', true);
+		return $cache;
+	}
+	
+	/**
+	 * Discards cached data
+	 */
+	protected function invalidateCache() {
+		$this
+			->getFilterCache()
+			->clean(Zend_Cache::CLEANING_MODE_ALL);
+	}
+	
+	/**
+	 * Determines the key to use for saving cached results for a query
+	 * 
+	 * @param SolrService $service Source service to query
+	 * @param array $arguments Arguments to be passed to SolrService::query
+	 * @return string Result key
+	 */
+	protected function getCacheKey($arguments) {
+		// Distinguish this service by path
+		$entropy = $this->parent->getPath();
+		
+		// Identify search by search query
+		$entropy .= serialize($arguments);
+		
+		// Distinguish this service by path and query arguments
+		return 'SolrQueryFilter_Cache_' . md5($entropy);
+	}
+	
+	public function search($query, $offset = 0, $limit = 10, $params = array(), $method = self::METHOD_GET) {
+		$this->cacheHit = false;
+		
+		// Check for cached result
+		$arguments = func_get_args();
+		if($result = $this->getCachedResult($arguments)) {
+			$this->cacheHit = true;
+			return $result;
+		}
+		
+		// Generate result
+		$result = $this->parent->search($query, $offset, $limit, $params, $method);
+		
+		// Save cached result
+		$this->setCachedResult($arguments, $result);
+		return $result;
+	}
+	
+	/**
+	 * Attempt to retrieve cached results for a query
+	 * 
+	 * @param array $arguments Arguments to be passed to SolrService::query
+	 * @return Apache_Solr_Response A resulting cached query, if available, or null otherwise
+	 */
+	protected function getCachedResult($arguments) {
+		// Bypass caching if disabled
+		if(!Config::inst()->get(get_class(), 'cache_enabled')) return null;
+		
+		// Retrieve result
+		$cache = $this->getFilterCache();
+		$cacheKey = $this->getCacheKey($arguments);
+		return $cache->load($cacheKey);
+	}
+	
+	/**
+	 * Save cached results
+	 * 
+	 * @param array $arguments Arguments to be passed to SolrService::query
+	 * @param Apache_Solr_Response $results Result of either call to SolrService::query, or
+	 * the value of any cached result. This may be null if no results are available.
+	 */
+	protected function setCachedResult($arguments, $results) {
+		// Bypass caching if disabled
+		if(!Config::inst()->get(get_class(), 'cache_enabled') || empty($results)) return;
+		
+		// Store result
+		$cache = $this->getFilterCache();
+		$cacheKey = $this->getCacheKey($arguments);
+		$cacheLifetime = Config::inst()->get(get_class(), 'cache_lifetime');
+		$cache->save($results, $cacheKey, array(), $cacheLifetime);
+	}
+
+	public function addDocument(\Apache_Solr_Document $document, $allowDups = false, $overwritePending = true, $overwriteCommitted = true, $commitWithin = 0) {
+		$this->invalidateCache();
+		return $this->parent->addDocument($document, $allowDups, $overwritePending, $overwriteCommitted, $commitWithin);
+	}
+	
+	public function deleteById($id, $fromPending = true, $fromCommitted = true, $timeout = 3600) {
+		$this->invalidateCache();
+		return $this->parent->deleteById($id, $fromPending, $fromCommitted, $timeout);
+	}
+	
+	public function commit($expungeDeletes = false, $waitFlush = true, $waitSearcher = true, $timeout = 3600) {
+		$this->invalidateCache();
+		return $this->parent->commit($expungeDeletes, $waitFlush, $waitSearcher, $timeout);
+	}
+
+	public function getPath() {
+		return $this->parent->getPath();
+	}
+	
+	public function getReport() {
+		return array(
+			'cachehit' => $this->cacheHit
+		);
+	}
+}

tests/SolrCacheTest.php

@@ -0,0 +1,133 @@
+<?php
+
+if (class_exists('Phockito')) Phockito::include_hamcrest();
+
+/**
+ * Description of SolrQueryFilterTest
+ *
+ * @author dmooyman
+ */
+class SolrCacheTest extends SapphireTest {
+
+	protected $extraDataObjects = array(
+		'SearchUpdaterTest_Container'
+	);
+
+	protected static $fixture_file = 'SolrCacheTest.yml';
+
+	public function setUp() {
+		parent::setUp();
+
+		if (!class_exists('Phockito')) {
+			$this->skipTest = true;
+			return $this->markTestSkipped("These tests need the Phockito module installed to run");
+		}
+
+		// flush cache
+		$cache = $this->getFilterCache();
+		$cache->clean(Zend_Cache::CLEANING_MODE_ALL);
+	}
+
+	protected function getFilterCache() {
+		$cache = SS_Cache::factory('SolrService_Cache');
+		$cache->setOption('automatic_serialization', true);
+		return $cache;
+	}
+
+	protected function getServiceMock($operations) {
+		$service = Phockito::mock('Solr3Service');
+		foreach($operations as $input => $items) {
+			$response = $this->getResponseMock($items);
+			Phockito::when($service->search($input, anything(), anything(), anything(), anything()))
+				->return($response);
+		}
+		return $service;
+	}
+
+	protected function getResponseMock($items) {
+		$response = Phockito::mock('Apache_Solr_Response');
+		Phockito::when($response->getHttpStatus())->return(200);
+		$response->response = new stdClass();
+		$response->response->numFound = count($items);
+		$response->response->docs = $items;
+		return $response;
+	}
+
+	/**
+	 * Performs a search using the relevant text
+	 * 
+	 * @param SolrQueryFilterTest_BaseIndex $index
+	 * @param string $text
+	 * @return ArrayData
+	 */
+	protected function doSearch($index, $text) {
+		$query = new SearchQuery();
+		$query->search($text);
+		return $index->search($query);
+	}
+
+	/**
+	 * Helper function for testCaching()
+	 * 
+	 * @param SolrQueryFilterTest_BaseIndex $index
+	 * @param SearchUpdaterTest_Container $first Expected results of search for 'Just First'
+	 * @param SearchUpdaterTest_Container $second Expected results of search for 'Get Second'
+	 */
+	protected function checkResults($index, $first, $second) {
+		// Cache search for first item
+		$result1 = $this->doSearch($index, 'Just First');
+		$this->assertEquals(1, $result1->Matches->count());
+		$this->assertEquals($first->ID, $result1->Matches->first()->ID);
+		$this->assertEquals('SearchUpdaterTest_Container', $result1->Matches->first()->ClassName);
+
+		// Cache search for second item
+		$result2 = $this->doSearch($index, 'Get Second');
+		$this->assertEquals(1, $result2->Matches->count());
+		$this->assertEquals($second->ID, $result2->Matches->first()->ID);
+		$this->assertEquals('SearchUpdaterTest_Container', $result2->Matches->last()->ClassName);
+	}
+
+	public function testCaching() {
+		// Setup mocks
+		$item1 = $this->objFromFixture('SearchUpdaterTest_Container', 'item1');
+		$item2 = $this->objFromFixture('SearchUpdaterTest_Container', 'item2');
+		$item3 = $this->objFromFixture('SearchUpdaterTest_Container', 'item3');
+		$service = $this->getServiceMock(array(
+			'+Just +First' => array($item1),
+			'+Get +Second' => array($item2),
+			'+Get +Both' => array($item1, $item2)
+		));
+		$index = singleton('SolrCacheTest_TestIndex');
+		$index->getService()->setParent($service);
+
+		// Do initial search (note that 'Get Both' is never called)
+		$this->checkResults($index, $item1, $item2);
+
+		// Change behaviour of service to see if cached results are still returned
+		$service = $this->getServiceMock(array(
+			'+Just +First' => array($item2),
+			'+Get +Second' => array($item3),
+			'+Get +Both' => array($item2, $item3)
+		));
+		$index->getService()->setParent($service);
+		$this->checkResults($index, $item1, $item2);
+
+		// Search for 'Get Both' to make sure an uncached hit still gets through
+		$bothResult = $this->doSearch($index, 'Get Both');
+		$this->assertEquals(2, $bothResult->Matches->count());
+		$this->assertEquals($item2->ID, $bothResult->Matches->first()->ID);
+		$this->assertEquals($item3->ID, $bothResult->Matches->last()->ID);
+	}
+}
+
+class SolrCacheTest_TestIndex extends SolrIndex implements TestOnly {
+
+	public function init() {
+		$this->addClass('SearchUpdaterTest_Container');
+
+		$this->addFilterField('Field1');
+		$this->addFilterField('MyDate', 'Date');
+		$this->addFilterField('HasOneObject.Field1');
+		$this->addFilterField('HasManyObjects.Field1');
+	}
+}

tests/SolrCacheTest.yml

@@ -0,0 +1,10 @@
+SearchUpdaterTest_Container:
+  item1:
+    Field1: 'First'
+    Field2: 'Second'
+  item2:
+    Field1: 'Alpha'
+    Field2: 'Bravo'
+  item3:
+    Field1: 'Electrode'
+    Field2: 'Diglett'

tests/SolrIndexVersionedTest.php

@@ -54,6 +54,11 @@ class SolrIndexVersionedTest extends SapphireTest {
 		return Phockito::mock('Solr3Service');
 	}
 	
+	protected function getExpectedDocumentId($id, $stage) {
+		// Prevent subsites from breaking tests
+		$subsites = class_exists('Subsite') ? '"SearchVariantSubsites":"0",' : '';
+		return $id.'-SiteTree-{'.$subsites.'"SearchVariantVersioned":"'.$stage.'"}';
+	}
 	
 	public function testPublishing() {
 		
@@ -68,7 +73,7 @@ class SolrIndexVersionedTest extends SapphireTest {
 		$item->write();
 		SearchUpdater::flush_dirty_indexes();
 		$doc = new SolrDocumentMatcher(array(
-			'_documentid' => $item->ID.'-SiteTree-{"SearchVariantVersioned":"Stage"}',
+			'_documentid' => $this->getExpectedDocumentId($item->ID, 'Stage'),
 			'ClassName' => 'SearchVariantVersionedTest_Item'
 		));
 		Phockito::verify($serviceMock)->addDocument($doc);
@@ -81,7 +86,7 @@ class SolrIndexVersionedTest extends SapphireTest {
 		$item->publish('Stage', 'Live');
 		SearchUpdater::flush_dirty_indexes();
 		$doc = new SolrDocumentMatcher(array(
-			'_documentid' => $item->ID.'-SiteTree-{"SearchVariantVersioned":"Live"}',
+			'_documentid' => $this->getExpectedDocumentId($item->ID, 'Live'),
 			'ClassName' => 'SearchVariantVersionedTest_Item'
 		));
 		Phockito::verify($serviceMock)->addDocument($doc);
@@ -104,9 +109,9 @@ class SolrIndexVersionedTest extends SapphireTest {
 		$item->delete();
 		SearchUpdater::flush_dirty_indexes();
 		Phockito::verify($serviceMock, 1)
-			->deleteById($id.'-SiteTree-{"SearchVariantVersioned":"Live"}');
+			->deleteById($this->getExpectedDocumentId($id, 'Live'));
 		Phockito::verify($serviceMock, 0)
-			->deleteById($id.'-SiteTree-{"SearchVariantVersioned":"Stage"}');
+			->deleteById($this->getExpectedDocumentId($id, 'Stage'));
 		
 		// Delete the stage record
 		Versioned::reading_stage('Stage');
@@ -118,9 +123,9 @@ class SolrIndexVersionedTest extends SapphireTest {
 		$item->delete();
 		SearchUpdater::flush_dirty_indexes();
 		Phockito::verify($serviceMock, 1)
-			->deleteById($id.'-SiteTree-{"SearchVariantVersioned":"Stage"}');
+			->deleteById($this->getExpectedDocumentId($id, 'Stage'));
 		Phockito::verify($serviceMock, 0)
-			->deleteById($id.'-SiteTree-{"SearchVariantVersioned":"Live"}');
+			->deleteById($this->getExpectedDocumentId($id, 'Live'));
 	}
 }