Static caching merges from dnc branch

git-svn-id: svn://svn.silverstripe.com/silverstripe/open/modules/cms/trunk@68901 467b73ca-7a2a-4603-9d3b-597d59a354a9

_config.php

@@ -15,7 +15,7 @@ Director::addRules(50, array(
 	'admin//ImageEditor/$Action' => 'ImageEditor',
 	'admin/cms//$Action/$ID/$OtherID' => 'CMSMain', 
 	'PageComment//$Action/$ID' => 'PageComment_Controller',
-	'dev/buildcache' => 'RebuildStaticCacheTask',
+	'dev/buildcache/$Action' => 'RebuildStaticCacheTask',
 ));
 
 CMSMenu::populate_menu();

code/staticpublisher/CachedPHPPage.tmpl

@@ -0,0 +1,28 @@
+<?php
+
+/**
+ * This is a system-generated PHP script that performs header management for the statically cached content given below.
+ */
+
+define('MAX_AGE', '**MAX_AGE**');
+define('LAST_MODIFIED', '**LAST_MODIFIED**');
+
+if(MAX_AGE > 0) {
+	header("Cache-Control: max-age=" . MAX_AGE);
+	header("Pragma:");
+} else {
+	header("Cache-Control: no-cache, max-age=0, must-revalidate");
+}
+
+header("Expires: " . gmdate('D, d M Y H:i:s', time() + MAX_AGE) . ' GMT');
+header("Last-modified: " . gmdate('D, d M Y H:i:s', strtotime(LAST_MODIFIED)) . ' GMT');
+
+if(isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) {
+	if(strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']) >= strtotime(LAST_MODIFIED)) {
+		header("Last-modified: " . gmdate('D, d M Y H:i:s', strtotime(LAST_MODIFIED)) . ' GMT', true, 304);
+		exit;
+	}
+}
+
+?>
+**CONTENT**

code/staticpublisher/CachedPHPRedirection.tmpl

@@ -0,0 +1,20 @@
+<?php
+
+/**
+ * This is a system-generated PHP script that performs header management for a 301 redirection.
+ */
+
+define('DESTINATION', '**DESTINATION**');
+define('MAX_AGE', 3600);
+
+if(MAX_AGE > 0) {
+	header("Cache-Control: max-age=" . MAX_AGE);
+	header("Pragma:");
+} else {
+	header("Cache-Control: no-cache, max-age=0, must-revalidate");
+}
+
+header("Expires: " . gmdate('D, d M Y H:i:s', time() + MAX_AGE) . ' GMT');
+header("Location: " . DESTINATION, true, 301);
+
+?>

code/staticpublisher/FilesystemPublisher.php

@@ -20,7 +20,8 @@ class FilesystemPublisher extends StaticPublisher {
 	/**
 	 * @param $destFolder The folder to save the cached site into
 	 * @param $fileExtension  The file extension to use, for example, 'html'.  If omitted, then each page will be placed
-	 * in its own directory, with the filename 'index.html'
+	 * in its own directory, with the filename 'index.html'.  If you set the extension to PHP, then a simple PHP script will
+	 * be generated that can do appropriate cache & redirect header negotation
 	 */
 	function __construct($destFolder, $fileExtension = null) {
 		if(substr($destFolder, -1) == '/') $destFolder = substr($destFolder, 0, -1);
@@ -28,7 +29,9 @@ class FilesystemPublisher extends StaticPublisher {
 		$this->fileExtension = $fileExtension;
 	}
 	
-	function publishPages($urls) {
+	function publishPages($urls) { 
+		// This can be quite memory hungry and time-consuming
+		// @todo - Make a more memory efficient publisher
 		set_time_limit(0);
 		ini_set("memory_limit" , -1);
 		
@@ -42,6 +45,11 @@ class FilesystemPublisher extends StaticPublisher {
 		$totalURLs = sizeof($urls);
 		foreach($urls as $url) {
 			$i++;
+
+			if($url && !is_string($url)) {
+				user_error("Bad url:" . var_export($url,true), E_USER_WARNING);
+				continue;
+			}
 			
 			if(StaticPublisher::echo_progress()) {
 				echo " * Publishing page $i/$totalURLs: $url\n";
@@ -49,15 +57,47 @@ class FilesystemPublisher extends StaticPublisher {
 			}
 			
 			Requirements::clear();
-			$response = Director::test($url);
+			$response = Director::test(str_replace('+', ' ', $url));
 			Requirements::clear();
+			
+			DataObject::flush_and_destroy_cache();
+			DataObject::destroy_cached_get_calls(false);
+			DataObject::cache_get_calls(false);
+			
+			//echo 'Memory: ' . round(memory_get_usage()/100000)/10 . "\n";
 			/*
 			if(!is_object($response)) {
 				echo "String response for url '$url'\n";
 				print_r($response);
 			}*/
-			if(is_object($response)) $content = $response->getBody();
+			
-			else $content = $response . '';
+			// Generate file content			
+			// PHP file caching will generate a simple script from a template
+			if($this->fileExtension == 'php') {
+				if(is_object($response)) {
+					if($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
+						$content = $this->generatePHPCacheRedirection($response->getHeader('Location'));
+					} else {
+						$content = $this->generatePHPCacheFile($response->getBody(), HTTP::get_cache_age(), date('Y-m-d H:i:s'));
+					}
+				} else {
+					$content = $this->generatePHPCacheFile($response . '', HTTP::get_cache_age(), date('Y-m-d H:i:s'));
+				}
+				
+			// HTML file caching generally just creates a simple file
+			} else {
+				if(is_object($response)) {
+					if($response->getStatusCode() == '301' || $response->getStatusCode() == '302') {
+						$absoluteURL = Director::absoluteURL($response->getHeader('Location'));
+						$content = "<meta http-equiv=\"refresh\" content=\"2; URL=$absoluteURL\">";
+					} else {
+						$content = $response->getBody();
+					}
+				} else {
+					$content = $response . '';
+				}
+			}
+
 
 			if($this->fileExtension) $filename = $url ? "$url.$this->fileExtension" : "index.$this->fileExtension";
 			else $filename = $url ? "$url/index.html" : "index.html";
@@ -113,6 +153,27 @@ class FilesystemPublisher extends StaticPublisher {
 			}
 		}
 	}
+	
+	/**
+	 * Generate the templated content for a PHP script that can serve up the given piece of content with the given age and expiry
+	 */
+	protected function generatePHPCacheFile($content, $age, $lastModified) {
+		$template = file_get_contents('../cms/code/staticpublisher/CachedPHPPage.tmpl');
+		return str_replace(
+				array('**MAX_AGE**', '**LAST_MODIFIED**', '**CONTENT**'),
+				array((int)$age, $lastModified, $content),
+				$template);
+	}
+	/**
+	 * Generate the templated content for a PHP script that can serve up a 301 redirect to the given destionation
+	 */
+	protected function generatePHPCacheRedirection($destination) {
+		$template = file_get_contents('../cms/code/staticpublisher/CachedPHPRedirection.tmpl');
+		return str_replace(
+				array('**DESTINATION**'),
+				array($destination),
+				$template);
+	}
 }
 
-?>
+?>

code/staticpublisher/RsyncMultiHostPublisher.php

@@ -0,0 +1,37 @@
+<?php
+
+/**
+ * This static publisher can be used to deploy static content to multiple hosts, by generating the cache files locally and then rsyncing then to
+ * each destination box.  This can be used to set up a load-balanced collection of static servers.
+ */
+class RsyncMultiHostPublisher extends FilesystemPublisher {
+	/**
+	 * Array of rsync targets to publish to.  These can either be local file names, or scp-style targets, in the form "user@server:path"
+	 */
+	static $targets;
+	
+	/**
+	 * Set the targets to publish to.
+	 * @param $targets An array of targets to publish to.  These can either be local file names, or scp-style targets, in the form "user@server:path"
+	 */
+	static function set_targets($targets) {
+		self::$targets = $targets;
+	}
+
+	function publishPages($urls) {
+		parent::publishPages($urls);
+		$base = Director::baseFolder();
+
+		// Get variable that can turn off the rsync component of publication 
+		if(isset($_GET['norsync']) && $_GET['norsync']) return;
+		
+		foreach(self::$targets as $target) {
+			// Transfer non-PHP content from everything to the target; that will ensure that we have all the JS/CSS/etc
+			$rsyncOutput = `cd $base; rsync -av -e ssh --exclude /.htaccess --exclude '*.php' --exclude '*.svn' --exclude '*~' --delete . $target`;
+			// Then transfer "safe" PHP from the cache/ directory
+			$rsyncOutput .= `cd $base; rsync -av -e ssh --exclude '*.svn' --exclude '*~' --delete cache $target`;
+			if(StaticPublisher::echo_progress()) echo $rsyncOutput;
+		}
+	}
+	
+}

code/staticpublisher/StaticPublisher.php

@@ -34,6 +34,10 @@ abstract class StaticPublisher extends DataObjectDecorator {
 		}
 		
 		foreach($urls as $i => $url) {
+			if(!is_string($url)) {
+				user_error("Bad URL: " . var_export($url, true), E_USER_WARNING);
+				continue;
+			}
 			$url = Director::makeRelative($url);
 			if(substr($url,-1) == '/') $url = substr($url,0,-1);
 			$urls[$i] = $url;
@@ -76,4 +80,4 @@ abstract class StaticPublisher extends DataObjectDecorator {
 	}
 }
 
-?>
+?>

tasks/RebuildStaticCacheTask.php

@@ -5,6 +5,8 @@
  */ 
 class RebuildStaticCacheTask extends Controller {
 	function init() {
+		Versioned::reading_stage('live');
+
 		if(!Director::is_cli() && !Director::isDev() && !Permission::check("ADMIN")) Security::permissionFailure();
 		parent::init();
 	}
@@ -23,7 +25,7 @@ class RebuildStaticCacheTask extends Controller {
 		
 		if($_GET['urls']) $urls = $_GET['urls'];
 		else $urls = $page->allPagesToCache();
-
+		
 		$this->rebuildCache($urls, true);
 	}
 	
@@ -40,21 +42,43 @@ class RebuildStaticCacheTask extends Controller {
 		$page = singleton('Page');
 		
 		foreach($urls as $i => $url) {
+			if($url && !is_string($url)) {
+				user_error("Bad URL: " . var_export($url, true), E_USER_WARNING);
+				continue;
+			}
+
 			$url = Director::makeRelative($url);
-			if(substr($url,-1) == '/') $url = substr($url,0,-1);
+			// Exclude absolute links
-			$urls[$i] = $url;
+			if(preg_match('/^https?:/', $url)) {
+				unset($urls[$i]);
+			} else {
+				if(substr($url,-1) == '/') $url = substr($url,0,-1);
+				$urls[$i] = $url;
+			}
 		}
 		$urls = array_unique($urls);
-		
+		sort($urls);
-		if($removeAll) {
+
-			echo "Removing old cache... \n";
+		$start = isset($_GET['start']) ? $_GET['start'] : 0;
-			flush();
+		$count = isset($_GET['count']) ? $_GET['count'] : sizeof($urls);
+		if(($start + $count) > sizeof($urls)) $count = sizeof($urls) - $start;
+
+		$urls = array_slice($urls, $start, $count);
+
+		if(!isset($_GET['urls']) && $start == 0) {
+			echo "Removing old cache... ";
 			Filesystem::removeFolder("../cache", true);
 			echo "done.\n\n";
 		}
-
 		echo  "Republishing " . sizeof($urls) . " urls...\n\n";
 		$page->publishPages($urls);
 		echo "\n\n== Done! ==";
 	}
+	
+	function show() {
+		$urls = singleton('Page')->allPagesToCache();
+		echo "<pre>\n";
+		print_r($urls);
+		echo "\n</pre>";
+	}
 }