added crontask and updatetask buildtask as per userhelp, repositories yml, updated robots file, removed update shell script

app/_config.php

@@ -58,9 +58,7 @@ Config::inst()->update('DocumentationSearch', 'boost_by_path', array(
 ));
 
 // Set shared index (avoid issues with different temp paths between CLI and web users)
-if(file_exists(BASE_PATH . '/.lucene-index')) {
-	Config::inst()->update('DocumentationSearch', 'index_location', BASE_PATH . '/.lucene-index');
-}
+Config::inst()->update('DocumentationSearch', 'index_location', BASE_PATH . '/assets/.lucene-index');
 
 // Fix invalid character in iconv
 // see http://stackoverflow.com/questions/4723135/invalid-characters-for-lucene-text-search

app/_config/config.yml

@@ -9,4 +9,4 @@ StaticExporter:
   disable_sitetree_export: true
 Controller:
   extensions:
-    - ControllerExtension
+    - ControllerExtension

app/_config/docs-repositiories.yml

@@ -0,0 +1,32 @@
+---
+Name: docs-repos
+After:
+  - framework/*
+  - cms/*
+---
+UpdateTask:
+  documentation_repositories:
+    -
+      - silverstripe/silverstripe-framework
+      - framework
+      - master
+    -
+      - silverstripe/silverstripe-framework
+      - framework
+      - "3.2"
+    -
+      - silverstripe/silverstripe-framework
+      - framework
+      - "3.1"
+    -
+      - silverstripe/silverstripe-framework
+      - framework
+      - "3.0"
+    -
+      - silverstripe/silverstripe-framework
+      - framework
+      - "3"
+    -
+      - silverstripe/silverstripe-framework
+      - framework
+      - "2.4"

app/_config/routes.yml

@@ -1,5 +1,5 @@
 ---
-Name: docs
+Name: docs-routes
 After: framework/routes#coreroutes
 ---
 Director:

app/code/UpdateDocsCronTask.php

@@ -0,0 +1,27 @@
+<?php
+
+class UpdateDocsCronTask implements CronTask {
+	/**
+	 * 
+	 * @return string
+	 */
+	public function getSchedule() {
+		return "0 20 * * *";
+	}
+
+	/**
+	 * 
+	 * @return BuildTask
+	 */
+	public function process() {
+
+		//rebuild the docs
+		$docstask = new UpdateTask();
+		$docstask->run(null);
+
+		//reindex the search
+		$searchtask = new RebuildLuceneDocsIndex();
+		$searchtask->run(null);
+
+	}
+}

app/code/UpdateTask.php

@@ -0,0 +1,116 @@
+<?php
+
+class UpdateTask extends BuildTask
+{
+    /**
+     * @var string
+     */
+    protected $title = "Updates source markdown files";
+
+    /**
+     * @var string
+     */
+    protected $description = "Downloads and cleans source markdown documentation files";
+
+    /**
+     * @var bool
+     */
+    protected $enabled = true;
+
+    /**
+     * @var SS_HTTPRequest $request
+     *
+     * @todo test the initial unlink works with cloned modules
+     */
+    public function run($request)
+    {
+        $this->printLine("updating...");
+
+        $repositories = $this->getRepositories();
+
+        foreach ($repositories as $repository) {
+            $this->cloneRepository($repository);
+            $this->cleanRepository($repository);
+        }
+    }
+
+    /**
+     * @return string
+     *
+     * @todo document this new configuration parameter
+     */
+    private function getPath()
+    {
+        return ASSETS_PATH;
+    }
+
+    /**
+     * @param string $message
+     */
+    private function printLine($message)
+    {
+        $this->eol = Director::is_cli() ? PHP_EOL : "<br>";
+        print $message . $this->eol;
+        flush();
+    }
+
+    /**
+     * Returns the array of repos to source markdown docs from
+     *
+     * @return array
+     *
+     */
+    private function getRepositories()
+    {
+        if($repos = $this->config()->documentation_repositories)
+        {
+            return $repos;
+        } else {
+            user_error("You need to set 'UpdateTask:documentation_repositories' array in your yaml configuration", E_USER_WARNING);
+            return null;
+        }
+    }
+
+    /**
+     * @param array $repository
+     *
+     * @todo test this works with all modules
+     */
+    private function cloneRepository(array $repository)
+    {
+        list($remote, $folder, $branch) = $repository;
+
+        $path = $this->getPath();
+
+        if (!file_exists("{$path}/src")) {
+            mkdir("{$path}/src");
+        }
+
+        if (!file_exists("{$path}/src/{$folder}_{$branch}")) {
+            $this->printLine("cloning " . $remote . "/" . $branch);
+
+            chdir("{$path}/src");
+            exec("git clone -q git://github.com/{$remote}.git {$folder}_{$branch} --quiet");
+        }
+
+        chdir("{$path}/src/{$folder}_{$branch}");
+        exec("git fetch origin");
+        exec("git checkout -q origin/{$branch}");
+    }
+
+    /**
+     * Clears out any non markdown files stored in assets
+     *
+     * @param array $repository
+     */
+    private function cleanRepository(array $repository)
+    {
+        $files = array_merge(glob("*"), glob(".*"));
+
+        foreach ($files as $file) {
+            if ($file !== "docs" && $file !== "." && $file !== "..") {
+                exec("rm -rf {$file}");
+            }
+        }
+    }
+}

bin/update.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-
-dir=$1/assets
-
-if [ ! "$dir" ]; then
-  echo "Usage: $0 /base/folder/to/docs"
-  exit 1
-fi
-
-#=== FUNCTION ================================================================
-# NAME: 		checkout
-# DESCRIPTION:	Checks out a specific branch of a module into a folder. Not
-#				particular good for taking up space, but at the moment separate
-#				folders for each version we need will do.
-#
-#				The master branch will checked out by default
-# PARAMETERS:
-#				$1 - module path on github (e.g silverstripe/sapphire.git)
-#				$2 - branch name (e.g 3.0)
-#				$3 - module name (e.g sapphire)
-#
-#===============================================================================
-# Parameters: github path
-function checkout {
-	# Create dirs
-	if [ ! -d $dir/src ]; then
-		mkdir $dir/src
-	fi
-
-	if [ ! -d $dir/src/$2_$3 ]; then
-		echo "Cloning $1 branch $3"
-		cd $dir/src
-		git clone -b $3 --depth=100 git://github.com/$1 $dir/src/$2_$3 --quiet
-	else
-		echo "Updating $2 from branch $3"
-		cd $dir/src/$2_$3
-		git reset --hard -q
-		git fetch origin -q
-		git pull -q origin $3
-	fi
-}
-
-# core
-checkout 'silverstripe/silverstripe-framework.git' 'framework' 'master'
-checkout 'silverstripe/silverstripe-framework.git' 'framework' '3'
-checkout 'silverstripe/silverstripe-framework.git' 'framework' '3.3'
-checkout 'silverstripe/silverstripe-framework.git' 'framework' '3.2'
-checkout 'silverstripe/silverstripe-framework.git' 'framework' '3.1'
-checkout 'silverstripe/silverstripe-framework.git' 'framework' '3.0' 
-checkout 'silverstripe/silverstripe-framework.git' 'framework' '2.4'
-
-echo "Done."

robots.txt

@@ -1,6 +1,5 @@
 User-agent: *
 Disallow: /admin
 Disallow: /assets/src
-Disallow: /en/4.0/
-Disallow: /en/3.3/
+Disallow: /assets/.lucene-index
 Crawl-Delay: 4