Merge pull request #139 from tractorcow/pulls/update-task

API Rewrite source file update task

app/_config.php

@@ -15,7 +15,7 @@ if(isset($_ENV['CLEARDB_DATABASE_URL'])) {
 	global $databaseConfig;
 
 	$parts = parse_url($_ENV['CLEARDB_DATABASE_URL']);
-	
+
 	$databaseConfig['type'] = 'MySQLDatabase';
 	$databaseConfig['server'] = $parts['host'];
 	$databaseConfig['username'] = $parts['user'];
@@ -52,6 +52,12 @@ DocumentationSearch::set_meta_data(array(
 	'Tags' => 'silverstripe sapphire php framework cms content management system'
 ));
 
+// SS Platform logging
+if(defined('AWS_SYSLOG_LEVEL')) {
+    $sysLogWriter = new SS_SysLogWriter('silverstripe', LOG_PID | LOG_CONS);
+    SS_Log::add_writer($sysLogWriter, (int)AWS_SYSLOG_LEVEL, '<=');
+}
+
 // Changelogs have heaps of phrases, but are rarely relevant for content searches
 Config::inst()->update('DocumentationSearch', 'boost_by_path', array(
 	'/^changelog/' => 0.05

app/_config/config.yml

@@ -12,4 +12,8 @@ Controller:
     - ControllerExtension
 QuickFeedbackExtension:
   redirect_field: true
-  rate_limit: 1
+  rate_limit: 1
+Injector:
+  MarkdownUpdater:
+    class: GitMarkdownUpdater
+

app/_config/docsviewer.yml

@@ -20,14 +20,13 @@ DocumentationManifest:
       Title: "Framework"
       Version: "3.4"
       Branch: "3.4"
-      Stable: false
+      Stable: true
       DefaultEntity: true
     -
       Path: "assets/src/framework_3.3/docs/"
       Title: "Framework"
       Version: "3.3"
       Branch: "3.3"
-      Stable: true
       DefaultEntity: true
     -
       Path: "assets/src/framework_3.2/docs/"

app/code/RefreshMarkdownTask.php

@@ -21,7 +21,7 @@ class RefreshMarkdownTask extends BuildTask
     /**
      * @var string
      */
-    protected $description = "Downloads a fresh version of markdown documentation files from source";
+    protected $description = "Downloads a fresh version of markdown documentation files from source. Options are force=1, addonly=1, and branch=<branch>";
 
     /**
      * @var bool
@@ -35,10 +35,50 @@ class RefreshMarkdownTask extends BuildTask
      */
     public function run($request)
     {
-        $this->request = $request;
+		$this->request = $request;
-        $repositories = $this->getRepositories();
+		$repositories = $this->getRepositories();
-        foreach ($repositories as $repository) {
+		$baseDir = $this->getSourceRoot();
-            $this->cloneRepository($repository);
+		$updater = $this->getUpdater();
+
+		// Ensure root directory exists
+		if(!$this->ensureRootDirectory($baseDir)) {
+			return;
+		}
+
+		// Update each repo
+        foreach ($repositories as list($repo, $folder, $branch)) {
+			$path =  "{$baseDir}/{$folder}_{$branch}";
+
+			// Pass in ?addonly=1 to only update new branches
+			if($this->request && $this->request->getVar('addonly') && file_exists($path)) {
+				$this->printLine("Skipping update of {$branch}: Already exists at {$path};");
+				continue;
+			}
+
+			// Check if we want to update only a specific branch
+			if($this->request && ($onlyBranch = $this->request->getVar('branch')) && $onlyBranch != $branch) {
+				$this->printLine("Skipping update of {$branch}: doesn't match provided filter");
+				continue;
+			}
+
+			// Pass in ?force=1 to force delete existing repos, rather than trying to update them
+			if($this->request && $this->request->getVar('force') && file_exists($path)) {
+				$this->printLine("Removing {$path}");
+				Filesystem::removeFolder($path);
+			}
+
+			// Update this repo
+			$this->printLine("Beginning update of {$branch}");
+            $errors = $updater->update($repo, $path, $branch);
+
+			// Handle result
+			if(empty($errors)) {
+				$this->printLine("Successful update of {$branch}");
+			} else {
+				foreach($errors as $error) {
+					$this->error($error);
+				}
+			}
         }
         $this->printLine(" ");
         $this->printLine("To re-index the freshly downloaded documentation files either:");
@@ -46,14 +86,22 @@ class RefreshMarkdownTask extends BuildTask
         $this->printLine("(2) point your browser at the url 'http://localhost/path/to/ssdocs/dev/tasks/RebuildLuceneDocsIndex?flush=1'");
     }
 
+	/**
+	 * Gets the service that will pull down remote markdown files
+	 * @return MarkdownUpdater
+	 */
+	protected function getUpdater() {
+		return Injector::inst()->get('MarkdownUpdater');
+	}
+
     /**
+	 * Get sources root directory
+	 *
      * @return string
-     *
-     * @todo document this new configuration parameter
      */
-    private function getPath()
+    private function getSourceRoot()
     {
-        return ASSETS_PATH;
+        return ASSETS_PATH . '/src';
     }
 
     /**
@@ -61,56 +109,56 @@ class RefreshMarkdownTask extends BuildTask
      */
     private function printLine($message)
     {
-        $this->eol = Director::is_cli() ? PHP_EOL : "<br>";
+        $eol = Director::is_cli() ? PHP_EOL : "<br>";
-        print $message . $this->eol;
+        echo $message . $eol;
-        flush();
+		if(ob_get_level() > 0) {
+			ob_flush();
+		}
+		flush();
+
+
     }
 
+	/**
+	 * Log an error
+	 *
+	 * @param string $message
+	 */
+	protected function error($message)
+	{
+		$this->printLine($message);
+		SS_Log::log($message, SS_Log::ERR);
+	}
+
     /**
      * Returns the array of repos to source markdown docs from
      *
      * @return array
      */
-    private function getRepositories()
+    protected function getRepositories()
     {
-        if($repos = $this->config()->documentation_repositories)
+        $repositories = $this->config()->documentation_repositories;
-        {
+		if(empty($repositories)) {
-            return $repos;
+			$this->error(
-        } else {
+				"You need to set 'RefreshMarkdownTask:documentation_repositories' array in a yaml configuration file"
-            user_error("You need to set 'RefreshMarkdownTask:documentation_repositories' array in a yaml configuration file", E_USER_WARNING);
+			);
-            return null;
+		}
-        }
+		return $repositories;
     }
 
-    /**
+	/**
-     * Clone $repository which contains the most current documentation source markdown files
+	 * Ensure root directory exists
-     *
+	 *
-     * @param array $repository
+	 * @param string $path
-     */
+	 * @return bool True if the directory exists and is writable
-    private function cloneRepository(array $repository)
+	 */
-    {
+	protected function ensureRootDirectory($path) {
-
+		Filesystem::makeFolder($path);
-        list($remote, $folder, $branch) = $repository;
+		if(is_dir($path) && is_writable($path)) {
-
+			return true;
-        $path = $this->getPath();
+		}
-
+		$this->error("Could not create {$path}");
-        exec("mkdir -p {$path}/src");
+		return false;
-        exec("chmod -R 775 {$path}/src" );
+	}
-        exec("rm -rf {$path}/src/{$folder}_{$branch}");
-        chdir("{$path}/src");
-
-        // If the dev=1 flag is used when RefreshMarkdownTask is run, a full git clone of the framework repository is kept
-        // to enable local development of framework and doc.silverstripe.org from within doc.silverstripe.org. Otherwise,
-        // only the documentation source files are downloaded, only allowing the viewing of documentation files. 
-
-        if ($this->request instanceof SS_HTTPRequest && $this->request->getVar('dev')) {
-            $this->printLine("Cloning repository {$remote}/{$branch} into assets/src/{$folder}_{$branch}");
-            exec("git clone --quiet https://github.com/{$remote}.git {$folder}_{$branch} --branch {$branch}");
-        } else {
-            $this->printLine("Downloading the latest documentation files from repository {$remote}/{$branch} to assets/src/{$folder}_{$branch}");
-            exec("svn export --quiet https://github.com/{$remote}.git/branches/{$branch}/docs {$folder}_{$branch}/docs");
-        }
-
-    }
 
 }

app/code/updaters/GitMarkdownUpdater.php

@@ -0,0 +1,86 @@
+<?php
+
+class GitMarkdownUpdater implements MarkdownUpdater
+{
+
+	public function update($repo, $path, $branch)
+	{
+		// Ensure git is available
+		$errors = array();
+		if (!$this->runCommand('git --version', $errors)) {
+			return $errors;
+		}
+
+		// Check if we should do a git update or re-clone
+		if (is_dir($path . '/.git')) {
+			return $this->doFetch($path, $branch);
+		} else {
+			return $this->doClone($repo, $path, $branch);
+		}
+	}
+
+	/**
+	 * Update existing git checkouts
+	 *
+	 * @param string $path
+	 * @param string $branch
+	 * @return array List of errors
+	 */
+	protected function doFetch($path, $branch) {
+		$errors = array();
+		$fetchCommand = sprintf(
+			'cd %s && git fetch origin %s',
+			escapeshellarg($path),
+			escapeshellarg($branch)
+		);
+		$resetCommand = sprintf(
+			'cd %s && git reset --hard origin/%s',
+			escapeshellarg($path),
+			escapeshellarg($branch)
+		);
+
+		// Run
+		if($this->runCommand($fetchCommand, $errors)) {
+			$this->runCommand($resetCommand, $errors);
+		}
+		return $errors;
+    }
+
+	/**
+	 * Clone a new git repo
+	 * @param string $repo Repo slug
+	 * @param string $path Destination path
+	 * @param string $branch Branch name
+	 * @return array
+	 */
+	protected function doClone($repo, $path, $branch) {
+		$errors = array();
+		$remote = "https://github.com/{$repo}.git";
+        $this->runCommand(sprintf(
+			"git clone --quiet %s %s --branch %s",
+			escapeshellarg($remote),
+			escapeshellarg($path),
+			escapeshellarg($branch)
+		), $errors);
+		return $errors;
+	}
+
+	/**
+	 * Run this command
+	 *
+	 * @param string $cmd
+	 * @param array $errors
+	 * @return bool Flag if the command was successful
+	 */
+	protected function runCommand($cmd, &$errors = array()) {
+		exec("{$cmd} 2>&1 >/dev/null", $output, $result);
+		if($result) {
+			$errors[] = "Error running command {$cmd}:";
+			foreach($output as $error) {
+				$errors[] = ' * ' . $error;
+			}
+			return false;
+		}
+		return true;
+	}
+}

app/code/updaters/MarkdownUpdater.php

@@ -0,0 +1,12 @@
+<?php
+
+interface MarkdownUpdater {
+
+	/**
+	 * @param string $repo
+	 * @param string $path
+	 * @param string $branch
+	 * @return array List of any errors that occurred
+	 */
+	public function update($repo, $path, $branch);
+}

app/code/updaters/SVNMarkdownUpdater.php

@@ -0,0 +1,33 @@
+<?php
+
+class SVNMarkdownUpdater implements MarkdownUpdater
+{
+	public function update($repo, $path, $branch)
+	{
+		$errors = array();
+		$svnPath = "https://github.com/{$repo}.git/branches/{$branch}/docs";
+		$svnDest = "{$path}/docs";
+		$this->runCommand(sprintf(
+			"svn export %s %s",
+			escapeshellarg($svnPath),
+			escapeshellarg($svnDest)
+		), $errors);
+		return $errors;
+    }
+
+	/**
+	 * Run this command
+	 *
+	 * @param string $cmd
+	 * @param array $errors
+	 * @return bool Flag if the command was successful
+	 */
+	protected function runCommand($cmd, &$errors = array()) {
+		exec($cmd, $output, $result);
+		if($result) {
+			$errors[] = "Error running command {$cmd}";
+			return false;
+		}
+		return true;
+	}
+}

composer.json

@@ -5,7 +5,7 @@
 		"ext-gd": "*",
 		"ext-mbstring": "*",
 		"silverstripe/docsviewer": "^2.0",
-		"silverstripe/framework": "3.2.1",
+		"silverstripe/framework": "^3.2",
 		"silverstripe/toolbar": "^4.0",
 		"silverstripe/dynamodb": "^1.1",
 		"mandrew/silverstripe-quickfeedback": "^0.2"

composer.lock

@@ -4,8 +4,8 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "hash": "7aca7c53bf2e2a013790143ca4493ffb",
+    "hash": "cc3e0d21493519332387f87c4e668fb5",
-    "content-hash": "99728b10c87af2ab40e8f0da741ef8c0",
+    "content-hash": "93abce3419beb70f39d06267aa1d5e84",
     "packages": [
         {
             "name": "aws/aws-sdk-php",
@@ -416,6 +416,7 @@
                 "rest",
                 "web service"
             ],
+            "abandoned": "guzzlehttp/guzzle",
             "time": "2015-03-18 18:23:50"
         },
         {
@@ -633,16 +634,16 @@
         },
         {
             "name": "silverstripe/framework",
-            "version": "3.2.1",
+            "version": "3.4.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/silverstripe/silverstripe-framework.git",
-                "reference": "0178fa5a1873a0f1d6100268c86dab81a18c23a5"
+                "reference": "8008fcbe9769f1d2cb53c542f302ad6d32d185d6"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/silverstripe/silverstripe-framework/zipball/0178fa5a1873a0f1d6100268c86dab81a18c23a5",
+                "url": "https://api.github.com/repos/silverstripe/silverstripe-framework/zipball/8008fcbe9769f1d2cb53c542f302ad6d32d185d6",
-                "reference": "0178fa5a1873a0f1d6100268c86dab81a18c23a5",
+                "reference": "8008fcbe9769f1d2cb53c542f302ad6d32d185d6",
                 "shasum": ""
             },
             "require": {
@@ -653,6 +654,11 @@
                 "phpunit/phpunit": "~3.7"
             },
             "type": "silverstripe-module",
+            "extra": {
+                "branch-alias": {
+                    "3.x-dev": "3.4.x-dev"
+                }
+            },
             "autoload": {
                 "classmap": [
                     "tests/behat/features/bootstrap"
@@ -678,7 +684,7 @@
                 "framework",
                 "silverstripe"
             ],
-            "time": "2015-11-16 03:17:10"
+            "time": "2016-06-02 22:59:29"
         },
         {
             "name": "silverstripe/toolbar",
@@ -726,16 +732,16 @@
         },
         {
             "name": "symfony/event-dispatcher",
-            "version": "v2.8.7",
+            "version": "v2.8.8",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/event-dispatcher.git",
-                "reference": "2a6b8713f8bdb582058cfda463527f195b066110"
+                "reference": "b180b70439dca70049b6b9b7e21d75e6e5d7aca9"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/2a6b8713f8bdb582058cfda463527f195b066110",
+                "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/b180b70439dca70049b6b9b7e21d75e6e5d7aca9",
-                "reference": "2a6b8713f8bdb582058cfda463527f195b066110",
+                "reference": "b180b70439dca70049b6b9b7e21d75e6e5d7aca9",
                 "shasum": ""
             },
             "require": {
@@ -782,7 +788,7 @@
             ],
             "description": "Symfony EventDispatcher Component",
             "homepage": "https://symfony.com",
-            "time": "2016-06-06 11:11:27"
+            "time": "2016-06-29 05:29:29"
         },
         {
             "name": "unclecheese/display-logic",
@@ -1197,16 +1203,16 @@
         },
         {
             "name": "symfony/yaml",
-            "version": "v2.8.7",
+            "version": "v2.8.8",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/yaml.git",
-                "reference": "815fabf3f48c7d1df345a69d1ad1a88f59757b34"
+                "reference": "dba4bb5846798cd12f32e2d8f3f35d77045773c8"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/yaml/zipball/815fabf3f48c7d1df345a69d1ad1a88f59757b34",
+                "url": "https://api.github.com/repos/symfony/yaml/zipball/dba4bb5846798cd12f32e2d8f3f35d77045773c8",
-                "reference": "815fabf3f48c7d1df345a69d1ad1a88f59757b34",
+                "reference": "dba4bb5846798cd12f32e2d8f3f35d77045773c8",
                 "shasum": ""
             },
             "require": {
@@ -1242,7 +1248,7 @@
             ],
             "description": "Symfony Yaml Component",
             "homepage": "https://symfony.com",
-            "time": "2016-06-06 11:11:27"
+            "time": "2016-06-29 05:29:29"
         }
     ],
     "aliases": [],