From 6604109f316ca93b74641f9af57cc9dbf8bb19b7 Mon Sep 17 00:00:00 2001 From: Damian Mooyman Date: Fri, 24 Jun 2016 17:45:13 +1200 Subject: [PATCH] API Split updater into a service and provide two default implementations: - GitMarkdownUpdater - SVNMarkdonUpdater Add logging for errors to SS_Log Secure escaping of shell arguments and error detection --- app/_config.php | 8 +- app/_config/config.yml | 6 +- app/code/RefreshMarkdownTask.php | 144 +++++++++++++++-------- app/code/updaters/GitMarkdownUpdater.php | 83 +++++++++++++ app/code/updaters/MarkdownUpdater.php | 12 ++ app/code/updaters/SVNMarkdownUpdater.php | 33 ++++++ 6 files changed, 234 insertions(+), 52 deletions(-) create mode 100644 app/code/updaters/GitMarkdownUpdater.php create mode 100644 app/code/updaters/MarkdownUpdater.php create mode 100644 app/code/updaters/SVNMarkdownUpdater.php diff --git a/app/_config.php b/app/_config.php index 120b348..fc9c99d 100644 --- a/app/_config.php +++ b/app/_config.php @@ -15,7 +15,7 @@ if(isset($_ENV['CLEARDB_DATABASE_URL'])) { global $databaseConfig; $parts = parse_url($_ENV['CLEARDB_DATABASE_URL']); - + $databaseConfig['type'] = 'MySQLDatabase'; $databaseConfig['server'] = $parts['host']; $databaseConfig['username'] = $parts['user']; @@ -52,6 +52,12 @@ DocumentationSearch::set_meta_data(array( 'Tags' => 'silverstripe sapphire php framework cms content management system' )); +// SS Platform logging +if(defined('AWS_SYSLOG_LEVEL')) { + $sysLogWriter = new SS_SysLogWriter('silverstripe', LOG_PID | LOG_CONS); + SS_Log::add_writer($sysLogWriter, (int)AWS_SYSLOG_LEVEL, '<='); +} + // Changelogs have heaps of phrases, but are rarely relevant for content searches Config::inst()->update('DocumentationSearch', 'boost_by_path', array( '/^changelog/' => 0.05 diff --git a/app/_config/config.yml b/app/_config/config.yml index 1e9964c..8baaffd 100644 --- a/app/_config/config.yml +++ b/app/_config/config.yml @@ -12,4 +12,8 @@ Controller: - ControllerExtension QuickFeedbackExtension: redirect_field: true - rate_limit: 1 \ No newline at end of file + rate_limit: 1 +Injector: + MarkdownUpdater: + class: GitMarkdownUpdater + diff --git a/app/code/RefreshMarkdownTask.php b/app/code/RefreshMarkdownTask.php index 49266ba..9182ca0 100644 --- a/app/code/RefreshMarkdownTask.php +++ b/app/code/RefreshMarkdownTask.php @@ -21,7 +21,7 @@ class RefreshMarkdownTask extends BuildTask /** * @var string */ - protected $description = "Downloads a fresh version of markdown documentation files from source"; + protected $description = "Downloads a fresh version of markdown documentation files from source. Options are force=1, addonly=1, and branch="; /** * @var bool @@ -35,10 +35,50 @@ class RefreshMarkdownTask extends BuildTask */ public function run($request) { - $this->request = $request; - $repositories = $this->getRepositories(); - foreach ($repositories as $repository) { - $this->cloneRepository($repository); + $this->request = $request; + $repositories = $this->getRepositories(); + $baseDir = $this->getSourceRoot(); + $updater = $this->getUpdater(); + + // Ensure root directory exists + if(!$this->ensureRootDirectory($baseDir)) { + return; + } + + // Update each repo + foreach ($repositories as list($repo, $folder, $branch)) { + $path = "{$baseDir}/{$folder}_{$branch}"; + + // Pass in ?force=1 to force delete existing repos, rather than trying to update them + if($this->request && $this->request->getVar('force') && file_exists($path)) { + $this->printLine("Removing {$path}"); + Filesystem::removeFolder($path); + } + + // Pass in ?addonly=1 to only update new branches + if($this->request && $this->request->getVar('addonly') && file_exists($path)) { + $this->printLine("Skipping update of {$branch}: Already exists at {$path};"); + continue; + } + + // Check if we want to update only a specific branch + if($this->request && ($onlyBranch = $this->request->getVar('branch')) && $onlyBranch != $branch) { + $this->printLine("Skipping update of {$branch}: doesn't match provided filter"); + continue; + } + + // Update this repo + $this->printLine("Beginning update of {$branch}"); + $errors = $updater->update($repo, $path, $branch); + + // Handle result + if(empty($errors)) { + $this->printLine("Successful update of {$branch}"); + } else { + foreach($errors as $error) { + $this->error($error); + } + } } $this->printLine(" "); $this->printLine("To re-index the freshly downloaded documentation files either:"); @@ -46,14 +86,22 @@ class RefreshMarkdownTask extends BuildTask $this->printLine("(2) point your browser at the url 'http://localhost/path/to/ssdocs/dev/tasks/RebuildLuceneDocsIndex?flush=1'"); } + /** + * Gets the service that will pull down remote markdown files + * @return MarkdownUpdater + */ + protected function getUpdater() { + return Injector::inst()->get('MarkdownUpdater'); + } + /** + * Get sources root directory + * * @return string - * - * @todo document this new configuration parameter */ - private function getPath() + private function getSourceRoot() { - return ASSETS_PATH; + return ASSETS_PATH . '/src'; } /** @@ -61,56 +109,52 @@ class RefreshMarkdownTask extends BuildTask */ private function printLine($message) { - $this->eol = Director::is_cli() ? PHP_EOL : "
"; - print $message . $this->eol; - flush(); + $eol = Director::is_cli() ? PHP_EOL : "
"; + echo $message . $eol; + ob_flush(); + flush(); } + /** + * Log an error + * + * @param string $message + */ + protected function error($message) + { + $this->printLine($message); + SS_Log::log($message, SS_Log::ERR); + } + /** * Returns the array of repos to source markdown docs from * * @return array */ - private function getRepositories() + protected function getRepositories() { - if($repos = $this->config()->documentation_repositories) - { - return $repos; - } else { - user_error("You need to set 'RefreshMarkdownTask:documentation_repositories' array in a yaml configuration file", E_USER_WARNING); - return null; - } + $repositories = $this->config()->documentation_repositories; + if(empty($repositories)) { + $this->error( + "You need to set 'RefreshMarkdownTask:documentation_repositories' array in a yaml configuration file" + ); + } + return $repositories; } - /** - * Clone $repository which contains the most current documentation source markdown files - * - * @param array $repository - */ - private function cloneRepository(array $repository) - { - - list($remote, $folder, $branch) = $repository; - - $path = $this->getPath(); - - exec("mkdir -p {$path}/src"); - exec("chmod -R 775 {$path}/src" ); - exec("rm -rf {$path}/src/{$folder}_{$branch}"); - chdir("{$path}/src"); - - // If the dev=1 flag is used when RefreshMarkdownTask is run, a full git clone of the framework repository is kept - // to enable local development of framework and doc.silverstripe.org from within doc.silverstripe.org. Otherwise, - // only the documentation source files are downloaded, only allowing the viewing of documentation files. - - if ($this->request instanceof SS_HTTPRequest && $this->request->getVar('dev')) { - $this->printLine("Cloning repository {$remote}/{$branch} into assets/src/{$folder}_{$branch}"); - exec("git clone --quiet https://github.com/{$remote}.git {$folder}_{$branch} --branch {$branch}"); - } else { - $this->printLine("Downloading the latest documentation files from repository {$remote}/{$branch} to assets/src/{$folder}_{$branch}"); - exec("svn export --quiet https://github.com/{$remote}.git/branches/{$branch}/docs {$folder}_{$branch}/docs"); - } - - } + /** + * Ensure root directory exists + * + * @param string $path + * @return bool True if the directory exists and is writable + */ + protected function ensureRootDirectory($path) { + Filesystem::makeFolder($path); + if(is_dir($path) && is_writable($path)) { + return true; + } + $this->error("Could not create {$path}"); + return false; + } } diff --git a/app/code/updaters/GitMarkdownUpdater.php b/app/code/updaters/GitMarkdownUpdater.php new file mode 100644 index 0000000..3817bbb --- /dev/null +++ b/app/code/updaters/GitMarkdownUpdater.php @@ -0,0 +1,83 @@ +runCommand('git --version', $errors)) { + return $errors; + } + + // Check if we should do a git update or re-clone + if (is_dir($path . '/.git')) { + return $this->doFetch($path, $branch); + } else { + return $this->doClone($repo, $path, $branch); + } + } + + /** + * Update existing git checkouts + * + * @param string $path + * @param string $branch + * @return array List of errors + */ + protected function doFetch($path, $branch) { + $errors = array(); + $checkoutCommand = sprintf( + 'cd %s && git checkout %s', + escapeshellarg($path), + escapeshellarg($branch) + ); + $pullCommand = sprintf( + 'cd %s && git pull origin %s', + escapeshellarg($path), + escapeshellarg($branch) + ); + + // Run + if($this->runCommand($checkoutCommand, $errors)) { + $this->runCommand($pullCommand, $errors); + } + return $errors; + } + + /** + * Clone a new git repo + * @param string $repo Repo slug + * @param string $path Destination path + * @param string $branch Branch name + * @return array + */ + protected function doClone($repo, $path, $branch) { + $errors = array(); + $remote = "https://github.com/{$repo}.git"; + $this->runCommand(sprintf( + "git clone --quiet %s %s --branch %s", + escapeshellarg($remote), + escapeshellarg($path), + escapeshellarg($branch) + ), $errors); + return $errors; + } + + /** + * Run this command + * + * @param string $cmd + * @param array $errors + * @return bool Flag if the command was successful + */ + protected function runCommand($cmd, &$errors = array()) { + exec($cmd, $output, $result); + if($result) { + $errors[] = "Error running command {$cmd}"; + return false; + } + return true; + } +} diff --git a/app/code/updaters/MarkdownUpdater.php b/app/code/updaters/MarkdownUpdater.php new file mode 100644 index 0000000..f5963f1 --- /dev/null +++ b/app/code/updaters/MarkdownUpdater.php @@ -0,0 +1,12 @@ +runCommand(sprintf( + "svn export %s %s", + escapeshellarg($svnPath), + escapeshellarg($svnDest) + ), $errors); + return $errors; + } + + /** + * Run this command + * + * @param string $cmd + * @param array $errors + * @return bool Flag if the command was successful + */ + protected function runCommand($cmd, &$errors = array()) { + exec($cmd, $output, $result); + if($result) { + $errors[] = "Error running command {$cmd}"; + return false; + } + return true; + } +}