ENHANCEMENT getlocalization build support

Squashed commits (sorry, too hard to untangle):
- Moved descriptions from custom "phing help" target
to the more standard 'phing -l' command, which keeps
the descriptions in one place, rather than duplicating
them between xml comments, "description" attrs and the "help" target.
- Prefixed helper targets to make it clear that they're internal
(= require temporary properties to work)
This commit is contained in:
Ingo Schommer 2012-06-22 12:13:43 +02:00
parent d5b72a8754
commit 00f66e201f
3 changed files with 347 additions and 41 deletions

6
build.properties.default Normal file
View File

@ -0,0 +1,6 @@
getlocalization.framework.project = sapphire
getlocalization.framework.user = silverstripe
getlocalization.framework.password =
getlocalization.cms.project = silverstripe_cms
getlocalization.cms.user = silverstripe
getlocalization.cms.password =

155
build.xml
View File

@ -16,6 +16,7 @@ phing help
<taskdef name="ssmodules" classname="tools.LoadModulesTask" classpath="${project.basedir}" />
<taskdef name="sschanglog" classname="tools.CreateChangelog" classpath="${project.basedir}" />
<taskdef name="gitstash" classname="tools.GitStashTask" classpath="${project.basedir}" />
<taskdef name="updateTranslationsTask" classname="tools.UpdateTranslationsTask" classpath="${project.basedir}" />
<property name="basedir" value="." override="true" />
<property name="dependent-modules-file" value="dependent-modules" override="true" />
@ -28,22 +29,19 @@ phing help
<available file="${dependent-modules-file}" property="dependent-modules-file-exists" />
<available file="${changelog-definition-file}" property="changelog-definition-file-exists" />
<!--
=================================================================
Helper Targets
=================================================================
-->
<target name="help">
<echo>
SilverStripe Project Build
------------------------------------
This build file contains targets to assist in creating new SilverStripe builds and releases.
Important targets:
* archive - Creates a tar.gz or zip file from the current source, removing version control specific files
* checkout - Switches all working copies to the specified tag or branch
* tag - Creates a new git tag in all the nested working copies (optionally pushes the created tag)
* pushtags - Pushes all local tags to their respective origin repositories
* update_modules - Checks out repositories defined in the 'dependent-modules' file into the current directory
* add_module - Checks out a module at a specific repository URL
* changelog - Create a changelog.md file from the repositories specified in the 'changelog-definitions' file
Run "phing -l" to get a full list of available targets.
Options:
@ -65,17 +63,16 @@ Options:
</echo>
</target>
<target name="gitRepositories">
<target name="_gitRepositories">
<findRepos TargetDir="${basedir}" />
</target>
<!-- find the git binary and set it -->
<target name="gitBinary">
<target name="_gitBinary" description="find the git binary and set it">
<exec command="which git" outputProperty="gitPath1" />
</target>
<!-- Tag a git repo with a specific tag (in $tagname) -->
<target name="tagTask" if="tagname,reponame,gitPath1">
<target name="_tagTask" if="tagname,reponame,gitPath1"
description="Tag a git repo with a specific tag.">
<gittag
repository="${reponame}"
name="${tagname}"
@ -84,21 +81,21 @@ Options:
<echo msg="git tag '${tagname}' added to '${reponame}' git repository" />
</target>
<!-- Push all local tags -->
<target name="pushTask" if="reponame,gitPath1">
<gitpush
<target name="_pushTask" if="reponame,gitPath1"
description="Push all local tags">
<ssgitpush
repository="${reponame}"
tags="true"
gitPath="${gitPath1}" />
<echo msg="pushed all tags to '${reponame}' git repository" />
</target>
<!-- Checkout the specified tag on all working copies -->
<target name="checkoutTask" if="reponame,gitPath1,tagname">
<target name="_checkoutTask" if="reponame,gitPath1,tagname"
description="Checkout the specified tag on all working copies">
<echo msg="checking out ${reponame}"/>
<gitstash repository="${reponame}" gitPath="${gitPath1}" />
<gitcheckout
<ssgitcheckout
repository="${reponame}"
branchname="${tagname}"
gitPath="${gitPath1}" />
@ -108,17 +105,23 @@ Options:
<echo msg="checked out ${tagname} tag/branch in '${reponame}' git repository" />
</target>
<target name="createDependentModulesFile" unless="dependent-modules-file-exists">
<target name="_createDependentModulesFile" unless="dependent-modules-file-exists">
<copy file="${dependent-modules-file}.default" tofile="${dependent-modules-file}" />
</target>
<target name="createChangelogDefinitionsFile" unless="changelog-definitions-file-exists">
<target name="_createChangelogDefinitionsFile" unless="changelog-definitions-file-exists">
<copy file="${changelog-definitions-file}.default" tofile="${changelog-definitions-file}" />
</target>
<!--
=================================================================
Main Targets
=================================================================
-->
<!-- tags all git repositories in the current directory with a tag name -->
<target name="tag" if="basedir" depends="gitRepositories,gitBinary">
<target name="tag" if="basedir"
description="Creates a new git tag in all the nested working copies (optionally pushes the created tag)"
depends="_gitRepositories,_gitBinary">
<if>
<isset property="tagname"/>
<then>
@ -131,7 +134,7 @@ Options:
</if>
<!-- find all git repos and run the tagTask on them -->
<foreach list="${GitReposList}" param="reponame" target="tagTask" />
<foreach list="${GitReposList}" param="reponame" target="_tagTask" />
<input propertyName="pushToOrigin" defaultValue="no" validArgs="yes,no" promptChar=":">Push local tags to origin?</input>
<if>
@ -142,13 +145,15 @@ Options:
</if>
</target>
<!-- Pushes all local tags to origin -->
<target name="pushtags" if="basedir" depends="gitRepositories,gitBinary">
<foreach list="${GitReposList}" param="reponame" target="pushTask" />
<target name="pushtags" if="basedir"
description="Pushes all local tags to their respective origin repositories"
depends="_gitRepositories,_gitBinary">
<foreach list="${GitReposList}" param="reponame" target="_pushTask" />
</target>
<!-- Switches all working copies to the specified tag or branch -->
<target name="checkout" if="basedir" depends="gitRepositories,gitBinary">
<target name="checkout" if="basedir"
description="Switches all working copies to the specified tag or branch"
depends="_gitRepositories,_gitBinary">
<if>
<isset property="tagname"/>
<then>
@ -179,7 +184,7 @@ Options:
</if>
<!-- find all git repos and run the checkoutTask on them -->
<foreach list="${GitReposList}" param="reponame" target="checkoutTask" />
<foreach list="${GitReposList}" param="reponame" target="_checkoutTask" />
</target>
<target name="archive" if="basedir" description="Creates a gzip archive from the current folder (removes any version control files)">
@ -325,7 +330,8 @@ Options:
<echo msg="##teamcity[publishArtifacts '${archivename}-framework-v${version}.zip']" />
</target>
<target name="upload-release">
<target name="upload-release"
description="Uploads archives previously created through 'ping archive' to a public webhost, and notifies a group of people of the new release. Requires working public key auth on the release destination.">
<if>
<not><isset property="version"/></not>
<then><input propertyName="version" defaultValue="x.y.z" promptChar=":">Please choose a version</input></then>
@ -350,7 +356,8 @@ Your friendly automated release script.
</target>
<target name="upload-nightly">
<target name="upload-nightly"
description="Uploads archives previously created through 'phing archive' to a public webhost">
<property name="nightly_dest" value="qa-servers@homer:/sites/ssorg-v2/www/assets/nightlies/" />
<exec command="scp -P 2222 SilverStripe-*.tar.gz ${nightly_dest}" />
<exec command="scp -P 2222 SilverStripe-*.zip ${nightly_dest}" />
@ -358,14 +365,13 @@ Your friendly automated release script.
<!-- Load modules where sensitive dependency exists -->
<target name="update_modules" depends="createDependentModulesFile">
<target name="update_modules"
description="Checks out repositories defined in the 'dependent-modules' file into the current directory"
depends="_createDependentModulesFile">
<ssmodules file="${basedir}/${dependent-modules-file}" noninteractive="${ni_build}"/>
</target>
<!-- Add a new module to the system. Run from the commandline, you can pass
in the details of the module as phing add_module -Dmodule=blog -Dmodurl=http://path/to/svn -->
<target name="add_module">
<target name="add_module" description="Checks out a module at a specific repository URL. Usage: phing add_module -Dmodule=blog -Dmodurl=http://path/to/svn.">
<if>
<isset property="modurl"/>
<then>
@ -391,10 +397,77 @@ Your friendly automated release script.
<ssmodules name="${module}" url="${modurl}" />
</target>
<!-- Create a changelog of git changes based on the -->
<target name="changelog" depends="createChangelogDefinitionsFile" if="basedir,changelogSort">
<target name="changelog"
description="Create a changelog.md file from the repositories specified in the 'changelog-definitions' file"
depends="_createChangelogDefinitionsFile"
if="basedir,changelogSort">
<sschanglog definitions="${changelog-definitions-file}" baseDir="${basedir}" sort="${changelogSort}"/>
<echo msg="${changelogOutput}" />
</target>
<target name="translations-staging-setup">
<exec command="git checkout master" dir="${module}" checkreturn="true" />
<exec command="git fetch origin" dir="${module}" checkreturn="true" />
<exec command="git branch -D translation-staging" dir="${module}" checkreturn="true" />
<exec command="git branch --track translation-staging origin/translation-staging" dir="${module}" checkreturn="true" />
<exec command="git checkout translation-staging" dir="${module}" checkreturn="true" />
</target>
<target name="translations-staging-teardown">
<exec command="git checkout master" dir="${module}" checkreturn="true" />
</target>
<target name="translations-update-git-masterfile"
description="Collect translation on a module, commit them into a specialized branch and push to the origin repository."
depends="translations-staging-setup">
<exec command="git merge origin/master" dir="${module}" checkreturn="true" />
<exec command="php framework/cli-script.php dev/tasks/i18nTextCollectorTask" passthru="true" />
<exec command="git status --short" dir="${module}" outputProperty="git.status.${module}" />
<if>
<istrue value="${git.status.${module}}" />
<then>
<exec command="git add lang/*" dir="${module}" passthru="true" checkreturn="true" />
<exec command="git commit -m 'MINOR Updated translations master'" dir="${module}" passthru="true" checkreturn="true" />
<exec command="git merge --force origin/master" dir="${module}" checkreturn="true" />
</then>
</if>
</target>
<target name="translations-update-gl-masterfile"
description="Pushes translation master files to getlocalization.com"
depends="translations-staging-setup">
<exec command="curl --fail --form file=@${module}/lang/en.yml --form name='lang/en.yml' --user ${getlocalization.${module}.user}:${getlocalization.${module}.password} https://api.getlocalization.com/${getlocalization.${module}.project}/api/update-master/" passthru="true" checkreturn="true" />
<exec command="git checkout master" dir="${module}" checkreturn="true" />
</target>
<target name="translations-update-gl-contribs" description="Update translations in working copy from getlocalization.com, and commit changes to a specialized branch and push to origin repository. Note: The API requests can take a couple of minutes."
depends="translations-staging-setup">
<exec command="git stash" dir="${module}" checkreturn="true" />
<updateTranslationsTask
glProductName="${getlocalization.${module}.project}"
glUser="${getlocalization.${module}.user}"
glPassword="${getlocalization.${module}.password}"
modulePath="${module}"
/>
<exec command="git status --short" dir="${module}" outputProperty="git.status.${module}" />
<if>
<istrue value="${git.status.${module}}" />
<then>
<exec command="git add lang/*" dir="${module}" passthru="true" checkreturn="true" />
<exec command="git add javascript/lang/*" dir="${module}" passthru="true" checkreturn="true" />
<exec command="git commit -m 'MINOR Updated translations'" dir="${module}" passthru="true" checkreturn="true" />
<exec command="git push origin/translation-staging" dir="${module}" checkreturn="true" />
</then>
</if>
<exec command="git checkout master" dir="${module}" checkreturn="true" />
<exec command="git stash pop" dir="${module}" checkreturn="true" />
</target>
<target name="translations-sync"
description="Wrapper task to handle updating translations and master files, using the getlocalization.com API, committing to a specialized branch in the working copy and pushing to the origin repository.">
<foreach list="cms" param="module" target="translations-update-git-masterfile" />
<foreach list="cms" param="module" target="translations-update-gl-masterfile" />
<foreach list="cms" param="module" target="translations-update-gl-contribs" />
</target>
</project>

View File

@ -0,0 +1,227 @@
<?php
/**
* Fetch translation files as a ZIP from getlocalization.com,
* extract the YAML files from it, and do some reformatting on them
* to make them suitable for usage in the SilverStripe translation system.
*
* Requires 'unzip' binary.
*/
class UpdateTranslationsTask extends SilverStripeBuildTask {
static $url_translations = 'https://api.getlocalization.com/%s/api/translations/zip/';
/**
* Absolute path to module base (not lang folder)
*/
protected $modulePath;
/**
* GetLocalization product name, as documented in their API:
* http://www.getlocalization.com/library/api/get-localization-file-management-api/
* @var [type]
*/
protected $glProductName;
protected $glUser;
protected $glPassword;
protected $langFolders = array(
'yml' => 'lang',
'js' => 'javascript/lang'
);
/**
* @var String If set, will use existing files rather than try to download them.
*/
protected $downloadPath;
public function getGlUser() {
return $this->glUser;
}
public function setGlUser($newGlUser) {
$this->glUser = $newGlUser;
return $this;
}
public function getGlPassword() {
return $this->glPassword;
}
public function setGlPassword($newGlPassword) {
$this->glPassword = $newGlPassword;
return $this;
}
public function setModulePath($path) {
$this->modulePath = $path;
}
public function setDownloadPath($path) {
$this->downloadPath = $path;
}
public function setGlProductName($name) {
$this->glProductName = $name;
}
public function main() {
if (!is_dir($this->modulePath)) {
throw new BuildException("Invalid target directory: $this->modulePath");
}
$downloadPath = $this->downloadPath ? $this->downloadPath : $this->download();
$files = $this->findFiles($downloadPath);
foreach($files as $file) {
$ext = pathinfo($file, PATHINFO_EXTENSION);
if($ext == 'yml') {
$this->processYmlFile($file);
} elseif($ext == 'js') {
$this->processJavascriptFile($file);
} else {
throw new LogicException(sprintf('Unknown extension: %s', $ext));
}
}
}
/**
* @return File path to a folder structure containing translation files
*/
protected function download() {
$tmpFolder = tempnam(sys_get_temp_dir(), $this->glProductName . '-');
$tmpFilePath = $tmpFolder . '.zip';
rename($tmpFolder, $tmpFilePath);
$url = sprintf(self::$url_translations, $this->glProductName);
$this->log(sprintf("Downloading $url to $tmpFilePath"));
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERPWD, $this->glUser. ":" . $this->glPassword);
$data = curl_exec($ch);
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if($code >= 400) {
throw new BuildException(sprintf(
'Error downloading %s: %s %s',
$url,
$code,
$data
));
}
if(curl_error($ch)) {
throw new BuildException(sprintf(
'Error downloading %s: %s (#%s)',
$url,
curl_error($ch),
curl_errno($ch)
));
}
curl_close($ch);
file_put_contents($tmpFilePath, $data);
$this->log(sprintf("Extracting to $tmpFolder"));
$this->exec("unzip $tmpFilePath -d $tmpFolder");
return $tmpFolder;
}
/**
* @param String Absolute path to a folder structure containing translation files
* @return Array with file paths
*/
protected function findFiles($path) {
// Recursively find files with certain extensions.
// Can't use glob() since its non-recursive.
// Directory structure doesn't matter here.
$files = array();
$matches = new RegexIterator(
new RecursiveIteratorIterator(
new RecursiveDirectoryIterator($path)
),
'/^.+\.(yml$|js)/i',
RecursiveRegexIterator::GET_MATCH
);
foreach($matches as $match) $files[] = $match[0];
return $files;
}
protected function processYmlFile($file) {
$this->log(sprintf("Processing $file"));
// Rename locale to correct convention (underscored rather than dashed).
// GL wants filenames to adhere to its saved locales, but SS framework
// can't easily change to that format for backwards compat reasons, so we need to convert.
// The passed in file name doesn't really matter here, only the contained locale.
// By convention, the first line in the YAML file is always the locale used, as a YAML "root key".
$content = file_get_contents($file);
preg_match('/^([\w-_]*):/', $content, $matches);
$locale = $matches[1];
$locale = str_replace('-', '_', $locale);
$locale = str_replace(':', '', $locale);
// Convert faulty multiline double quoted string YAML
// to block format, in order to allow the YAML Parser to open it later
// TODO Remove once getlocalization.com has fixed their output format (see support.getlocalization.com #2022)
$isBlock = false;
$blockIndex = -1;
$lines = explode(PHP_EOL, $content);
$keyedLineRegex = '/^\s*[\w\d-_]*:\s*/';
$leadingQuoteRegex = '/^\s*\"/';
$trailingQuoteRegex = '/[^\\\\]\"$/';
foreach($lines as $i => $line) {
preg_match($keyedLineRegex, $line, $matches);
$key = $matches ? $matches[0] : null;
$val = trim(preg_replace($keyedLineRegex, '', $line));
// If its a multiline double quoted string (no unescaped closing quote)
if($val && $line != '"' && preg_match($leadingQuoteRegex, $val) && !preg_match($trailingQuoteRegex, $val)) {
$isBlock = true;
$blockIndex = $i;
} elseif($key) {
$isBlock = false;
$blockIndex = -1;
} else {
$lines[$blockIndex] .= $line;
unset($lines[$i]);
}
}
$content = implode(PHP_EOL, $lines);
// Parse YML as a sanity check,
// and reorder alphabetically by key to ensure consistent diffs.
require_once '../framework/thirdparty/zend_translate_railsyaml/library/Translate/Adapter/thirdparty/sfYaml/lib/sfYaml.php';
require_once '../framework/thirdparty/zend_translate_railsyaml/library/Translate/Adapter/thirdparty/sfYaml/lib/sfYamlParser.php';
require_once '../framework/thirdparty/zend_translate_railsyaml/library/Translate/Adapter/thirdparty/sfYaml/lib/sfYamlDumper.php';
$yamlHandler = new sfYaml();
$yml = $yamlHandler->parse($content);
if(isset($yml[$locale]) && is_array($yml[$locale])) {
ksort($yml[$locale]);
foreach($yml[$locale] as $k => &$v) {
if(is_array($v)) ksort($v);
}
}
$content = $yamlHandler->dump($yml, 99); // don't inline first levels
// Save into correct path, overwriting existing files
$path = $this->modulePath . '/' . $this->langFolders['yml'] . '/' . $locale . '.yml';
$this->log("Saving to $path");
file_put_contents($path, $content);
}
protected function processJavascriptFile($file) {
$this->log(sprintf("Processing $file"));
$locale = pathinfo($file, PATHINFO_FILENAME);
$locale = str_replace('-', '_', $locale);
// Save into correct path, overwriting existing files
$path = $this->modulePath . '/' . $this->langFolders['js'] . '/' . $locale . '.yml';
$this->log("Saving to $path");
file_put_contents($path, file_get_contents($file));
}
}
?>