Merge pull request #269 from creative-commoners/pulls/4/behat-rerun
[Backport] Rerun failed features in ci
This commit is contained in:
Guy Sartorelli
GitHub
commit
5c69f38876
|
|
@ -8,6 +8,7 @@ parameters:
|
|||
silverstripe_extension.ajax_steps: ~
|
||||
silverstripe_extension.ajax_timeout: ~
|
||||
silverstripe_extension.admin_url: ~
|
||||
silverstripe_extension.is_ci: ~
|
||||
silverstripe_extension.login_url: ~
|
||||
silverstripe_extension.screenshot_path: ~
|
||||
silverstripe_extension.module:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ use Behat\Testwork\ServiceContainer\Extension as ExtensionInterface;
|
|||
use RuntimeException;
|
||||
use Symfony\Component\DependencyInjection\Definition;
|
||||
use Symfony\Component\DependencyInjection\Reference;
|
||||
use Behat\Behat\Tester\ServiceContainer\TesterExtension;
|
||||
use SilverStripe\BehatExtension\Utility\RerunTotalStatistics;
|
||||
use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester;
|
||||
|
||||
/*
|
||||
* This file is part of the SilverStripe\BehatExtension
|
||||
|
|
@ -98,6 +101,22 @@ class Extension implements ExtensionInterface
|
|||
$container->setParameter('silverstripe_extension.region_map', $config['region_map']);
|
||||
}
|
||||
$container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
|
||||
$container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);
|
||||
|
||||
// When running in CI, behat scenarios will occasionally sporadically fail
|
||||
// Replaces services with custom implementations that will rerun failed features
|
||||
// Note that features rather than scenarios need to be rerun to ensure that
|
||||
// everything is setup and torn down correctly and that "Background" bits of
|
||||
// feature fits are rerun
|
||||
if ($config['is_ci']) {
|
||||
$definition = new Definition(RerunRuntimeSuiteTester::class, array(
|
||||
new Reference(TesterExtension::SPECIFICATION_TESTER_ID)
|
||||
));
|
||||
$container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition);
|
||||
|
||||
$definition = new Definition(RerunTotalStatistics::class);
|
||||
$container->setDefinition('output.pretty.statistics', $definition);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -140,6 +159,9 @@ class Extension implements ExtensionInterface
|
|||
info('Number of seconds that @retry tags will retry for')->
|
||||
defaultValue(2)->
|
||||
end()->
|
||||
scalarNode('is_ci')->
|
||||
defaultValue(false)->
|
||||
end()->
|
||||
arrayNode('ajax_steps')->
|
||||
defaultValue(array(
|
||||
'go to',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
<?php
|
||||
|
||||
namespace SilverStripe\BehatExtension\Utility;
|
||||
|
||||
use Behat\Testwork\Environment\Environment;
|
||||
use Behat\Testwork\Specification\SpecificationIterator;
|
||||
use Behat\Testwork\Tester\Result\IntegerTestResult;
|
||||
use Behat\Testwork\Tester\Result\TestResult;
|
||||
use Behat\Testwork\Tester\Result\TestResults;
|
||||
use Behat\Testwork\Tester\Result\TestWithSetupResult;
|
||||
use Behat\Testwork\Tester\Setup\SuccessfulSetup;
|
||||
use Behat\Testwork\Tester\Setup\SuccessfulTeardown;
|
||||
use Behat\Testwork\Tester\SpecificationTester;
|
||||
use Behat\Testwork\Tester\SuiteTester;
|
||||
|
||||
/**
|
||||
* Copy paste of Behat\Testwork\Tester\Runtime\RuntimeSuiteTester which is a final class
|
||||
*
|
||||
* Modified so that it reruns failed features
|
||||
*/
|
||||
class RerunRuntimeSuiteTester implements SuiteTester
|
||||
{
|
||||
/**
|
||||
* @var SpecificationTester
|
||||
*/
|
||||
private $specTester;
|
||||
|
||||
/**
|
||||
* Initializes tester.
|
||||
*
|
||||
* @param SpecificationTester $specTester
|
||||
*/
|
||||
public function __construct(SpecificationTester $specTester)
|
||||
{
|
||||
$this->specTester = $specTester;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function setUp(Environment $env, SpecificationIterator $iterator, $skip)
|
||||
{
|
||||
return new SuccessfulSetup();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function test(Environment $env, SpecificationIterator $iterator, $skip = false)
|
||||
{
|
||||
$results = array();
|
||||
foreach ($iterator as $specification) {
|
||||
$setup = $this->specTester->setUp($env, $specification, $skip);
|
||||
$localSkip = !$setup->isSuccessful() || $skip;
|
||||
$testResult = $this->specTester->test($env, $specification, $localSkip);
|
||||
$teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
|
||||
|
||||
// start modifications here
|
||||
if (!$testResult->isPassed()) {
|
||||
file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL);
|
||||
$setup = $this->specTester->setUp($env, $specification, $skip);
|
||||
$localSkip = !$setup->isSuccessful() || $skip;
|
||||
$testResult = $this->specTester->test($env, $specification, $localSkip);
|
||||
$teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
|
||||
}
|
||||
// end modifications here
|
||||
|
||||
$integerResult = new IntegerTestResult($testResult->getResultCode());
|
||||
$results[] = new TestWithSetupResult($setup, $integerResult, $teardown);
|
||||
}
|
||||
|
||||
return new TestResults($results);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result)
|
||||
{
|
||||
return new SuccessfulTeardown();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,314 @@
|
|||
<?php
|
||||
|
||||
namespace SilverStripe\BehatExtension\Utility;
|
||||
|
||||
use Behat\Behat\Tester\Result\StepResult;
|
||||
use Behat\Testwork\Counter\Memory;
|
||||
use Behat\Testwork\Counter\Timer;
|
||||
use Behat\Testwork\Tester\Result\TestResult;
|
||||
use Behat\Testwork\Tester\Result\TestResults;
|
||||
use Behat\Behat\Output\Statistics\Statistics;
|
||||
use Behat\Behat\Output\Statistics\ScenarioStat;
|
||||
use Behat\Behat\Output\Statistics\StepStat;
|
||||
use Behat\Behat\Output\Statistics\HookStat;
|
||||
|
||||
/**
|
||||
* Copy paste of Behat\Behat\Output\Statistics\TotalStatistics which is a final class
|
||||
*
|
||||
* Modified to remove duplicated stats from reruns
|
||||
*/
|
||||
class RerunTotalStatistics implements Statistics
|
||||
{
|
||||
/**
|
||||
* @var Timer
|
||||
*/
|
||||
private $timer;
|
||||
/**
|
||||
* @var Memory
|
||||
*/
|
||||
private $memory;
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
private $scenarioCounters = array();
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
private $stepCounters = array();
|
||||
/**
|
||||
* @var ScenarioStat[]
|
||||
*/
|
||||
private $failedScenarioStats = array();
|
||||
/**
|
||||
* @var ScenarioStat[]
|
||||
*/
|
||||
private $skippedScenarioStats = array();
|
||||
/**
|
||||
* @var StepStat[]
|
||||
*/
|
||||
private $failedStepStats = array();
|
||||
/**
|
||||
* @var StepStat[]
|
||||
*/
|
||||
private $pendingStepStats = array();
|
||||
/**
|
||||
* @var HookStat[]
|
||||
*/
|
||||
private $failedHookStats = array();
|
||||
// start modifications here
|
||||
/**
|
||||
* @var StepStat[]
|
||||
*/
|
||||
private $passedStepStats = array();
|
||||
// end modifications here
|
||||
|
||||
/**
|
||||
* Initializes statistics.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->resetAllCounters();
|
||||
|
||||
$this->timer = new Timer();
|
||||
$this->memory = new Memory();
|
||||
}
|
||||
|
||||
public function resetAllCounters()
|
||||
{
|
||||
$this->scenarioCounters = $this->stepCounters = array(
|
||||
TestResult::PASSED => 0,
|
||||
TestResult::FAILED => 0,
|
||||
StepResult::UNDEFINED => 0,
|
||||
TestResult::PENDING => 0,
|
||||
TestResult::SKIPPED => 0
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts timer.
|
||||
*/
|
||||
public function startTimer()
|
||||
{
|
||||
$this->timer->start();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops timer.
|
||||
*/
|
||||
public function stopTimer()
|
||||
{
|
||||
$this->timer->stop();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns timer object.
|
||||
*
|
||||
* @return Timer
|
||||
*/
|
||||
public function getTimer()
|
||||
{
|
||||
return $this->timer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns memory usage object.
|
||||
*
|
||||
* @return Memory
|
||||
*/
|
||||
public function getMemory()
|
||||
{
|
||||
return $this->memory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers scenario stat.
|
||||
*
|
||||
* @param ScenarioStat $stat
|
||||
*/
|
||||
public function registerScenarioStat(ScenarioStat $stat)
|
||||
{
|
||||
if (TestResults::NO_TESTS === $stat->getResultCode()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->scenarioCounters[$stat->getResultCode()]++;
|
||||
|
||||
// start modifications here
|
||||
if (TestResult::FAILED === $stat->getResultCode()) {
|
||||
// Ensure that any scenario reruns aren't counted as additional failures
|
||||
$alreadyHasFailure = false;
|
||||
foreach ($this->failedScenarioStats as $failedStat) {
|
||||
if ($failedStat->getPath() === $stat->getPath()) {
|
||||
$alreadyHasFailure = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!$alreadyHasFailure) {
|
||||
$this->failedScenarioStats[] = $stat;
|
||||
} else {
|
||||
$this->scenarioCounters[TestResult::FAILED]--;
|
||||
}
|
||||
}
|
||||
|
||||
if (TestResult::PASSED == $stat->getResultCode()) {
|
||||
// Remove the scenario from the failed scenarios list if it passes on rerun
|
||||
$newFailedScenarioStats = [];
|
||||
foreach ($this->failedScenarioStats as $failedStat) {
|
||||
if ($failedStat->getPath() !== $stat->getPath()) {
|
||||
$newFailedScenarioStats[] = $failedStat;
|
||||
} else {
|
||||
$this->scenarioCounters[TestResult::FAILED]--;
|
||||
}
|
||||
}
|
||||
$this->failedScenarioStats = $newFailedScenarioStats;
|
||||
}
|
||||
// end modifications here
|
||||
|
||||
if (TestResult::SKIPPED === $stat->getResultCode()) {
|
||||
$this->skippedScenarioStats[] = $stat;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers step stat.
|
||||
*
|
||||
* @param StepStat $stat
|
||||
*/
|
||||
public function registerStepStat(StepStat $stat)
|
||||
{
|
||||
$this->stepCounters[$stat->getResultCode()]++;
|
||||
|
||||
// start modifications here
|
||||
if (TestResult::FAILED === $stat->getResultCode()) {
|
||||
// Ensure that any scenario reruns don't double count step failures
|
||||
$alreadyHasFailure = false;
|
||||
foreach ($this->failedStepStats as $failedStat) {
|
||||
if ($failedStat->getPath() === $stat->getPath()) {
|
||||
$alreadyHasFailure = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!$alreadyHasFailure) {
|
||||
$this->failedStepStats[] = $stat;
|
||||
} else {
|
||||
$this->stepCounters[TestResult::FAILED]--;
|
||||
}
|
||||
}
|
||||
|
||||
if (TestResult::PASSED == $stat->getResultCode()) {
|
||||
// Remove any duplicate passes on scenario rerun
|
||||
$alreadyHasSuccess = false;
|
||||
foreach ($this->passedStepStats as $passedStat) {
|
||||
if ($passedStat->getPath() === $stat->getPath()) {
|
||||
$alreadyHasSuccess = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!$alreadyHasSuccess) {
|
||||
$this->passedStepStats[] = $stat;
|
||||
} else {
|
||||
$this->stepCounters[TestResult::PASSED]--;
|
||||
}
|
||||
|
||||
// Remove the step from the failed steps list if it passes on scenario rerun
|
||||
$newFailedStepStats = [];
|
||||
foreach ($this->failedStepStats as $failedStat) {
|
||||
if ($failedStat->getPath() !== $stat->getPath()) {
|
||||
$newFailedStepStats[] = $failedStat;
|
||||
} else {
|
||||
$this->stepCounters[TestResult::FAILED]--;
|
||||
}
|
||||
}
|
||||
$this->failedStepStats = $newFailedStepStats;
|
||||
}
|
||||
// end modifications here
|
||||
|
||||
if (TestResult::PENDING === $stat->getResultCode()) {
|
||||
$this->pendingStepStats[] = $stat;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers hook stat.
|
||||
*
|
||||
* @param HookStat $stat
|
||||
*/
|
||||
public function registerHookStat(HookStat $stat)
|
||||
{
|
||||
if ($stat->isSuccessful()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->failedHookStats[] = $stat;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns counters for different scenario result codes.
|
||||
*
|
||||
* @return array[]
|
||||
*/
|
||||
public function getScenarioStatCounts()
|
||||
{
|
||||
return $this->scenarioCounters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns skipped scenario stats.
|
||||
*
|
||||
* @return ScenarioStat[]
|
||||
*/
|
||||
public function getSkippedScenarios()
|
||||
{
|
||||
return $this->skippedScenarioStats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns failed scenario stats.
|
||||
*
|
||||
* @return ScenarioStat[]
|
||||
*/
|
||||
public function getFailedScenarios()
|
||||
{
|
||||
return $this->failedScenarioStats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns counters for different step result codes.
|
||||
*
|
||||
* @return array[]
|
||||
*/
|
||||
public function getStepStatCounts()
|
||||
{
|
||||
return $this->stepCounters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns failed step stats.
|
||||
*
|
||||
* @return StepStat[]
|
||||
*/
|
||||
public function getFailedSteps()
|
||||
{
|
||||
return $this->failedStepStats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns pending step stats.
|
||||
*
|
||||
* @return StepStat[]
|
||||
*/
|
||||
public function getPendingSteps()
|
||||
{
|
||||
return $this->pendingStepStats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns failed hook stats.
|
||||
*
|
||||
* @return HookStat[]
|
||||
*/
|
||||
public function getFailedHookStats()
|
||||
{
|
||||
return $this->failedHookStats;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue