[Backport] Rerun failed features in ci

This commit is contained in:
Steve Boyd 2024-04-15 14:30:19 +12:00
parent 6c9692a4b6
commit 1452c35e08
4 changed files with 419 additions and 0 deletions

View File

@ -8,6 +8,7 @@ parameters:
silverstripe_extension.ajax_steps: ~ silverstripe_extension.ajax_steps: ~
silverstripe_extension.ajax_timeout: ~ silverstripe_extension.ajax_timeout: ~
silverstripe_extension.admin_url: ~ silverstripe_extension.admin_url: ~
silverstripe_extension.is_ci: ~
silverstripe_extension.login_url: ~ silverstripe_extension.login_url: ~
silverstripe_extension.screenshot_path: ~ silverstripe_extension.screenshot_path: ~
silverstripe_extension.module: silverstripe_extension.module:

View File

@ -18,6 +18,9 @@ use Behat\Testwork\ServiceContainer\Extension as ExtensionInterface;
use RuntimeException; use RuntimeException;
use Symfony\Component\DependencyInjection\Definition; use Symfony\Component\DependencyInjection\Definition;
use Symfony\Component\DependencyInjection\Reference; use Symfony\Component\DependencyInjection\Reference;
use Behat\Behat\Tester\ServiceContainer\TesterExtension;
use SilverStripe\BehatExtension\Utility\RerunTotalStatistics;
use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester;
/* /*
* This file is part of the SilverStripe\BehatExtension * This file is part of the SilverStripe\BehatExtension
@ -98,6 +101,22 @@ class Extension implements ExtensionInterface
$container->setParameter('silverstripe_extension.region_map', $config['region_map']); $container->setParameter('silverstripe_extension.region_map', $config['region_map']);
} }
$container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']); $container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
$container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);
// When running in CI, behat scenarios will occasionally sporadically fail
// Replaces services with custom implementations that will rerun failed features
// Note that features rather than scenarios need to be rerun to ensure that
// everything is setup and torn down correctly and that "Background" bits of
// feature fits are rerun
if ($config['is_ci']) {
$definition = new Definition(RerunRuntimeSuiteTester::class, array(
new Reference(TesterExtension::SPECIFICATION_TESTER_ID)
));
$container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition);
$definition = new Definition(RerunTotalStatistics::class);
$container->setDefinition('output.pretty.statistics', $definition);
}
} }
/** /**
@ -140,6 +159,9 @@ class Extension implements ExtensionInterface
info('Number of seconds that @retry tags will retry for')-> info('Number of seconds that @retry tags will retry for')->
defaultValue(2)-> defaultValue(2)->
end()-> end()->
scalarNode('is_ci')->
defaultValue(false)->
end()->
arrayNode('ajax_steps')-> arrayNode('ajax_steps')->
defaultValue(array( defaultValue(array(
'go to', 'go to',

View File

@ -0,0 +1,82 @@
<?php
namespace SilverStripe\BehatExtension\Utility;
use Behat\Testwork\Environment\Environment;
use Behat\Testwork\Specification\SpecificationIterator;
use Behat\Testwork\Tester\Result\IntegerTestResult;
use Behat\Testwork\Tester\Result\TestResult;
use Behat\Testwork\Tester\Result\TestResults;
use Behat\Testwork\Tester\Result\TestWithSetupResult;
use Behat\Testwork\Tester\Setup\SuccessfulSetup;
use Behat\Testwork\Tester\Setup\SuccessfulTeardown;
use Behat\Testwork\Tester\SpecificationTester;
use Behat\Testwork\Tester\SuiteTester;
/**
* Copy paste of Behat\Testwork\Tester\Runtime\RuntimeSuiteTester which is a final class
*
* Modified so that it reruns failed features
*/
class RerunRuntimeSuiteTester implements SuiteTester
{
/**
* @var SpecificationTester
*/
private $specTester;
/**
* Initializes tester.
*
* @param SpecificationTester $specTester
*/
public function __construct(SpecificationTester $specTester)
{
$this->specTester = $specTester;
}
/**
* {@inheritdoc}
*/
public function setUp(Environment $env, SpecificationIterator $iterator, $skip)
{
return new SuccessfulSetup();
}
/**
* {@inheritdoc}
*/
public function test(Environment $env, SpecificationIterator $iterator, $skip = false)
{
$results = array();
foreach ($iterator as $specification) {
$setup = $this->specTester->setUp($env, $specification, $skip);
$localSkip = !$setup->isSuccessful() || $skip;
$testResult = $this->specTester->test($env, $specification, $localSkip);
$teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
// start modifications here
if (!$testResult->isPassed()) {
file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL);
$setup = $this->specTester->setUp($env, $specification, $skip);
$localSkip = !$setup->isSuccessful() || $skip;
$testResult = $this->specTester->test($env, $specification, $localSkip);
$teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
}
// end modifications here
$integerResult = new IntegerTestResult($testResult->getResultCode());
$results[] = new TestWithSetupResult($setup, $integerResult, $teardown);
}
return new TestResults($results);
}
/**
* {@inheritdoc}
*/
public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result)
{
return new SuccessfulTeardown();
}
}

View File

@ -0,0 +1,314 @@
<?php
namespace SilverStripe\BehatExtension\Utility;
use Behat\Behat\Tester\Result\StepResult;
use Behat\Testwork\Counter\Memory;
use Behat\Testwork\Counter\Timer;
use Behat\Testwork\Tester\Result\TestResult;
use Behat\Testwork\Tester\Result\TestResults;
use Behat\Behat\Output\Statistics\Statistics;
use Behat\Behat\Output\Statistics\ScenarioStat;
use Behat\Behat\Output\Statistics\StepStat;
use Behat\Behat\Output\Statistics\HookStat;
/**
* Copy paste of Behat\Behat\Output\Statistics\TotalStatistics which is a final class
*
* Modified to remove duplicated stats from reruns
*/
class RerunTotalStatistics implements Statistics
{
/**
* @var Timer
*/
private $timer;
/**
* @var Memory
*/
private $memory;
/**
* @var array
*/
private $scenarioCounters = array();
/**
* @var array
*/
private $stepCounters = array();
/**
* @var ScenarioStat[]
*/
private $failedScenarioStats = array();
/**
* @var ScenarioStat[]
*/
private $skippedScenarioStats = array();
/**
* @var StepStat[]
*/
private $failedStepStats = array();
/**
* @var StepStat[]
*/
private $pendingStepStats = array();
/**
* @var HookStat[]
*/
private $failedHookStats = array();
// start modifications here
/**
* @var StepStat[]
*/
private $passedStepStats = array();
// end modifications here
/**
* Initializes statistics.
*/
public function __construct()
{
$this->resetAllCounters();
$this->timer = new Timer();
$this->memory = new Memory();
}
public function resetAllCounters()
{
$this->scenarioCounters = $this->stepCounters = array(
TestResult::PASSED => 0,
TestResult::FAILED => 0,
StepResult::UNDEFINED => 0,
TestResult::PENDING => 0,
TestResult::SKIPPED => 0
);
}
/**
* Starts timer.
*/
public function startTimer()
{
$this->timer->start();
}
/**
* Stops timer.
*/
public function stopTimer()
{
$this->timer->stop();
}
/**
* Returns timer object.
*
* @return Timer
*/
public function getTimer()
{
return $this->timer;
}
/**
* Returns memory usage object.
*
* @return Memory
*/
public function getMemory()
{
return $this->memory;
}
/**
* Registers scenario stat.
*
* @param ScenarioStat $stat
*/
public function registerScenarioStat(ScenarioStat $stat)
{
if (TestResults::NO_TESTS === $stat->getResultCode()) {
return;
}
$this->scenarioCounters[$stat->getResultCode()]++;
// start modifications here
if (TestResult::FAILED === $stat->getResultCode()) {
// Ensure that any scenario reruns aren't counted as additional failures
$alreadyHasFailure = false;
foreach ($this->failedScenarioStats as $failedStat) {
if ($failedStat->getPath() === $stat->getPath()) {
$alreadyHasFailure = true;
break;
}
}
if (!$alreadyHasFailure) {
$this->failedScenarioStats[] = $stat;
} else {
$this->scenarioCounters[TestResult::FAILED]--;
}
}
if (TestResult::PASSED == $stat->getResultCode()) {
// Remove the scenario from the failed scenarios list if it passes on rerun
$newFailedScenarioStats = [];
foreach ($this->failedScenarioStats as $failedStat) {
if ($failedStat->getPath() !== $stat->getPath()) {
$newFailedScenarioStats[] = $failedStat;
} else {
$this->scenarioCounters[TestResult::FAILED]--;
}
}
$this->failedScenarioStats = $newFailedScenarioStats;
}
// end modifications here
if (TestResult::SKIPPED === $stat->getResultCode()) {
$this->skippedScenarioStats[] = $stat;
}
}
/**
* Registers step stat.
*
* @param StepStat $stat
*/
public function registerStepStat(StepStat $stat)
{
$this->stepCounters[$stat->getResultCode()]++;
// start modifications here
if (TestResult::FAILED === $stat->getResultCode()) {
// Ensure that any scenario reruns don't double count step failures
$alreadyHasFailure = false;
foreach ($this->failedStepStats as $failedStat) {
if ($failedStat->getPath() === $stat->getPath()) {
$alreadyHasFailure = true;
break;
}
}
if (!$alreadyHasFailure) {
$this->failedStepStats[] = $stat;
} else {
$this->stepCounters[TestResult::FAILED]--;
}
}
if (TestResult::PASSED == $stat->getResultCode()) {
// Remove any duplicate passes on scenario rerun
$alreadyHasSuccess = false;
foreach ($this->passedStepStats as $passedStat) {
if ($passedStat->getPath() === $stat->getPath()) {
$alreadyHasSuccess = true;
break;
}
}
if (!$alreadyHasSuccess) {
$this->passedStepStats[] = $stat;
} else {
$this->stepCounters[TestResult::PASSED]--;
}
// Remove the step from the failed steps list if it passes on scenario rerun
$newFailedStepStats = [];
foreach ($this->failedStepStats as $failedStat) {
if ($failedStat->getPath() !== $stat->getPath()) {
$newFailedStepStats[] = $failedStat;
} else {
$this->stepCounters[TestResult::FAILED]--;
}
}
$this->failedStepStats = $newFailedStepStats;
}
// end modifications here
if (TestResult::PENDING === $stat->getResultCode()) {
$this->pendingStepStats[] = $stat;
}
}
/**
* Registers hook stat.
*
* @param HookStat $stat
*/
public function registerHookStat(HookStat $stat)
{
if ($stat->isSuccessful()) {
return;
}
$this->failedHookStats[] = $stat;
}
/**
* Returns counters for different scenario result codes.
*
* @return array[]
*/
public function getScenarioStatCounts()
{
return $this->scenarioCounters;
}
/**
* Returns skipped scenario stats.
*
* @return ScenarioStat[]
*/
public function getSkippedScenarios()
{
return $this->skippedScenarioStats;
}
/**
* Returns failed scenario stats.
*
* @return ScenarioStat[]
*/
public function getFailedScenarios()
{
return $this->failedScenarioStats;
}
/**
* Returns counters for different step result codes.
*
* @return array[]
*/
public function getStepStatCounts()
{
return $this->stepCounters;
}
/**
* Returns failed step stats.
*
* @return StepStat[]
*/
public function getFailedSteps()
{
return $this->failedStepStats;
}
/**
* Returns pending step stats.
*
* @return StepStat[]
*/
public function getPendingSteps()
{
return $this->pendingStepStats;
}
/**
* Returns failed hook stats.
*
* @return HookStat[]
*/
public function getFailedHookStats()
{
return $this->failedHookStats;
}
}