Merge pull request #269 from creative-commoners/pulls/4/behat-rerun

[Backport] Rerun failed features in ci

config/silverstripe.yml

@@ -8,6 +8,7 @@ parameters:
   silverstripe_extension.ajax_steps: ~
   silverstripe_extension.ajax_timeout: ~
   silverstripe_extension.admin_url: ~
+  silverstripe_extension.is_ci: ~
   silverstripe_extension.login_url: ~
   silverstripe_extension.screenshot_path: ~
   silverstripe_extension.module:

src/Extension.php

@@ -18,6 +18,9 @@ use Behat\Testwork\ServiceContainer\Extension as ExtensionInterface;
 use RuntimeException;
 use Symfony\Component\DependencyInjection\Definition;
 use Symfony\Component\DependencyInjection\Reference;
+use Behat\Behat\Tester\ServiceContainer\TesterExtension;
+use SilverStripe\BehatExtension\Utility\RerunTotalStatistics;
+use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester;
 
 /*
  * This file is part of the SilverStripe\BehatExtension
@@ -98,6 +101,22 @@ class Extension implements ExtensionInterface
              $container->setParameter('silverstripe_extension.region_map', $config['region_map']);
         }
         $container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
+        $container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);
+
+        // When running in CI, behat scenarios will occasionally sporadically fail
+        // Replaces services with custom implementations that will rerun failed features
+        // Note that features rather than scenarios need to be rerun to ensure that
+        // everything is setup and torn down correctly and that "Background" bits of
+        // feature fits are rerun
+        if ($config['is_ci']) {
+            $definition = new Definition(RerunRuntimeSuiteTester::class, array(
+                new Reference(TesterExtension::SPECIFICATION_TESTER_ID)
+            ));
+            $container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition);
+
+            $definition = new Definition(RerunTotalStatistics::class);
+            $container->setDefinition('output.pretty.statistics', $definition);
+        }
     }
 
     /**
@@ -140,6 +159,9 @@ class Extension implements ExtensionInterface
                     info('Number of seconds that @retry tags will retry for')->
                     defaultValue(2)->
                 end()->
+                scalarNode('is_ci')->
+                    defaultValue(false)->
+                end()->
                 arrayNode('ajax_steps')->
                     defaultValue(array(
                         'go to',

src/Utility/RerunRuntimeSuiteTester.php

@@ -0,0 +1,82 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Testwork\Environment\Environment;
+use Behat\Testwork\Specification\SpecificationIterator;
+use Behat\Testwork\Tester\Result\IntegerTestResult;
+use Behat\Testwork\Tester\Result\TestResult;
+use Behat\Testwork\Tester\Result\TestResults;
+use Behat\Testwork\Tester\Result\TestWithSetupResult;
+use Behat\Testwork\Tester\Setup\SuccessfulSetup;
+use Behat\Testwork\Tester\Setup\SuccessfulTeardown;
+use Behat\Testwork\Tester\SpecificationTester;
+use Behat\Testwork\Tester\SuiteTester;
+
+/**
+ * Copy paste of Behat\Testwork\Tester\Runtime\RuntimeSuiteTester which is a final class
+ *
+ * Modified so that it reruns failed features
+ */
+class RerunRuntimeSuiteTester implements SuiteTester
+{
+    /**
+     * @var SpecificationTester
+     */
+    private $specTester;
+
+    /**
+     * Initializes tester.
+     *
+     * @param SpecificationTester $specTester
+     */
+    public function __construct(SpecificationTester $specTester)
+    {
+        $this->specTester = $specTester;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function setUp(Environment $env, SpecificationIterator $iterator, $skip)
+    {
+        return new SuccessfulSetup();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function test(Environment $env, SpecificationIterator $iterator, $skip = false)
+    {
+        $results = array();
+        foreach ($iterator as $specification) {
+            $setup = $this->specTester->setUp($env, $specification, $skip);
+            $localSkip = !$setup->isSuccessful() || $skip;
+            $testResult = $this->specTester->test($env, $specification, $localSkip);
+            $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
+
+            // start modifications here
+            if (!$testResult->isPassed()) {
+                file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL);
+                $setup = $this->specTester->setUp($env, $specification, $skip);
+                $localSkip = !$setup->isSuccessful() || $skip;
+                $testResult = $this->specTester->test($env, $specification, $localSkip);
+                $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
+            }
+            // end modifications here
+
+            $integerResult = new IntegerTestResult($testResult->getResultCode());
+            $results[] = new TestWithSetupResult($setup, $integerResult, $teardown);
+        }
+
+        return new TestResults($results);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result)
+    {
+        return new SuccessfulTeardown();
+    }
+}

src/Utility/RerunTotalStatistics.php

@@ -0,0 +1,314 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Behat\Tester\Result\StepResult;
+use Behat\Testwork\Counter\Memory;
+use Behat\Testwork\Counter\Timer;
+use Behat\Testwork\Tester\Result\TestResult;
+use Behat\Testwork\Tester\Result\TestResults;
+use Behat\Behat\Output\Statistics\Statistics;
+use Behat\Behat\Output\Statistics\ScenarioStat;
+use Behat\Behat\Output\Statistics\StepStat;
+use Behat\Behat\Output\Statistics\HookStat;
+
+/**
+ * Copy paste of Behat\Behat\Output\Statistics\TotalStatistics which is a final class
+ *
+ * Modified to remove duplicated stats from reruns
+ */
+class RerunTotalStatistics implements Statistics
+{
+    /**
+     * @var Timer
+     */
+    private $timer;
+    /**
+     * @var Memory
+     */
+    private $memory;
+    /**
+     * @var array
+     */
+    private $scenarioCounters = array();
+    /**
+     * @var array
+     */
+    private $stepCounters = array();
+    /**
+     * @var ScenarioStat[]
+     */
+    private $failedScenarioStats = array();
+    /**
+     * @var ScenarioStat[]
+     */
+    private $skippedScenarioStats = array();
+    /**
+     * @var StepStat[]
+     */
+    private $failedStepStats = array();
+    /**
+     * @var StepStat[]
+     */
+    private $pendingStepStats = array();
+    /**
+     * @var HookStat[]
+     */
+    private $failedHookStats = array();
+    // start modifications here
+    /**
+     * @var StepStat[]
+     */
+    private $passedStepStats = array();
+    // end modifications here
+
+    /**
+     * Initializes statistics.
+     */
+    public function __construct()
+    {
+        $this->resetAllCounters();
+
+        $this->timer = new Timer();
+        $this->memory = new Memory();
+    }
+
+    public function resetAllCounters()
+    {
+        $this->scenarioCounters = $this->stepCounters = array(
+            TestResult::PASSED    => 0,
+            TestResult::FAILED    => 0,
+            StepResult::UNDEFINED => 0,
+            TestResult::PENDING   => 0,
+            TestResult::SKIPPED   => 0
+        );
+    }
+
+    /**
+     * Starts timer.
+     */
+    public function startTimer()
+    {
+        $this->timer->start();
+    }
+
+    /**
+     * Stops timer.
+     */
+    public function stopTimer()
+    {
+        $this->timer->stop();
+    }
+
+    /**
+     * Returns timer object.
+     *
+     * @return Timer
+     */
+    public function getTimer()
+    {
+        return $this->timer;
+    }
+
+    /**
+     * Returns memory usage object.
+     *
+     * @return Memory
+     */
+    public function getMemory()
+    {
+        return $this->memory;
+    }
+
+    /**
+     * Registers scenario stat.
+     *
+     * @param ScenarioStat $stat
+     */
+    public function registerScenarioStat(ScenarioStat $stat)
+    {
+        if (TestResults::NO_TESTS === $stat->getResultCode()) {
+            return;
+        }
+
+        $this->scenarioCounters[$stat->getResultCode()]++;
+
+        // start modifications here
+        if (TestResult::FAILED === $stat->getResultCode()) {
+            // Ensure that any scenario reruns aren't counted as additional failures
+            $alreadyHasFailure = false;
+            foreach ($this->failedScenarioStats as $failedStat) {
+                if ($failedStat->getPath() === $stat->getPath()) {
+                    $alreadyHasFailure = true;
+                    break;
+                }
+            }
+            if (!$alreadyHasFailure) {
+                $this->failedScenarioStats[] = $stat;
+            } else {
+                $this->scenarioCounters[TestResult::FAILED]--;
+            }
+        }
+
+        if (TestResult::PASSED == $stat->getResultCode()) {
+            // Remove the scenario from the failed scenarios list if it passes on rerun
+            $newFailedScenarioStats = [];
+            foreach ($this->failedScenarioStats as $failedStat) {
+                if ($failedStat->getPath() !== $stat->getPath()) {
+                    $newFailedScenarioStats[] = $failedStat;
+                } else {
+                    $this->scenarioCounters[TestResult::FAILED]--;
+                }
+            }
+            $this->failedScenarioStats = $newFailedScenarioStats;
+        }
+        // end modifications here
+
+        if (TestResult::SKIPPED === $stat->getResultCode()) {
+            $this->skippedScenarioStats[] = $stat;
+        }
+    }
+
+    /**
+     * Registers step stat.
+     *
+     * @param StepStat $stat
+     */
+    public function registerStepStat(StepStat $stat)
+    {
+        $this->stepCounters[$stat->getResultCode()]++;
+
+        // start modifications here
+        if (TestResult::FAILED === $stat->getResultCode()) {
+            // Ensure that any scenario reruns don't double count step failures
+            $alreadyHasFailure = false;
+            foreach ($this->failedStepStats as $failedStat) {
+                if ($failedStat->getPath() === $stat->getPath()) {
+                    $alreadyHasFailure = true;
+                    break;
+                }
+            }
+            if (!$alreadyHasFailure) {
+                $this->failedStepStats[] = $stat;
+            } else {
+                $this->stepCounters[TestResult::FAILED]--;
+            }
+        }
+
+        if (TestResult::PASSED == $stat->getResultCode()) {
+            // Remove any duplicate passes on scenario rerun
+            $alreadyHasSuccess = false;
+            foreach ($this->passedStepStats as $passedStat) {
+                if ($passedStat->getPath() === $stat->getPath()) {
+                    $alreadyHasSuccess = true;
+                    break;
+                }
+            }
+            if (!$alreadyHasSuccess) {
+                $this->passedStepStats[] = $stat;
+            } else {
+                $this->stepCounters[TestResult::PASSED]--;
+            }
+
+            // Remove the step from the failed steps list if it passes on scenario rerun
+            $newFailedStepStats = [];
+            foreach ($this->failedStepStats as $failedStat) {
+                if ($failedStat->getPath() !== $stat->getPath()) {
+                    $newFailedStepStats[] = $failedStat;
+                } else {
+                    $this->stepCounters[TestResult::FAILED]--;
+                }
+            }
+            $this->failedStepStats = $newFailedStepStats;
+        }
+        // end modifications here
+
+        if (TestResult::PENDING === $stat->getResultCode()) {
+            $this->pendingStepStats[] = $stat;
+        }
+    }
+
+    /**
+     * Registers hook stat.
+     *
+     * @param HookStat $stat
+     */
+    public function registerHookStat(HookStat $stat)
+    {
+        if ($stat->isSuccessful()) {
+            return;
+        }
+
+        $this->failedHookStats[] = $stat;
+    }
+
+    /**
+     * Returns counters for different scenario result codes.
+     *
+     * @return array[]
+     */
+    public function getScenarioStatCounts()
+    {
+        return $this->scenarioCounters;
+    }
+
+    /**
+     * Returns skipped scenario stats.
+     *
+     * @return ScenarioStat[]
+     */
+    public function getSkippedScenarios()
+    {
+        return $this->skippedScenarioStats;
+    }
+
+    /**
+     * Returns failed scenario stats.
+     *
+     * @return ScenarioStat[]
+     */
+    public function getFailedScenarios()
+    {
+        return $this->failedScenarioStats;
+    }
+
+    /**
+     * Returns counters for different step result codes.
+     *
+     * @return array[]
+     */
+    public function getStepStatCounts()
+    {
+        return $this->stepCounters;
+    }
+
+    /**
+     * Returns failed step stats.
+     *
+     * @return StepStat[]
+     */
+    public function getFailedSteps()
+    {
+        return $this->failedStepStats;
+    }
+
+    /**
+     * Returns pending step stats.
+     *
+     * @return StepStat[]
+     */
+    public function getPendingSteps()
+    {
+        return $this->pendingStepStats;
+    }
+
+    /**
+     * Returns failed hook stats.
+     *
+     * @return HookStat[]
+     */
+    public function getFailedHookStats()
+    {
+        return $this->failedHookStats;
+    }
+}