API @retry behaviour

src/Context/BasicContext.php

@@ -17,6 +17,7 @@ use Behat\Testwork\Tester\Result\TestResult;
 use Exception;
 use SilverStripe\Assets\File;
 use SilverStripe\Assets\Filesystem;
+use SilverStripe\BehatExtension\Utility\StepHelper;
 use WebDriver\Exception as WebDriverException;
 use WebDriver\Session as WebDriverSession;
 
@@ -31,13 +32,7 @@ use WebDriver\Session as WebDriverSession;
 class BasicContext implements Context
 {
     use MainContextAwareTrait;
-
-    /**
-     * Work-around for https://github.com/Behat/Behat/issues/653
-     *
-     * @var ScenarioNode
-     */
-    protected $currentScenario = null;
+    use StepHelper;
 
     /**
      * Date format in date() syntax
@@ -71,28 +66,6 @@ class BasicContext implements Context
         return $context->getSession($name);
     }
 
-    /**
-     * Work-around for https://github.com/Behat/Behat/issues/653
-     *
-     * @BeforeScenario
-     * @param BeforeScenarioScope $event
-     */
-    public function handleScenarioBegin(BeforeScenarioScope $event)
-    {
-        $this->currentScenario = $event->getScenario();
-    }
-
-    /**
-     * Work-around for https://github.com/Behat/Behat/issues/653
-     *
-     * @AfterScenario
-     * @param AfterScenarioScope $event
-     */
-    public function handleScenarioEnd(AfterScenarioScope $event)
-    {
-        $this->currentScenario = null;
-    }
-
     /**
      * @AfterStep
      *
@@ -447,11 +420,11 @@ JS;
      */
     public function iShouldSeeAButton($negative, $text)
     {
-        $matchedEl = $this->findNamedButton($text);
+        $button = $this->findNamedButton($text);
         if (trim($negative)) {
-            assertNull($matchedEl, sprintf('%s button found', $text));
+            assertNull($button, sprintf('%s button found', $text));
         } else {
-            assertNotNull($matchedEl, sprintf('%s button not found', $text));
+            assertNotNull($button, sprintf('%s button not found', $text));
         }
     }
 
@@ -1262,15 +1235,16 @@ JS;
      */
     protected function stepHasTag(StepScope $event, $tag)
     {
-        // Check scenario
-        if ($this->currentScenario && $this->currentScenario->hasTag($tag)) {
-            return true;
-        }
         // Check feature
         $feature = $event->getFeature();
         if ($feature && $feature->hasTag($tag)) {
             return true;
         }
+        // Check scenario
+        $scenario = $this->getStepScenario($feature, $event->getStep());
+        if ($scenario && $scenario->hasTag($tag)) {
+            return true;
+        }
         return false;
     }
 }

src/Context/LoginContext.php

@@ -59,7 +59,7 @@ class LoginContext implements Context
     }
 
     /**
-     * @When /^I log in with "(?<username>[^"]*)" and "(?<password>[^"]*)"$/
+     * @When /^I log in with "([^"]*)" and "([^"]*)"$/
      * @param string $email
      * @param string $password
      */

src/Context/RetryableContextTrait.php

@@ -1,25 +0,0 @@
-<?php
-
-namespace SilverStripe\BehatExtension\Context;
-
-trait RetryableContextTrait
-{
-    /**
-     * Invoke callback for a non-empty result with a given timeout
-     *
-     * @param callable $callback
-     * @param int $timeout Number of seconds to retry for
-     * @return mixed Result of invoking $try, or null if timed out
-     */
-    protected function retryUntil($callback, $timeout = 3)
-    {
-        do {
-            $result = $callback();
-            if ($result) {
-                return $result;
-            }
-            sleep(1);
-        } while (--$timeout >= 0);
-        return null;
-    }
-}

src/Extension.php

@@ -2,11 +2,13 @@
 
 namespace SilverStripe\BehatExtension;
 
+use Behat\Testwork\Call\ServiceContainer\CallExtension;
 use Behat\Testwork\Cli\ServiceContainer\CliExtension;
 use Behat\Testwork\Suite\Cli\InitializationController;
 use Behat\Testwork\Suite\ServiceContainer\SuiteExtension;
 use SilverStripe\BehatExtension\Controllers\ModuleInitialisationController;
 use SilverStripe\BehatExtension\Controllers\ModuleSuiteLocator;
+use SilverStripe\BehatExtension\Utility\RetryableCallHandler;
 use Symfony\Component\Config\FileLocator;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
@@ -63,6 +65,7 @@ class Extension implements ExtensionInterface
         // Add CLI substitutions
         $this->loadSuiteLocator($container);
         $this->loadBootstrapController($container);
+        $this->loadCallHandlers($container, $config['error_reporting'], $config['retry_seconds']);
 
         // Set various paths
         $container->setParameter('silverstripe_extension.admin_url', $config['admin_url']);
@@ -110,6 +113,14 @@ class Extension implements ExtensionInterface
                 scalarNode('bootstrap_file')->
                     defaultNull()->
                 end()->
+                scalarNode('error_reporting')->
+                    info('Call executor will catch exceptions matching this level')->
+                    defaultValue(E_ALL | E_STRICT)->
+                end()->
+                scalarNode('retry_seconds')->
+                    info('Number of seconds that @retry tags will retry for')->
+                    defaultValue(2)->
+                end()->
                 arrayNode('ajax_steps')->
                     defaultValue(array(
                         'go to',
@@ -161,4 +172,18 @@ class Extension implements ExtensionInterface
         $definition->addTag(CliExtension::CONTROLLER_TAG, ['priority' => 900]);
         $container->setDefinition(CliExtension::CONTROLLER_TAG . '.initialization', $definition);
     }
+
+    /**
+     * Shivs in custom call handler
+     *
+     * @param ContainerBuilder $container
+     * @param integer $errorReporting
+     * @param int $retrySeconds
+     */
+    protected function loadCallHandlers(ContainerBuilder $container, $errorReporting, $retrySeconds)
+    {
+        $definition = new Definition(RetryableCallHandler::class, [$errorReporting, $retrySeconds]);
+        $definition->addTag(CallExtension::CALL_HANDLER_TAG, ['priority' => 50]);
+        $container->setDefinition(CallExtension::CALL_HANDLER_TAG . '.runtime', $definition);
+    }
 }

src/Utility/RetryableCallHandler.php

@@ -0,0 +1,192 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Behat\Definition\Call\DefinitionCall;
+use Behat\Testwork\Call\Call;
+use Behat\Testwork\Call\CallResult;
+use Behat\Testwork\Call\Exception\CallErrorException;
+use Behat\Testwork\Call\Handler\CallHandler;
+use Behat\Testwork\Call\Handler\RuntimeCallHandler;
+use Exception;
+
+/**
+ * Replaces RuntimeCallHandler with a retry feature
+ * All scenarios or features OPT-IN to retry behaviour with
+ * the @retry tag.
+ *
+ * Note: most of this class is duplicated (sad face) due to final class
+ * @see RuntimeCallHandler
+ */
+class RetryableCallHandler implements CallHandler
+{
+    use StepHelper;
+
+    const RETRY_TAG = 'retry';
+
+    /**
+     * @var integer
+     */
+    private $errorReportingLevel;
+
+    /**
+     * @var bool
+     */
+    private $obStarted = false;
+
+    /**
+     * @var int
+     */
+    protected $retrySeconds;
+
+    /**
+     * Initializes executor.
+     *
+     * @param int $errorReportingLevel
+     * @param int $retrySeconds
+     */
+    public function __construct($errorReportingLevel = E_ALL, $retrySeconds = 3)
+    {
+        $this->errorReportingLevel = $errorReportingLevel;
+        $this->retrySeconds = $retrySeconds;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function supportsCall(Call $call)
+    {
+        return true;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function handleCall(Call $call)
+    {
+        $this->startErrorAndOutputBuffering($call);
+        $result = $this->executeCall($call);
+        $this->stopErrorAndOutputBuffering();
+
+        return $result;
+    }
+
+    /**
+     * Used as a custom error handler when step is running.
+     *
+     * @see set_error_handler()
+     *
+     * @param integer $level
+     * @param string  $message
+     * @param string  $file
+     * @param integer $line
+     *
+     * @return Boolean
+     *
+     * @throws CallErrorException
+     */
+    public function handleError($level, $message, $file, $line)
+    {
+        if ($this->errorLevelIsNotReportable($level)) {
+            return false;
+        }
+
+        throw new CallErrorException($level, $message, $file, $line);
+    }
+
+    /**
+     * Executes single call.
+     *
+     * @param Call $call
+     *
+     * @return CallResult
+     */
+    private function executeCall(Call $call)
+    {
+        $callable = $call->getBoundCallable();
+        $arguments = $call->getArguments();
+        $retry = $this->isCallRetryable($call);
+        $return = $exception = null;
+
+        try {
+            // Determine whether to call with retries
+            if ($retry) {
+                $return = $this->retryThrowable(function () use ($callable, $arguments) {
+                    return call_user_func_array($callable, $arguments);
+                }, $this->retrySeconds);
+            } else {
+                $return = call_user_func_array($callable, $arguments);
+            }
+        } catch (Exception $caught) {
+            $exception = $caught;
+        }
+
+        $stdOut = $this->getBufferedStdOut();
+
+        return new CallResult($call, $return, $exception, $stdOut);
+    }
+
+    /**
+     * Returns buffered stdout.
+     *
+     * @return null|string
+     */
+    private function getBufferedStdOut()
+    {
+        return ob_get_length() ? ob_get_contents() : null;
+    }
+
+    /**
+     * Starts error handler and stdout buffering.
+     *
+     * @param Call $call
+     */
+    private function startErrorAndOutputBuffering(Call $call)
+    {
+        $errorReporting = $call->getErrorReportingLevel() ? : $this->errorReportingLevel;
+        set_error_handler(array($this, 'handleError'), $errorReporting);
+        $this->obStarted = ob_start();
+    }
+
+    /**
+     * Stops error handler and stdout buffering.
+     */
+    private function stopErrorAndOutputBuffering()
+    {
+        if ($this->obStarted) {
+            ob_end_clean();
+        }
+        restore_error_handler();
+    }
+
+    /**
+     * Checks if provided error level is not reportable.
+     *
+     * @param integer $level
+     *
+     * @return Boolean
+     */
+    private function errorLevelIsNotReportable($level)
+    {
+        return !(error_reporting() & $level);
+    }
+
+    /**
+     * Determine if the call is retryable
+     *
+     * @param Call $call
+     * @return bool
+     */
+    protected function isCallRetryable(Call $call)
+    {
+        if (!($call instanceof DefinitionCall)) {
+            return false;
+        }
+        $feature = $call->getFeature();
+        if ($feature->hasTag(self::RETRY_TAG)) {
+            return true;
+        }
+        $scenario = $this->getStepScenario($feature, $call->getStep());
+        return $scenario && $scenario->hasTag(self::RETRY_TAG);
+    }
+}

src/Utility/StepHelper.php

@@ -0,0 +1,63 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Gherkin\Node\FeatureNode;
+use Behat\Gherkin\Node\NodeInterface;
+use Behat\Gherkin\Node\ScenarioInterface;
+use \Exception;
+
+/**
+ * Helpers for working with steps
+ *
+ * Note: Add `@retry` to any feature / scenario to make it retryable
+ */
+trait StepHelper
+{
+    /**
+     * Get scenario from step in a feature node
+     * See https://github.com/Behat/Behat/issues/653
+     *
+     * @param FeatureNode $feature
+     * @param NodeInterface $step
+     * @return ScenarioInterface
+     */
+    protected function getStepScenario(FeatureNode $feature, NodeInterface $step)
+    {
+        $scenario = null;
+        foreach ($feature->getScenarios() as $nextScenario) {
+            if ($nextScenario->getLine() > $step->getLine()) {
+                break;
+            }
+            $scenario = $nextScenario;
+        }
+        return $scenario;
+    }
+
+    /**
+     * Retry until no exceptions are thrown, or until
+     * $timeout seconds are reached.
+     *
+     * If timeout reached, re-throws the first exception.
+     *
+     * @param callable $callback
+     * @param int $timeout
+     * @return mixed
+     * @throws Exception
+     */
+    protected function retryThrowable($callback, $timeout = 3)
+    {
+        $firstEx = null;
+        do {
+            try {
+                return call_user_func($callback);
+            } catch (Exception $ex) {
+                if (!$firstEx) {
+                    $firstEx = $ex;
+                }
+            }
+            sleep(1);
+        } while (--$timeout >= 0);
+        throw $firstEx;
+    }
+}