Merge pull request #268 from creative-commoners/pulls/5/rerun

NEW Rerun failed features in ci

.github/workflows/merge-up.yml

@@ -0,0 +1,17 @@
+name: Merge-up
+
+on:
+  # At 2:20 PM UTC, only on Saturday
+  schedule:
+    - cron: '20 14 * * 6'
+  workflow_dispatch:
+
+jobs:
+  merge-up:
+    name: Merge-up
+    # Only run cron on the silverstripe account
+    if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Merge-up
+        uses: silverstripe/gha-merge-up@v1

README.md

@@ -38,27 +38,21 @@ Note: The extension has only been tested with the `selenium2` Mink driver.
 
 ## Installation
 
-Simply [install Silverstripe through Composer](http://doc.silverstripe.org/framework/en/installation/composer).
-Skip this step if adding the module to an existing project.
+In a Silverstripe CMS project (see [getting started docs](https://docs.silverstripe.org/en/getting_started/)) add the Silverstripe Behat extension via Composer.
 
-	composer create-project silverstripe/installer my-test-project 4.x-dev
+```sh
+composer require --dev silverstripe/behat-extension
+```
 
-Switch to the newly created webroot, and add the Silverstripe Behat extension.
+Download the standalone [Google Chrome WebDriver](https://chromedriver.storage.googleapis.com/index.html)
 
-	cd my-test-project
-	composer require --dev silverstripe/behat-extension
-
-Download the standalone [Google Chrome WebDriver](http://chromedriver.storage.googleapis.com/index.html?path=2.8/)
-
-Now install the Silverstripe project as usual by opening it in a browser and following the instructions.
-Protip: You can skip this step by using `[SS_DATABASE_CHOOSE_NAME]` in a global
-[`.env`](https://docs.silverstripe.org/en/getting_started/environment_management/) file one level above the webroot.
-
-Unless you have [`SS_BASE_URL`](http://doc.silverstripe.org/framework/en/topics/commandline#configuration) set up, 
+Unless you have [`SS_BASE_URL`](http://doc.silverstripe.org/framework/en/topics/commandline#configuration) set up,
 you also need to specify the URL for your webroot. Either add it to the existing `behat.yml` configuration file
 in your project root, or set is as an environment variable in your terminal session:
 
-	export BEHAT_PARAMS="extensions[SilverStripe\BehatExtension\MinkExtension][base_url]=http://localhost/"
+```sh
+export BEHAT_PARAMS="extensions[SilverStripe\BehatExtension\MinkExtension][base_url]=http://localhost/"
+```
 
 ## Usage
 
@@ -66,17 +60,23 @@ in your project root, or set is as an environment variable in your terminal sess
 
 You can run the server locally in a separate Terminal session:
 
-    chromedriver
+```sh
+chromedriver
+```
 
 ### Running the Tests
 
 Now you can run the tests (for example for the `framework` module):
 
-	vendor/bin/behat @framework
+```sh
+vendor/bin/behat @framework
+```
 
 Or even run a single scenario by it's name (supports regular expressions):
 
-	vendor/bin/behat --name 'My scenario title' @framework
+```sh
+vendor/bin/behat --name 'My scenario title' @framework
+```
 
 This will start a Chrome browser by default. Other browsers and profiles can be configured in `behat.yml`.
 
@@ -85,7 +85,9 @@ This will start a Chrome browser by default. Other browsers and profiles can be
 If running with `silverstripe/serve` and `chromedriver`, you can also use the following command
 which will automatically start and stop these services for individual tests.
 
-    vendor/bin/behat-ss @framework
+```sh
+vendor/bin/behat-ss @framework
+```
 
 This automates:
  - starting server
@@ -128,30 +130,32 @@ number that failed.
 
 Example: behat.yml
 
-	default:
-	  suites:
-	    framework:
-          paths:
-            - %paths.modules.framework%/tests/behat/features
-          contexts:
-            - SilverStripe\Framework\Tests\Behaviour\FeatureContext
-            - SilverStripe\Framework\Tests\Behaviour\CmsFormsContext
-            - SilverStripe\Framework\Tests\Behaviour\CmsUiContext
-            - SilverStripe\BehatExtension\Context\BasicContext
-            - SilverStripe\BehatExtension\Context\EmailContext
-            - SilverStripe\BehatExtension\Context\LoginContext
-            -
-              SilverStripe\BehatExtension\Context\FixtureContext:
-                - %paths.modules.framework%/tests/behat/features/files/
-	  extensions:
-        SilverStripe\BehatExtension\MinkExtension:
-          default_session: facebook_web_driver
-          javascript_session: facebook_web_driver
-          facebook_web_driver:
-            browser: chrome
-            wd_host: "http://127.0.0.1:9515" #chromedriver port
-        SilverStripe\BehatExtension\Extension:
-          screenshot_path: %paths.base%/artifacts/screenshots
+```yml
+default:
+    suites:
+    framework:
+        paths:
+        - '%paths.modules.framework%/tests/behat/features'
+        contexts:
+        - SilverStripe\Framework\Tests\Behaviour\FeatureContext
+        - SilverStripe\Framework\Tests\Behaviour\CmsFormsContext
+        - SilverStripe\Framework\Tests\Behaviour\CmsUiContext
+        - SilverStripe\BehatExtension\Context\BasicContext
+        - SilverStripe\BehatExtension\Context\EmailContext
+        - SilverStripe\BehatExtension\Context\LoginContext
+        -
+            SilverStripe\BehatExtension\Context\FixtureContext:
+            - '%paths.modules.framework%/tests/behat/features/files/'
+    extensions:
+    SilverStripe\BehatExtension\MinkExtension:
+        default_session: facebook_web_driver
+        javascript_session: facebook_web_driver
+        facebook_web_driver:
+        browser: chrome
+        wd_host: "http://127.0.0.1:9515" #chromedriver port
+    SilverStripe\BehatExtension\Extension:
+        screenshot_path: '%paths.base%/artifacts/screenshots'
+```
 
 ## Module Initialisation
 
@@ -167,9 +171,11 @@ Since step definitions are quite domain specific, its likely that you'll need yo
 The Silverstripe Behat extension provides an initializer script which generates a template
 in the recommended folder structure:
 
-	vendor/bin/behat --init @mymodule --namespace="MyVendor\MyModule"
+```sh
+vendor/bin/behat --init @mymodule --namespace="MyVendor\MyModule"
+```
 
-	Note: namespace is mandatory
+**Note: namespace is mandatory**
 
 You'll now have a class located in `mymodule/tests/behat/src/FeatureContext.php`,
 which will have a psr-4 class mapping added to composer.json by default.
@@ -182,7 +188,9 @@ The extension comes with several `BehatContext` subclasses come with some extra
 Some of them are just helpful in general website testing, other's are specific to SilverStripe.
 To find out all available steps (and the files they are defined in), run the following:
 
-	vendor/bin/behat @mymodule --definitions=i
+```sh
+vendor/bin/behat @mymodule --definitions=i
+```
 
 Note: There are more specific step definitions in the Silverstripe `framework` module
 for interacting with the CMS interfaces (see `framework/tests/behat/features/bootstrap`).
@@ -207,38 +215,40 @@ scenario automatically.
 If you need more flexibility and transparency about which records are being created,
 use the inline definition syntax. The following example shows some syntax variations:
 
-	Feature: Do something with pages
-		As an site owner
-		I want to manage pages
+```cucumber
+Feature: Do something with pages
+    As an site owner
+    I want to manage pages
 
-		Background:
-			# Creates a new page without data. Can be accessed later under this identifier
-			Given a "page" "Page 1"
-			# Uses a custom RegistrationPage type
-			And an "error page" "Register"
-			# Creates a page with inline properties
-			And a "page" "Page 2" with "URLSegment"="page-1" and "Content"="my page 1"
-			# Field names can be tabular, and based on DataObject::$field_labels
-			And the "page" "Page 3" has the following data
-			 | Content | <blink> |
-			 | My Property | foo |
-			 | My Boolean | bar |
-			# Pages are published by default, can be explicitly unpublished
-			And the "page" "Page 1" is not published
-			# Create a hierarchy, and reference a record created earlier
-			And the "page" "Page 1.1" is a child of a "page" "Page 1"
-			# Specific page type step
-			And a "page" "My Redirect" which redirects to a "page" "Page 1"
-			And a "member" "Website User" with "FavouritePage"="=>Page.Page 1"
+    Background:
+        # Creates a new page without data. Can be accessed later under this identifier
+        Given a "page" "Page 1"
+        # Uses a custom RegistrationPage type
+        And an "error page" "Register"
+        # Creates a page with inline properties
+        And a "page" "Page 2" with "URLSegment"="page-1" and "Content"="my page 1"
+        # Field names can be tabular, and based on DataObject::$field_labels
+        And the "page" "Page 3" has the following data
+            | Content | <blink> |
+            | My Property | foo |
+            | My Boolean | bar |
+        # Pages are published by default, can be explicitly unpublished
+        And the "page" "Page 1" is not published
+        # Create a hierarchy, and reference a record created earlier
+        And the "page" "Page 1.1" is a child of a "page" "Page 1"
+        # Specific page type step
+        And a "page" "My Redirect" which redirects to a "page" "Page 1"
+        And a "member" "Website User" with "FavouritePage"="=>Page.Page 1"
 
-		@javascript
-		Scenario: View a page in the tree
-			Given I am logged in with "ADMIN" permissions
-			And I go to "/admin/pages"
-			Then I should see "Page 1"
+        @javascript
+        Scenario: View a page in the tree
+            Given I am logged in with "ADMIN" permissions
+            And I go to "/admin/pages"
+            Then I should see "Page 1"
+```
 
  * Fixtures are created where you defined them. If you want the fixtures to be created
-   before every scenario, define them in 
+   before every scenario, define them in
    [Background](http://docs.behat.org/en/latest/user_guide/writing_scenarios.html#backgrounds).
    If you want them to be created only when a particular scenario runs, define them there.
  * Fixtures are cleared between scenarios.
@@ -259,8 +269,10 @@ use the inline definition syntax. The following example shows some syntax variat
 As a convention, Silverstripe Behat tests live in a `tests/behat` subfolder
 of your module. You can create it with the following commands:
 
-	mkdir -p mymodule/tests/behat/features/
-	mkdir -p mymodule/tests/behat/src/
+```sh
+mkdir -p mymodule/tests/behat/features/
+mkdir -p mymodule/tests/behat/src/
+```
 
 ### FeatureContext
 
@@ -269,16 +281,17 @@ here as well. The only major difference is the base class from which
 to extend your own `FeatureContext`: It should be `SilverStripeContext`
 rather than `BehatContext`.
 
-Example: mymodule/tests/behat/src/FeatureContext.php
+Example: `mymodule/tests/behat/src/FeatureContext.php`
 
-	<?php
-	namespace MyModule\Test\Behaviour;
+```php
+namespace MyModule\Test\Behaviour;
 
-	use SilverStripe\BehatExtension\Context\SilverStripeContext;
+use SilverStripe\BehatExtension\Context\SilverStripeContext;
 
-	class FeatureContext extends SilverStripeContext
-	{
-	}
+class FeatureContext extends SilverStripeContext
+{
+}
+```
 
 ### Screen Size
 
@@ -287,7 +300,9 @@ define the desired browser window size through a `capabilities` definition.
 By default, Selenium doesn't support this though, so we've added a workaround
 through an environment variable:
 
-	BEHAT_SCREEN_SIZE=320x600 vendor/bin/behat
+```sh
+BEHAT_SCREEN_SIZE=320x600 vendor/bin/behat
+```
 
 ### Inspecting PHP sessions
 
@@ -298,15 +313,18 @@ of the `TestSessionEnvironment`, in order to share it with Behat CLI.
 
 Example: Retrieve the currently logged-in member
 
-    use SilverStripe\TestSession\TestsessionEnvironment;
+```php
+use SilverStripe\TestSession\TestsessionEnvironment;
 
-	$env = Injector::inst()->get(TestSessionEnvironment::class);
-	$state = $env->getState();
-	if(isset($state->session['loggedInAs'])) {
-		$member = \Member::get()->byID($state->session['loggedInAs']);
-	} else {
-		$member = null;
-	}
+$env = Injector::inst()->get(TestSessionEnvironment::class);
+$state = $env->getState();
+
+if (isset($state->session['loggedInAs'])) {
+    $member = \Member::get()->byID($state->session['loggedInAs']);
+} else {
+    $member = null;
+}
+```
 
 ## FAQ
 
@@ -387,19 +405,23 @@ otherwise you'll always have an active debugging session in CLI, never in the br
 
 Then you can choose to enable XDebug for the current CLI run:
 
-	XDEBUG_CONFIG="idekey=macgdbp" vendor/bin/behat
+```sh
+XDEBUG_CONFIG="idekey=macgdbp" vendor/bin/behat
+```
 
 Or you can use the `TESTSESSION_PARAMS` environment variable to pass additional
 parameters to `dev/testsession/start`, and debug in the browser instead.
 
-	TESTSESSION_PARAMS="XDEBUG_SESSION_START=macgdbp" vendor/bin/behat @app
+```sh
+TESTSESSION_PARAMS="XDEBUG_SESSION_START=macgdbp" vendor/bin/behat @app
+```
 
 The `macgdbp` IDE key needs to match your `xdebug.idekey` php.ini setting.
 
 ### How do I set up continuous integration through Travis?
 
 Check out the [travis.yml](https://github.com/silverstripe/silverstripe-framework/blob/master/.travis.yml)
-in `silverstripe/framework` for a good example on how to set up Behat tests through 
+in `silverstripe/framework` for a good example on how to set up Behat tests through
 [travis-ci.org](http://travis-ci.org).
 
 ## Cheatsheet
@@ -410,6 +432,7 @@ It's based on the `vendor/bin/behat -di @cms` output.
 
 ### Basics
 
+```cucumber
 	 Then /^(?:|I )should see "(?P<text>(?:[^"]|\\")*)"$/
 	    - Checks, that page contains specified text.
 
@@ -473,9 +496,11 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	Then /^the "([^"]*)" table should not contain "([^"]*)"$/
 
 	Given /^I click on "([^"]*)" in the "([^"]*)" table$/
+```
 
 ### Navigation
 
+```cucumber
 	Given /^(?:|I )am on homepage$/
 	    - Opens homepage.
 
@@ -496,9 +521,11 @@ It's based on the `vendor/bin/behat -di @cms` output.
 
 	 When /^(?:|I )move forward one page$/
 	    - Moves forward one page in history
+```
 
 ### Forms
 
+```cucumber
 	When /^(?:|I )press "(?P<button>(?:[^"]|\\")*)"$/
 	    - Presses button with specified id|name|title|alt|value.
 
@@ -544,9 +571,7 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	 Then /^the "(?P<checkbox>(?:[^"]|\\")*)" checkbox should not be checked$/
 	    - Checks, that checkbox with specified in|name|label|value is unchecked.
 
-	Given /^(?:|I )attach the file "(?P[^"]*)" to "(?P<field>(?:[^"]|\\")*)" with HTML5$/
-
-	When /^I fill in the "(?P<field>([^"]*))" HTML field with "(?P<value>([^"]*))"$/
+	 When /^I fill in the "(?P<field>([^"]*))" HTML field with "(?P<value>([^"]*))"$/
 
 	 When /^I fill in "(?P<value>([^"]*))" for the "(?P<field>([^"]*))" HTML field$/
 
@@ -564,9 +589,11 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	  - Check an individual input button from a group of inputs
 	  - Example: I select "Admins" from "Groups" input group
 	   (where "Groups" is the title of the CheckboxSetField or OptionsetField form field)
+```
 
 ### Interactions
 
+```cucumber
 	Given /^I press the "([^"]*)" button$/
 
 	Given /^I (click|double click) "([^"]*)" in the "([^"]*)" element$/
@@ -584,9 +611,11 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	Given /^I confirm the dialog$/
 
 	Given /^I dismiss the dialog$/
+```
 
 ### Login
 
+```cucumber
 	Given /^I am logged in with "([^"]*)" permissions$/
 	    - Creates a member in a group with the correct permissions.
 
@@ -597,15 +626,15 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	Given /^I should see a log-in form$/
 
 	 Then /^I will see a "bad" log-in message$/
+```
 
 ### CMS UI
 
+```cucumber
 	 Then /^I should see an edit page form$/
 
 	 Then /^I should see the CMS$/
 
-	 Then /^I should see a "([^"]*)" notice$/
-
 	 Then /^I should see a "([^"]*)" message$/
 
 	Given /^I should see a "([^"]*)" button in CMS Content Toolbar$/
@@ -623,9 +652,11 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	Given /^the preview contains "([^"]*)"$/
 
 	Given /^the preview does not contain "([^"]*)"$/
+```
 
 ### Fixtures
 
+```cucumber
 	Given /^(?:(an|a|the) )"(?<type>[^"]+)" "(?<id>[^"]+)" (:?which )?redirects to (?:(an|a|the) )"(?<targetType>[^"]+)" "(?<targetId>[^"]+)"$/
 	    - Find or create a redirector page and link to another existing page.
 
@@ -667,14 +698,18 @@ It's based on the `vendor/bin/behat -di @cms` output.
 	Given /^the CMS settings have the following data$/
 		- Example: Given the CMS settings has the following data
 		- Note: It only works with the Silverstripe CMS module installed
+```
 
 ### Environment
 
+```cucumber
 	Given /^the current date is "([^"]*)"$/
 	Given /^the current time is "([^"]*)"$/
+```
 
 ### Email
 
+```cucumber
 	Given /^there should (not |)be an email (to|from) "([^"]*)"$/
 
 	Given /^there should (not |)be an email (to|from) "([^"]*)" titled "([^"]*)"$/
@@ -701,6 +736,7 @@ It's based on the `vendor/bin/behat -di @cms` output.
 
     When /^I click on the http link "([^"]*)" in the email$/
         - Example: When I click on the http link "http://localhost/changepassword" in the email
+```
 
 ### Transformations
 
@@ -708,18 +744,18 @@ Behat [transformations](http://docs.behat.org/en/v2.5/guides/2.definitions.html#
 have the ability to change step arguments based on their original value,
 for example to cast any argument matching the `\d` regex into an actual PHP integer.
 
- * `/^(?:(the|a)) time of (?<val>.*)$/`: Transforms relative time statements compatible with 
- [strtotime()](http://www.php.net/manual/en/datetime.formats.relative.php). Example: "the time of 1 hour ago" might 
+ * `/^(?:(the|a)) time of (?<val>.*)$/`: Transforms relative time statements compatible with
+ [strtotime()](http://www.php.net/manual/en/datetime.formats.relative.php). Example: "the time of 1 hour ago" might
  return "22:00:00" if its currently "23:00:00".
- * `/^(?:(the|a)) date of (?<val>.*)$/`: Transforms relative date statements compatible with 
- [strtotime()](http://www.php.net/manual/en/datetime.formats.relative.php). Example: "the date of 2 days ago" might 
+ * `/^(?:(the|a)) date of (?<val>.*)$/`: Transforms relative date statements compatible with
+ [strtotime()](http://www.php.net/manual/en/datetime.formats.relative.php). Example: "the date of 2 days ago" might
  return "2013-10-10" if its currently the 12th of October 2013.
- * `/^(?:(the|a)) datetime of (?<val>.*)$/`: Transforms relative date and time statements compatible with 
- [strtotime()](http://www.php.net/manual/en/datetime.formats.relative.php). Example: "the datetime of 2 days ago" might 
+ * `/^(?:(the|a)) datetime of (?<val>.*)$/`: Transforms relative date and time statements compatible with
+ [strtotime()](http://www.php.net/manual/en/datetime.formats.relative.php). Example: "the datetime of 2 days ago" might
  return "2013-10-10 23:00:00" if its currently the 12th of October 2013.
 
 ## Useful resources
 
-* [Silverstripe CMS architecture](http://doc.silverstripe.org/sapphire/en/trunk/reference/cms-architecture)
+* [Silverstripe CMS architecture](https://docs.silverstripe.org/sapphire/en/trunk/reference/cms-architecture)
 * [Silverstripe Framework Test Module](https://github.com/silverstripe-labs/silverstripe-frameworktest)
-* [Silverstripe Unit and Integration Testing](http://doc.silverstripe.org/sapphire/en/trunk/topics/testing)
+* [Silverstripe Unit and Integration Testing](https://docs.silverstripe.org/sapphire/en/trunk/topics/testing)

composer.json

@@ -21,17 +21,17 @@
         }
     ],
     "require": {
-        "php": "^7.4 || ^8.0",
+        "php": "^8.1",
         "phpunit/phpunit": "^9.5",
-        "squizlabs/php_codesniffer": "^3",
-        "behat/behat": "^3.9",
-        "behat/mink": "^1.7",
-        "behat/mink-extension": "^2.1",
-        "silverstripe/mink-facebook-web-driver": "^1",
-        "symfony/dom-crawler": "^3 || ^4",
-        "silverstripe/testsession": "^2.2",
-        "silverstripe/framework": "^4.10",
-        "symfony/finder": "^3.2 || ^4"
+        "squizlabs/php_codesniffer": "^3.7",
+        "behat/behat": "^3.11.0",
+        "behat/mink": "^1.10.0",
+        "friends-of-behat/mink-extension": "^2",
+        "silverstripe/mink-facebook-web-driver": "^2",
+        "symfony/dom-crawler": "^6.1",
+        "silverstripe/testsession": "^3",
+        "silverstripe/framework": "^5",
+        "symfony/finder": "^6.1"
     },
     "autoload": {
         "psr-4": {

config/silverstripe.yml

@@ -8,6 +8,7 @@ parameters:
   silverstripe_extension.ajax_steps: ~
   silverstripe_extension.ajax_timeout: ~
   silverstripe_extension.admin_url: ~
+  silverstripe_extension.is_ci: ~
   silverstripe_extension.login_url: ~
   silverstripe_extension.screenshot_path: ~
   silverstripe_extension.module:

docs/circleci-tutorial.md

@@ -39,7 +39,7 @@ dependencies:
       cat << 'EOF' > _ss_environment.php
       <?php
       define('SS_DATABASE_SERVER', '127.0.0.1');
-      define('SS_DATABASE_CLASS', 'MySQLPDODatabase');
+      define('SS_DATABASE_CLASS', 'MySQLDatabase');
       define('SS_DATABASE_USERNAME', 'ubuntu');
       define('SS_DATABASE_PASSWORD', '');
       define('SS_ENVIRONMENT_TYPE', 'dev');

src/Context/BasicContext.php

@@ -2,16 +2,15 @@
 
 namespace SilverStripe\BehatExtension\Context;
 
-use SilverStripe\Dev\Deprecation;
 use Exception;
 use InvalidArgumentException;
 use Behat\Behat\Context\Context;
 use Behat\Behat\Context\Environment\InitializedContextEnvironment;
-use Behat\Behat\Definition\Call;
 use Behat\Behat\Hook\Scope\AfterScenarioScope;
 use Behat\Behat\Hook\Scope\AfterStepScope;
 use Behat\Behat\Hook\Scope\BeforeScenarioScope;
 use Behat\Behat\Hook\Scope\BeforeStepScope;
+use Behat\Mink\Driver\Selenium2Driver;
 use Behat\Mink\Element\NodeElement;
 use Behat\Mink\Exception\ElementNotFoundException;
 use Behat\Mink\Session;
@@ -19,9 +18,11 @@ use Behat\Testwork\Tester\Result\TestResult;
 use Facebook\WebDriver\Exception\WebDriverException;
 use Facebook\WebDriver\WebDriver;
 use Facebook\WebDriver\WebDriverAlert;
+use Facebook\WebDriver\WebDriverBy;
 use Facebook\WebDriver\WebDriverExpectedCondition;
 use Facebook\WebDriver\WebDriverKeys;
 use PHPUnit\Framework\Assert;
+use PHPUnit\Framework\ExpectationFailedException;
 use SilverStripe\Assets\File;
 use SilverStripe\Assets\Filesystem;
 use SilverStripe\BehatExtension\Utility\StepHelper;
@@ -562,7 +563,12 @@ JS;
      */
     public function iSeeTheDialogText($expected)
     {
-        $text = $this->getExpectedAlert()->getText();
+        $driver = $this->getSession()->getDriver();
+        if ($driver instanceof Selenium2Driver) {
+            $text = $driver->getWebDriverSession()->getAlert_text();
+        } else {
+            $text = $this->getExpectedAlert()->getText();
+        }
         Assert::assertStringContainsString($expected, $text);
     }
 
@@ -597,12 +603,17 @@ JS;
      */
     public function iConfirmTheDialog()
     {
-        $session = $this->getWebDriverSession();
-        $session->wait()->until(
-            WebDriverExpectedCondition::alertIsPresent(),
-            "Alert is expected"
-        );
-        $session->switchTo()->alert()->accept();
+        $driver = $this->getSession()->getDriver();
+        if ($driver instanceof Selenium2Driver) {
+            $driver->getWebDriverSession()->accept_alert();
+        } else {
+            $session = $this->getWebDriverSession();
+            $session->wait()->until(
+                WebDriverExpectedCondition::alertIsPresent(),
+                "Alert is expected"
+            );
+            $session->switchTo()->alert()->accept();
+        }
         $this->handleAjaxTimeout();
     }
 
@@ -611,7 +622,12 @@ JS;
      */
     public function iDismissTheDialog()
     {
-        $this->getExpectedAlert()->dismiss();
+        $driver = $this->getSession()->getDriver();
+        if ($driver instanceof Selenium2Driver) {
+            $driver->getWebDriverSession()->dismiss_alert();
+        } else {
+            $this->getExpectedAlert()->dismiss();
+        }
         $this->handleAjaxTimeout();
     }
 
@@ -630,31 +646,6 @@ JS;
         return $driver->getWebDriver();
     }
 
-    /**
-     * @Given /^(?:|I )attach the file "(?P<path>[^"]*)" to "(?P<field>(?:[^"]|\\")*)" with HTML5$/
-     * @param string $field
-     * @param string $path
-     * @return Call\Given
-     *
-     * @deprecated 4.5.0 Use iAttachTheFileToTheField() instead
-     */
-    public function iAttachTheFileTo($field, $path)
-    {
-        Deprecation::notice('4.5.0', 'Use iAttachTheFileToTheField() instead');
-        // Remove wrapped button styling to make input field accessible to Selenium
-        $js = <<<JS
-let input = jQuery('[name="$field"]');
-if(input.closest('.ss-uploadfield-item-info').length) {
-    while(!input.parent().is('.ss-uploadfield-item-info')) input = input.unwrap();
-}
-JS;
-
-        $this->getSession()->executeScript($js);
-        $this->getSession()->wait(1000);
-
-        return $this->getMainContext()->attachFileToField($field, $path);
-    }
-
     /**
      * Select an individual input from within a group, matched by the top-most label.
      *
@@ -1387,13 +1378,56 @@ JS;
      *
      * @Then /^I add "([^"]+)" to the "([^"]+)" tag field$/
      * @param string $value
-     * @param string $locator
+     * @param string $selector
      */
-    public function iAddToTheTagField($value, $locator)
+    public function iAddToTheTagField($value, $selector)
     {
-        $tagFieldInput = $this->getElement($locator);
-        $tagFieldInput->setValue($value);
-        $tagFieldInput->getParent()->getParent()->getParent()->getParent()->find('css', '.Select-menu-outer')->click();
+        $page = $this->getSession()->getPage();
+        /** @var NodeElement $parentElement */
+        $parentElement = null;
+        $dropdown = null;
+        $this->retryThrowable(function () use (&$parentElement, &$page, $selector) {
+            $parentElement = $page->find('css', $selector);
+            Assert::assertNotNull($parentElement, sprintf('"%s" element not found', $selector));
+            $page = $this->getSession()->getPage();
+        });
+
+        $this->retryThrowable(function () use (&$dropdown, $parentElement, $selector) {
+            $dropdown = $parentElement->find('css', '.ss-tag-field__dropdown-indicator');
+            Assert::assertNotNull($dropdown, sprintf('Unable to find the dropdown in "%s"', $selector));
+            $dropdown->click();
+        });
+
+        $inputField = null;
+        try {
+            // Try setting to a value already in the dropdown
+            $this->retryThrowable(function () use ($value, $parentElement, $selector) {
+                $element = $parentElement->find('xpath', sprintf('//*[count(*)=0 and .="%s"]', $value));
+                Assert::assertNotNull($element, sprintf('"%s" not found in "%s"', $value, $selector));
+                $element->click();
+            });
+        } catch (ExpectationFailedException $e) {
+            // Try creating a new value
+            $this->retryThrowable(function () use (&$inputField, $value, $parentElement, $selector) {
+                /** @var NodeElement $parentElement */
+                $inputField = $parentElement->find('css', '.ss-tag-field__input');
+                Assert::assertNotNull($inputField, sprintf('Could not create "%s" in "%s"', $value, $selector));
+                // We need to type the value in - react won't accept us just setting the value via js
+                $inputField->focus();
+                /** @var FacebookWebDriver $driver */
+                $driver = $this->getSession()->getDriver();
+                $keyboard = $driver->getWebDriver()->getKeyboard();
+                $keyboard->sendKeys($value);
+            });
+
+            // Try selecting the 'Create "$value"' option
+            $this->retryThrowable(function () use ($value, $parentElement, $selector) {
+                $createOption = 'Create "' . $value . '"';
+                $element = $parentElement->find('xpath', sprintf('//*[count(*)=0 and .=\'%s\']', $createOption));
+                Assert::assertNotNull($element, sprintf('"%s" not found in "%s"', $createOption, $selector));
+                $element->click();
+            });
+        }
     }
 
     /**
@@ -1549,4 +1583,71 @@ JS;
         }
         $this->getSession()->executeScript("document.location.href = '{$href}';");
     }
+
+    /**
+     * @Given /^I focus on the "([^"]+)" element$/
+     */
+    public function iFocusOnTheElement(string $selector)
+    {
+        $page = $this->getSession()->getPage();
+        $element = $page->find('css', $selector);
+        Assert::assertNotNull($element, sprintf('Element %s not found', $selector));
+        $element->focus();
+    }
+
+    /**
+     * @Given /^I type "([^"]+)" in the field$/
+     *
+     * This method is used to type into the active element.
+     * It's used to type into fields that are input fields and currently active (has focus).
+     * Should be used together with the step "I focus on the "selector" element"
+     *
+     * Example:
+     * When I focus on the "selector" element
+     * And I type "text" in the field
+     */
+    public function iTypeInTheField(string $data)
+    {
+        $driver = $this->getSession()->getDriver()->getWebDriver();
+        $driver->switchTo()->activeElement();
+        $keyboard = $driver->getKeyboard();
+        $keyboard->sendKeys([$data]);
+    }
+
+    /**
+     * @Given /^the active element should be "([^"]+)"$/
+     *
+     * Example: And the active element should be "selector"
+     */
+    public function theActiveElementShouldBe(string $selector)
+    {
+        $driver = $this->getSession()->getDriver()->getWebDriver();
+        $element = $driver->findElement(WebDriverBy::cssSelector($selector));
+        Assert::assertNotNull($element, sprintf('Element %s not found', $selector));
+        Assert::assertTrue($element->equals($driver->switchTo()->activeElement()));
+    }
+
+    /**
+     * @Given /^I type "([^"]+)" in the active element "([^"]+)"$/
+     *
+     * This method is used to type into the active element.
+     *
+     * Example: And I type "text" in the active element "selector"
+     */
+    public function iTypeInTheActiveElement(string $data, string $selector)
+    {
+        $this->theActiveElementShouldBe($selector);
+        $this->iTypeInTheField($data);
+    }
+
+    /**
+     * Globally press the key i.e. not type into an input and confirm dialog
+     *
+     * @When /^I press the "([^"]+)" key and confirm the dialog$/
+     */
+    public function iPressTheKeyAndConfirmTheDialog(string $key)
+    {
+        $this->iPressTheKeyGlobally($key);
+        $this->iConfirmTheDialog();
+    }
 }

src/Context/EmailContext.php

@@ -8,9 +8,11 @@ use Behat\Gherkin\Node\TableNode;
 use Behat\Mink\Session;
 use PHPUnit\Framework\Assert;
 use SilverStripe\BehatExtension\Utility\TestMailer;
-use SilverStripe\Control\Email\Mailer;
 use SilverStripe\Core\Injector\Injector;
 use Symfony\Component\DomCrawler\Crawler;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\Mailer\MailerInterface;
+use Symfony\Component\Mailer\Transport\NullTransport;
 
 /**
  * Context used to define steps related to email sending.
@@ -48,8 +50,10 @@ class EmailContext implements Context
     {
         // Also set through the 'supportbehat' extension
         // to ensure its available both in CLI execution and the tested browser session
-        $this->mailer = new TestMailer();
-        Injector::inst()->registerService($this->mailer, Mailer::class);
+        $dispatcher = Injector::inst()->get(EventDispatcherInterface::class . '.mailer');
+        $transport = new NullTransport($dispatcher);
+        $this->mailer = new TestMailer($transport, $dispatcher);
+        Injector::inst()->registerService($this->mailer, MailerInterface::class);
     }
 
     /**
@@ -60,7 +64,7 @@ class EmailContext implements Context
      */
     public function thereIsAnEmailFromTo($negate, $direction, $email)
     {
-        $to = ($direction == 'to') ? $email : null;
+        $to = ($direction == 'to') ? $email : '';
         $from = ($direction == 'from') ? $email : null;
         $match = $this->mailer->findEmail($to, $from);
         if (trim($negate ?? '')) {
@@ -80,12 +84,12 @@ class EmailContext implements Context
      */
     public function thereIsAnEmailFromToTitled($negate, $direction, $email, $subject)
     {
-        $to = ($direction == 'to') ? $email : null;
+        $to = ($direction == 'to') ? $email : '';
         $from = ($direction == 'from') ? $email : null;
         $match = $this->mailer->findEmail($to, $from, $subject);
         $allMails = $this->mailer->findEmails($to, $from);
         $allTitles = $allMails ? '"' . implode('","', array_map(function ($email) {
-            return $email->Subject;
+            return $email['Subject'];
         }, $allMails)) . '"' : null;
         if (trim($negate ?? '')) {
             Assert::assertNull($match);
@@ -121,10 +125,10 @@ class EmailContext implements Context
 
         $email = $this->lastMatchedEmail;
         $emailContent = null;
-        if ($email->Content) {
-            $emailContent = $email->Content;
+        if ($email['Content']) {
+            $emailContent = $email['Content'];
         } else {
-            $emailContent = $email->PlainContent;
+            $emailContent = $email['PlainContent'];
         }
 
         if (trim($negate ?? '')) {
@@ -150,8 +154,7 @@ class EmailContext implements Context
         }
 
         $email = $this->lastMatchedEmail;
-        $emailContent = ($email->Content) ? ($email->Content) : ($email->PlainContent);
-        $emailPlainText = strip_tags($emailContent ?? '');
+        $emailPlainText = $email['PlainContent'] ? $email['PlainContent'] : strip_tags($email['Content']);
         $emailPlainText = preg_replace("/\h+/", " ", $emailPlainText ?? '');
 
         Assert::assertStringContainsString($content, $emailPlainText);
@@ -170,7 +173,7 @@ class EmailContext implements Context
         $match = $this->mailer->findEmail($to, $from);
         Assert::assertNotNull($match);
 
-        $crawler = new Crawler($match->Content);
+        $crawler = new Crawler($match['Content']);
         $linkEl = $crawler->selectLink($linkSelector);
         Assert::assertNotNull($linkEl);
         $link = $linkEl->attr('href');
@@ -193,7 +196,7 @@ class EmailContext implements Context
         $match = $this->mailer->findEmail($to, $from, $title);
         Assert::assertNotNull($match);
 
-        $crawler = new Crawler($match->Content);
+        $crawler = new Crawler($match['Content']);
         $linkEl = $crawler->selectLink($linkSelector);
         Assert::assertNotNull($linkEl);
         $link = $linkEl->attr('href');
@@ -215,7 +218,7 @@ class EmailContext implements Context
         }
 
         $match = $this->lastMatchedEmail;
-        $crawler = new Crawler($match->Content);
+        $crawler = new Crawler($match['Content']);
         $linkEl = $crawler->selectLink($linkSelector);
         Assert::assertNotNull($linkEl);
         $link = $linkEl->attr('href');
@@ -250,10 +253,10 @@ class EmailContext implements Context
 
         $email = $this->lastMatchedEmail;
         $emailContent = null;
-        if ($email->Content) {
-            $emailContent = $email->Content;
+        if ($email['Content']) {
+            $emailContent = $email['Content'];
         } else {
-            $emailContent = $email->PlainContent;
+            $emailContent = $email['PlainContent'];
         }
         // Convert html content to plain text
         $emailContent = strip_tags($emailContent ?? '');
@@ -279,7 +282,7 @@ class EmailContext implements Context
      */
     public function thereIsAnEmailTitled($negate, $subject)
     {
-        $match = $this->mailer->findEmail(null, null, $subject);
+        $match = $this->mailer->findEmail('', null, $subject);
         if (trim($negate ?? '')) {
             Assert::assertNull($match);
         } else {
@@ -305,9 +308,9 @@ class EmailContext implements Context
 
         $match = $this->lastMatchedEmail;
         if (trim($negate ?? '')) {
-            Assert::assertStringNotContainsString($from, $match->From);
+            Assert::assertStringNotContainsString($from, $match['From']);
         } else {
-            Assert::assertStringContainsString($from, $match->From);
+            Assert::assertStringContainsString($from, $match['From']);
         }
     }
 
@@ -324,9 +327,9 @@ class EmailContext implements Context
 
         $match = $this->lastMatchedEmail;
         if (trim($negate ?? '')) {
-            Assert::assertStringNotContainsString($to, $match->To);
+            Assert::assertStringNotContainsString($to, $match['To']);
         } else {
-            Assert::assertStringContainsString($to, $match->To);
+            Assert::assertStringContainsString($to, $match['To']);
         }
     }
 
@@ -344,7 +347,7 @@ class EmailContext implements Context
         }
 
         $email = $this->lastMatchedEmail;
-        $html = $email->Content;
+        $html = $email['Content'];
         $dom = new \DOMDocument();
         $dom->loadHTML($html);
 

src/Context/FixtureContext.php

@@ -31,6 +31,8 @@ use SilverStripe\Security\Member;
 use SilverStripe\Security\Permission;
 use SilverStripe\Versioned\Versioned;
 use SilverStripe\Core\Config\Config;
+use SilverStripe\Security\PermissionRole;
+use SilverStripe\Security\PermissionRoleCode;
 
 /**
  * Context used to create fixtures in the SilverStripe ORM.
@@ -70,6 +72,11 @@ class FixtureContext implements Context
      */
     protected $activatedConfigFiles = array();
 
+    /**
+     * @var string[][] Tracks any extensions that have been added to classes
+     */
+    protected $addedExtensions = array();
+
     /**
      * @var array Stores the asset tuples.
      */
@@ -294,7 +301,6 @@ class FixtureContext implements Context
     {
 
         $class = $this->convertTypeToClass($type);
-        // TODO Support more than one record
         $fields = $this->convertFields($class, $fieldsTable->getRowsHash());
         $fields = $this->prepareFixture($class, $id, $fields);
 
@@ -517,7 +523,6 @@ class FixtureContext implements Context
         $yaml = implode("\n  ", $yaml);
 
         // Save fixtures into database
-        // TODO Run prepareAsset() for each File and Folder record
         $yamlFixture = new YamlFixture($yaml);
         $yamlFixture->writeInto($this->getFixtureFactory());
     }
@@ -617,6 +622,48 @@ class FixtureContext implements Context
         }
     }
 
+    /**
+     * Example: Given a "role" "Some role" with permissions "Access to 'Pages' section" and "Access to 'Files' section"
+     *
+     * @Given /^(?:an|a|the) "role" "([^"]+)" (?:with|has) permissions (.*)$/
+     * @param string $id
+     * @param string $permissionStr
+     */
+    public function stepCreateRoleWithPermissions($id, $permissionStr)
+    {
+        // Convert natural language permissions to codes
+        preg_match_all('/"([^"]+)"/', $permissionStr ?? '', $matches);
+        $permissions = $matches[1];
+        $codes = Permission::get_codes(false);
+
+        $role = $this->getFixtureFactory()->get(PermissionRole::class, $id);
+        if (!$role) {
+            $role = $this->getFixtureFactory()->createObject(PermissionRole::class, $id);
+        }
+
+        foreach ($permissions as $permission) {
+            $found = false;
+            foreach ($codes as $code => $details) {
+                if ($permission == $code
+                    || $permission == $details['name']
+                ) {
+                    $permissionRoleCode = PermissionRoleCode::create([
+                        'RoleID' => $role->ID,
+                        'Code' => $code,
+                    ]);
+                    $permissionRoleCode->write();
+                    $found = true;
+                }
+            }
+            if (!$found) {
+                throw new InvalidArgumentException(sprintf(
+                    'No permission found for "%s"',
+                    $permission
+                ));
+            }
+        }
+    }
+
     /**
      * Navigates to a record based on its identifier set during fixture creation,
      * using its RelativeLink() method to map the record to a URL.
@@ -671,6 +718,10 @@ class FixtureContext implements Context
             }
         }
         $targetClass::add_extension($extension);
+        if (!array_key_exists($targetClass, $this->addedExtensions)) {
+            $this->addedExtensions[$targetClass] = [];
+        }
+        $this->addedExtensions[$targetClass][] = $extension;
 
         // Write config for this extension too...
         $snakedExtension = strtolower(str_replace('\\', '-', $extension ?? '') ?? '');
@@ -804,6 +855,7 @@ YAML;
     public function afterResetConfig(AfterScenarioScope $event)
     {
         $this->clearConfigFiles();
+        $this->clearExtensions();
         // Flush
         $this->getMainContext()->visit('/?flush');
     }
@@ -895,6 +947,18 @@ YAML;
         return Injector::inst()->get(AssetStore::class);
     }
 
+    /**
+     * Selects the first image match in the HTML editor (tinymce)
+     *
+     * @When /^I select the image "([^"]+)" in the "([^"]+)" HTML field$/
+     * @param string $filename
+     * @param string $field
+     */
+    public function iSelectTheImageInHtmlField($filename, $field)
+    {
+        $this->selectInTheHtmlField("img[src*='$filename']", $field);
+    }
+
     /**
      * Selects the first match of $select in the given HTML editor (tinymce)
      */
@@ -922,18 +986,6 @@ YAML;
         $this->getMainContext()->getSession()->executeScript($js);
     }
 
-    /**
-     * Selects the first image match in the HTML editor (tinymce)
-     *
-     * @When /^I select the image "([^"]+)" in the "([^"]+)" HTML field$/
-     * @param string $filename
-     * @param string $field
-     */
-    public function iSelectTheImageInHtmlField($filename, $field)
-    {
-        $this->selectInTheHtmlField("img[src*='$filename']", $field);
-    }
-
     /**
      * Locate an HTML editor field
      *
@@ -1026,6 +1078,17 @@ YAML;
         return join('/', $paths);
     }
 
+    protected function clearExtensions()
+    {
+        foreach ($this->addedExtensions as $targetClass => $extensions) {
+            foreach ($extensions as $extension) {
+                /** @var Extensible $targetClass */
+                $targetClass::remove_extension($extension);
+            }
+        }
+        $this->addedExtensions = [];
+    }
+
     protected function clearConfigFiles()
     {
         // No files to cleanup

src/Context/LoginContext.php

@@ -242,7 +242,6 @@ class LoginContext implements Context
 
     /**
      * @Then /^the password for "([^"]*)" should be "([^"]*)"$/
-     * @skipUpgrade
      * @param string $id
      * @param string $password
      */

src/Context/SilverStripeContext.php

@@ -79,6 +79,10 @@ abstract class SilverStripeContext extends MinkContext implements SilverStripeAw
      */
     protected $xpathEscaper;
 
+    private int $screenWidth = 1024;
+
+    private int $screenHeight = 768;
+
     /**
      * Initializes context.
      * Every scenario gets it's own context object.
@@ -266,10 +270,10 @@ abstract class SilverStripeContext extends MinkContext implements SilverStripeAw
 
         if ($screenSize = Environment::getEnv('BEHAT_SCREEN_SIZE')) {
             list($screenWidth, $screenHeight) = explode('x', $screenSize ?? '');
-            $this->getSession()->resizeWindow((int)$screenWidth, (int)$screenHeight);
-        } else {
-            $this->getSession()->resizeWindow(1024, 768);
+            $this->screenWidth = (int)$screenWidth;
+            $this->screenHeight = (int)$screenHeight;
         }
+        $this->getSession()->resizeWindow($this->screenWidth, $this->screenHeight);
 
         // Reset everything
         foreach (ClassInfo::implementorsOf(Resettable::class) as $class) {
@@ -621,4 +625,49 @@ abstract class SilverStripeContext extends MinkContext implements SilverStripeAw
 EOS;
         $this->getSession()->getDriver()->executeScript($script);
     }
+
+    /**
+     * Set the screen to a specific width, using the default height.
+     *
+     * Example: Given I set the screen width to 500px
+     *
+     * @Given /^I set the screen width to ([\d]+)px$/
+     */
+    public function setScreenWidth($width)
+    {
+        $this->getSession()->resizeWindow($width, $this->screenHeight);
+    }
+
+    /**
+     * Set the screen to a specific height, using the default width.
+     * Example: Given I set the screen height to 500px
+     *
+     * @Given /^I set the screen height to ([\d]+)px$/
+     */
+    public function setScreenHeight($height)
+    {
+        $this->getSession()->resizeWindow($this->screenWidth, $height);
+    }
+
+    /**
+     * Set the screen to a specific width and height.
+     * Example: Given I set the screen size to 1024px by 768px
+     *
+     * @Given /^I set the screen size to ([\d]+)px by ([\d]+)px$/
+     */
+    public function setScreenSize($width, $height)
+    {
+        $this->getSession()->resizeWindow($width, $height);
+    }
+
+    /**
+     * Reset the screen size to what it was at the start of the test
+     * Example: Given I reset the screen size
+     *
+     * @Given /^I reset the screen size$/
+     */
+    public function resetScreenSize()
+    {
+        $this->getSession()->resizeWindow($this->screenWidth, $this->screenHeight);
+    }
 }

src/Controllers/ModuleSuiteLocator.php

@@ -186,7 +186,7 @@ class ModuleSuiteLocator implements Controller
         // Resolve variables
         $resolvedConfig = $this->container->getParameterBag()->resolveValue($suiteConfig);
         return [
-            'type' => null, // @todo figure out what this is for
+            'type' => null,
             'settings' => $resolvedConfig,
         ];
     }

src/Extension.php

@@ -16,8 +16,12 @@ use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
 use Behat\Testwork\ServiceContainer\ExtensionManager;
 use Behat\Testwork\ServiceContainer\Extension as ExtensionInterface;
 use RuntimeException;
+use Symfony\Component\Console\Input\ArgvInput;
 use Symfony\Component\DependencyInjection\Definition;
 use Symfony\Component\DependencyInjection\Reference;
+use Behat\Behat\Tester\ServiceContainer\TesterExtension;
+use SilverStripe\BehatExtension\Utility\RerunTotalStatistics;
+use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester;
 
 /*
  * This file is part of the SilverStripe\BehatExtension
@@ -42,7 +46,6 @@ class Extension implements ExtensionInterface
     */
     const SILVERSTRIPE_ID = 'silverstripe_extension';
 
-
     /**
     * {@inheritDoc}
     */
@@ -78,8 +81,10 @@ class Extension implements ExtensionInterface
     public function load(ContainerBuilder $container, array $config)
     {
         // Load yml config
-        $loader = new YamlFileLoader($container, new FileLocator(__DIR__ . '/../config'));
-        $loader->load('silverstripe.yml');
+        if ($this->getShouldBootstrap($container)) {
+            $loader = new YamlFileLoader($container, new FileLocator(__DIR__ . '/../config'));
+            $loader->load('silverstripe.yml');
+        }
 
         // Add CLI substitutions
         $this->loadSuiteLocator($container);
@@ -98,6 +103,22 @@ class Extension implements ExtensionInterface
              $container->setParameter('silverstripe_extension.region_map', $config['region_map']);
         }
         $container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
+        $container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);
+
+        // When running in CI, behat scenarios will occasionally sporadically fail
+        // Replaces services with custom implementations that will rerun failed features
+        // Note that features rather than scenarios need to be rerun to ensure that
+        // everything is setup and torn down correctly and that "Background" bits of
+        // feature fits are rerun
+        if ($config['is_ci']) {
+            $definition = new Definition(RerunRuntimeSuiteTester::class, array(
+                new Reference(TesterExtension::SPECIFICATION_TESTER_ID)
+            ));
+            $container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition);
+
+            $definition = new Definition(RerunTotalStatistics::class);
+            $container->setDefinition('output.pretty.statistics', $definition);
+        }
     }
 
     /**
@@ -105,8 +126,10 @@ class Extension implements ExtensionInterface
      */
     public function process(ContainerBuilder $container)
     {
-        $corePass = new Compiler\CoreInitializationPass();
-        $corePass->process($container);
+        if ($this->getShouldBootstrap($container)) {
+            $corePass = new Compiler\CoreInitializationPass();
+            $corePass->process($container);
+        }
     }
 
     public function configure(ArrayNodeDefinition $builder)
@@ -140,6 +163,9 @@ class Extension implements ExtensionInterface
                     info('Number of seconds that @retry tags will retry for')->
                     defaultValue(2)->
                 end()->
+                scalarNode('is_ci')->
+                    defaultValue(false)->
+                end()->
                 arrayNode('ajax_steps')->
                     defaultValue(array(
                         'go to',
@@ -203,4 +229,20 @@ class Extension implements ExtensionInterface
         $definition->addTag(CallExtension::CALL_HANDLER_TAG, ['priority' => 50]);
         $container->setDefinition(CallExtension::CALL_HANDLER_TAG . '.runtime', $definition);
     }
+
+    /**
+     * Check whether the extension should bootstrap or not.
+     * The extension should always bootstrap unless the `-h` or `--help` option is passed.
+     */
+    private function getShouldBootstrap(ContainerBuilder $container): bool
+    {
+        if (!$container->has('cli.input')) {
+            // If the input isn't there for some bizarre reason, just assume we should bootstrap.
+            return true;
+        }
+
+        /** @var ArgvInput $input */
+        $input = $container->get('cli.input');
+        return !$input->hasParameterOption(['--help', '-h']);
+    }
 }

src/Utility/RerunRuntimeSuiteTester.php

@@ -0,0 +1,82 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Testwork\Environment\Environment;
+use Behat\Testwork\Specification\SpecificationIterator;
+use Behat\Testwork\Tester\Result\IntegerTestResult;
+use Behat\Testwork\Tester\Result\TestResult;
+use Behat\Testwork\Tester\Result\TestResults;
+use Behat\Testwork\Tester\Result\TestWithSetupResult;
+use Behat\Testwork\Tester\Setup\SuccessfulSetup;
+use Behat\Testwork\Tester\Setup\SuccessfulTeardown;
+use Behat\Testwork\Tester\SpecificationTester;
+use Behat\Testwork\Tester\SuiteTester;
+
+/**
+ * Copy paste of Behat\Testwork\Tester\Runtime\RuntimeSuiteTester which is a final class
+ *
+ * Modified so that it reruns failed features
+ */
+class RerunRuntimeSuiteTester implements SuiteTester
+{
+    /**
+     * @var SpecificationTester
+     */
+    private $specTester;
+
+    /**
+     * Initializes tester.
+     *
+     * @param SpecificationTester $specTester
+     */
+    public function __construct(SpecificationTester $specTester)
+    {
+        $this->specTester = $specTester;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function setUp(Environment $env, SpecificationIterator $iterator, $skip)
+    {
+        return new SuccessfulSetup();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function test(Environment $env, SpecificationIterator $iterator, $skip = false)
+    {
+        $results = array();
+        foreach ($iterator as $specification) {
+            $setup = $this->specTester->setUp($env, $specification, $skip);
+            $localSkip = !$setup->isSuccessful() || $skip;
+            $testResult = $this->specTester->test($env, $specification, $localSkip);
+            $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
+
+            // start modifications here
+            if (!$testResult->isPassed()) {
+                file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL);
+                $setup = $this->specTester->setUp($env, $specification, $skip);
+                $localSkip = !$setup->isSuccessful() || $skip;
+                $testResult = $this->specTester->test($env, $specification, $localSkip);
+                $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
+            }
+            // end modifications here
+
+            $integerResult = new IntegerTestResult($testResult->getResultCode());
+            $results[] = new TestWithSetupResult($setup, $integerResult, $teardown);
+        }
+
+        return new TestResults($results);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result)
+    {
+        return new SuccessfulTeardown();
+    }
+}

src/Utility/RerunTotalStatistics.php

@@ -0,0 +1,314 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Behat\Tester\Result\StepResult;
+use Behat\Testwork\Counter\Memory;
+use Behat\Testwork\Counter\Timer;
+use Behat\Testwork\Tester\Result\TestResult;
+use Behat\Testwork\Tester\Result\TestResults;
+use Behat\Behat\Output\Statistics\Statistics;
+use Behat\Behat\Output\Statistics\ScenarioStat;
+use Behat\Behat\Output\Statistics\StepStat;
+use Behat\Behat\Output\Statistics\HookStat;
+
+/**
+ * Copy paste of Behat\Behat\Output\Statistics\TotalStatistics which is a final class
+ *
+ * Modified to remove duplicated stats from reruns
+ */
+class RerunTotalStatistics implements Statistics
+{
+    /**
+     * @var Timer
+     */
+    private $timer;
+    /**
+     * @var Memory
+     */
+    private $memory;
+    /**
+     * @var array
+     */
+    private $scenarioCounters = array();
+    /**
+     * @var array
+     */
+    private $stepCounters = array();
+    /**
+     * @var ScenarioStat[]
+     */
+    private $failedScenarioStats = array();
+    /**
+     * @var ScenarioStat[]
+     */
+    private $skippedScenarioStats = array();
+    /**
+     * @var StepStat[]
+     */
+    private $failedStepStats = array();
+    /**
+     * @var StepStat[]
+     */
+    private $pendingStepStats = array();
+    /**
+     * @var HookStat[]
+     */
+    private $failedHookStats = array();
+    // start modifications here
+    /**
+     * @var StepStat[]
+     */
+    private $passedStepStats = array();
+    // end modifications here
+
+    /**
+     * Initializes statistics.
+     */
+    public function __construct()
+    {
+        $this->resetAllCounters();
+
+        $this->timer = new Timer();
+        $this->memory = new Memory();
+    }
+
+    public function resetAllCounters()
+    {
+        $this->scenarioCounters = $this->stepCounters = array(
+            TestResult::PASSED    => 0,
+            TestResult::FAILED    => 0,
+            StepResult::UNDEFINED => 0,
+            TestResult::PENDING   => 0,
+            TestResult::SKIPPED   => 0
+        );
+    }
+
+    /**
+     * Starts timer.
+     */
+    public function startTimer()
+    {
+        $this->timer->start();
+    }
+
+    /**
+     * Stops timer.
+     */
+    public function stopTimer()
+    {
+        $this->timer->stop();
+    }
+
+    /**
+     * Returns timer object.
+     *
+     * @return Timer
+     */
+    public function getTimer()
+    {
+        return $this->timer;
+    }
+
+    /**
+     * Returns memory usage object.
+     *
+     * @return Memory
+     */
+    public function getMemory()
+    {
+        return $this->memory;
+    }
+
+    /**
+     * Registers scenario stat.
+     *
+     * @param ScenarioStat $stat
+     */
+    public function registerScenarioStat(ScenarioStat $stat)
+    {
+        if (TestResults::NO_TESTS === $stat->getResultCode()) {
+            return;
+        }
+
+        $this->scenarioCounters[$stat->getResultCode()]++;
+
+        // start modifications here
+        if (TestResult::FAILED === $stat->getResultCode()) {
+            // Ensure that any scenario reruns aren't counted as additional failures
+            $alreadyHasFailure = false;
+            foreach ($this->failedScenarioStats as $failedStat) {
+                if ($failedStat->getPath() === $stat->getPath()) {
+                    $alreadyHasFailure = true;
+                    break;
+                }
+            }
+            if (!$alreadyHasFailure) {
+                $this->failedScenarioStats[] = $stat;
+            } else {
+                $this->scenarioCounters[TestResult::FAILED]--;
+            }
+        }
+
+        if (TestResult::PASSED == $stat->getResultCode()) {
+            // Remove the scenario from the failed scenarios list if it passes on rerun
+            $newFailedScenarioStats = [];
+            foreach ($this->failedScenarioStats as $failedStat) {
+                if ($failedStat->getPath() !== $stat->getPath()) {
+                    $newFailedScenarioStats[] = $failedStat;
+                } else {
+                    $this->scenarioCounters[TestResult::FAILED]--;
+                }
+            }
+            $this->failedScenarioStats = $newFailedScenarioStats;
+        }
+        // end modifications here
+
+        if (TestResult::SKIPPED === $stat->getResultCode()) {
+            $this->skippedScenarioStats[] = $stat;
+        }
+    }
+
+    /**
+     * Registers step stat.
+     *
+     * @param StepStat $stat
+     */
+    public function registerStepStat(StepStat $stat)
+    {
+        $this->stepCounters[$stat->getResultCode()]++;
+
+        // start modifications here
+        if (TestResult::FAILED === $stat->getResultCode()) {
+            // Ensure that any scenario reruns don't double count step failures
+            $alreadyHasFailure = false;
+            foreach ($this->failedStepStats as $failedStat) {
+                if ($failedStat->getPath() === $stat->getPath()) {
+                    $alreadyHasFailure = true;
+                    break;
+                }
+            }
+            if (!$alreadyHasFailure) {
+                $this->failedStepStats[] = $stat;
+            } else {
+                $this->stepCounters[TestResult::FAILED]--;
+            }
+        }
+
+        if (TestResult::PASSED == $stat->getResultCode()) {
+            // Remove any duplicate passes on scenario rerun
+            $alreadyHasSuccess = false;
+            foreach ($this->passedStepStats as $passedStat) {
+                if ($passedStat->getPath() === $stat->getPath()) {
+                    $alreadyHasSuccess = true;
+                    break;
+                }
+            }
+            if (!$alreadyHasSuccess) {
+                $this->passedStepStats[] = $stat;
+            } else {
+                $this->stepCounters[TestResult::PASSED]--;
+            }
+
+            // Remove the step from the failed steps list if it passes on scenario rerun
+            $newFailedStepStats = [];
+            foreach ($this->failedStepStats as $failedStat) {
+                if ($failedStat->getPath() !== $stat->getPath()) {
+                    $newFailedStepStats[] = $failedStat;
+                } else {
+                    $this->stepCounters[TestResult::FAILED]--;
+                }
+            }
+            $this->failedStepStats = $newFailedStepStats;
+        }
+        // end modifications here
+
+        if (TestResult::PENDING === $stat->getResultCode()) {
+            $this->pendingStepStats[] = $stat;
+        }
+    }
+
+    /**
+     * Registers hook stat.
+     *
+     * @param HookStat $stat
+     */
+    public function registerHookStat(HookStat $stat)
+    {
+        if ($stat->isSuccessful()) {
+            return;
+        }
+
+        $this->failedHookStats[] = $stat;
+    }
+
+    /**
+     * Returns counters for different scenario result codes.
+     *
+     * @return array[]
+     */
+    public function getScenarioStatCounts()
+    {
+        return $this->scenarioCounters;
+    }
+
+    /**
+     * Returns skipped scenario stats.
+     *
+     * @return ScenarioStat[]
+     */
+    public function getSkippedScenarios()
+    {
+        return $this->skippedScenarioStats;
+    }
+
+    /**
+     * Returns failed scenario stats.
+     *
+     * @return ScenarioStat[]
+     */
+    public function getFailedScenarios()
+    {
+        return $this->failedScenarioStats;
+    }
+
+    /**
+     * Returns counters for different step result codes.
+     *
+     * @return array[]
+     */
+    public function getStepStatCounts()
+    {
+        return $this->stepCounters;
+    }
+
+    /**
+     * Returns failed step stats.
+     *
+     * @return StepStat[]
+     */
+    public function getFailedSteps()
+    {
+        return $this->failedStepStats;
+    }
+
+    /**
+     * Returns pending step stats.
+     *
+     * @return StepStat[]
+     */
+    public function getPendingSteps()
+    {
+        return $this->pendingStepStats;
+    }
+
+    /**
+     * Returns failed hook stats.
+     *
+     * @return HookStat[]
+     */
+    public function getFailedHookStats()
+    {
+        return $this->failedHookStats;
+    }
+}

src/Utility/TestMailer.php

@@ -2,8 +2,14 @@
 
 namespace SilverStripe\BehatExtension\Utility;
 
+use InvalidArgumentException;
+use SilverStripe\Control\Email\Email;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use SilverStripe\Dev\TestMailer as BaseTestMailer;
 use SilverStripe\TestSession\TestSessionEnvironment;
+use Symfony\Component\Mailer\Envelope;
+use Symfony\Component\Mailer\Transport\TransportInterface;
+use Symfony\Component\Mime\RawMessage;
 
 /**
  * Same principle as core TestMailer class,
@@ -17,15 +23,33 @@ class TestMailer extends BaseTestMailer
      */
     protected $testSessionEnvironment;
 
-    public function __construct()
-    {
+    public function __construct(
+        TransportInterface $transport,
+        EventDispatcherInterface $dispatcher
+    ) {
+        parent::__construct($transport, $dispatcher);
         $this->testSessionEnvironment = TestSessionEnvironment::singleton();
     }
 
+    public function send(RawMessage $message, Envelope $envelope = null): void
+    {
+        parent::send($message, $envelope);
+        /** @var Email $email */
+        $email = $message;
+        $data = $this->createData($email);
+        // save email to testsession state
+        $state = $this->testSessionEnvironment->getState();
+        if (!isset($state->emails)) {
+            $state->emails = array();
+        }
+        $state->emails[] = array_filter($data ?? []);
+        $this->testSessionEnvironment->applyState($state);
+    }
+
     /**
      * Clear the log of emails sent
      */
-    public function clearEmails()
+    public function clearEmails(): void
     {
         $state = $this->testSessionEnvironment->getState();
         if (isset($state->emails)) {
@@ -34,12 +58,17 @@ class TestMailer extends BaseTestMailer
         $this->testSessionEnvironment->applyState($state);
     }
 
-    public function findEmail($to = null, $from = null, $subject = null, $content = null)
-    {
+    public function findEmail(
+        string $to,
+        ?string $from = null,
+        ?string $subject = null,
+        ?string $content = null
+    ): ?array {
         $matches = $this->findEmails($to, $from, $subject, $content);
-                //got the count of matches emails
-                $emailCount = count($matches ?? []);
-                //get the last(latest) one
+        //got the count of matches emails
+        $emailCount = count($matches ?? []);
+        //get the last(latest) one
+
         return $matches ? $matches[$emailCount-1] : null;
     }
 
@@ -80,14 +109,14 @@ class TestMailer extends BaseTestMailer
                 }
             }
             if ($matched) {
-                $matches[] = $email;
+                $matches[] = json_decode(json_encode($email), true);
             }
         }
 
         return $matches;
     }
 
-    protected function saveEmail($data)
+    protected function saveEmail(array $data)
     {
         $state = $this->testSessionEnvironment->getState();
         if (!isset($state->emails)) {