Retry failed tests in CI (#601)

* Retry failed tests in CI

* Only re-run specs in CI environment

Gemfile

@@ -108,6 +108,7 @@ group :test do
   gem "coveralls_reborn", require: false
   gem "rails-controller-testing"
   gem "rspec_junit_formatter"
+  gem "rspec-retry"
   gem "selenium-webdriver"
   gem "simplecov", require: false
   gem "webmock"

Gemfile.lock

@@ -372,6 +372,8 @@ GEM
       rspec-expectations (~> 3.12)
       rspec-mocks (~> 3.12)
       rspec-support (~> 3.12)
+    rspec-retry (0.6.2)
+      rspec-core (> 3.3)
     rspec-support (3.12.1)
     rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
@@ -543,6 +545,7 @@ DEPENDENCIES
   rails-controller-testing
   redis (~> 4.0)
   rspec-rails (~> 6.0.0)
+  rspec-retry
   rspec_junit_formatter
   selenium-webdriver
   sidekiq

spec/spec_helper.rb

@@ -11,6 +11,10 @@
                                                      ])
 end
 
+# Adds the ability to retry flaky tests.
+# See https://github.com/NoRedInk/rspec-retry
+require "rspec/retry"
+
 require "factory_bot"
 FactoryBot.find_definitions
 
@@ -102,6 +106,27 @@
   #   # the seed, which is printed after each run.
   #   #     --seed 1234
   config.order = :random
+
+  ### Re-trying flaky system specs
+  # show retry status in spec process
+  config.verbose_retry = true
+  # show exception that triggers a retry if verbose_retry is set to true
+  config.display_try_failure_messages = true
+
+  # run retry only on tests that use javascript and only in CI environment
+  if ENV["CI"]
+    config.around :each, :js do |ex|
+      ex.run_with_retry retry: 3
+    end
+  end
+
+  # callback to be run between retries
+  config.retry_callback = proc do |ex|
+    # run some additional clean up task - can be filtered by example metadata
+    if ex.metadata[:js]
+      Capybara.reset!
+    end
+  end
   #
   #   # Seed global randomization in this process using the `--seed` CLI option.
   #   # Setting this allows you to use `--seed` to deterministically reproduce