Use maze-runner to trigger stress test

.buildkite/pipeline.yml

@@ -205,8 +205,13 @@ steps:
     timeout_in_minutes: 3
     agents:
       queue: opensource-mac-cocoa
+    env:
+      STRESS_TEST: "true"
     commands:
-      - cd features/fixtures/macos-stress-test && make build run
+      - bundle install
+      - bundle exec maze-runner
+        features/stress_test.feature
+        --no-log-requests
     artifact_paths:
       - features/fixtures/macos-stress-test/BugsnagStressTest.stdout.log
       - features/fixtures/macos-stress-test/BugsnagStressTest.stderr.log

Gemfile

@@ -7,7 +7,7 @@ gem 'xcpretty'
 
 # A reference to Maze Runner is only needed for running tests locally and if committed it must be
 # portable for CI, e.g. a specific release.  However, leaving it commented out would mean quicker CI.
-gem 'bugsnag-maze-runner', git: 'https://github.com/bugsnag/maze-runner', tag: 'v4.11.1'
+gem 'bugsnag-maze-runner', git: 'https://github.com/bugsnag/maze-runner', tag: 'v4.13.0'
 
 # Locally, you can run against Maze Runner branches and uncommitted changes:
 # gem 'bugsnag-maze-runner', path: '../maze-runner'

Gemfile.lock

@@ -1,9 +1,9 @@
 GIT
   remote: https://github.com/bugsnag/maze-runner
-  revision: b5925737b0c3db49ac38a5a68e6bfe38026ef508
-  tag: v4.11.1
+  revision: 3161e2a7115a2642a1929aa5c037ead65d4ebfba
+  tag: v4.13.0
   specs:
-    bugsnag-maze-runner (4.11.1)
+    bugsnag-maze-runner (4.13.0)
       appium_lib (~> 11.2.0)
       boring (~> 0.1.0)
       cucumber (~> 3.1.2)
@@ -35,7 +35,7 @@ GEM
       appium_lib_core (~> 4.1)
       nokogiri (~> 1.8, >= 1.8.1)
       tomlrb (~> 1.1)
-    appium_lib_core (4.4.1)
+    appium_lib_core (4.5.0)
       faye-websocket (~> 0.11.0)
       selenium-webdriver (~> 3.14, >= 3.14.1)
     atomos (0.1.3)
@@ -170,7 +170,7 @@ GEM
     nap (1.1.0)
     netrc (0.11.0)
     no_proxy_fix (0.1.2)
-    nokogiri (1.11.1)
+    nokogiri (1.11.2)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     octokit (4.20.0)

features/fixtures/macos-stress-test/BugsnagStressTest/main.m

@@ -11,7 +11,7 @@
 #import <mach/task.h>
 #import <mach/task_info.h>
 
-static NSString * const kNotifyEndpoint = @"http://bs-local.com:9339/notify";
+static NSString * const kNotifyEndpoint = @"http://localhost:9339/notify";
 
 static const int kNumberOfIterations = 5000;
 

features/fixtures/macos-stress-test/Makefile

@@ -19,3 +19,4 @@ run:
 	rm -rf $(HOME)/Library/Application\ Support/com.bugsnag.Bugsnag
 	QUIET=true /usr/bin/time -l ./build/usr/local/bin/BugsnagStressTest
 	rm -rf $(HOME)/Library/Application\ Support/com.bugsnag.Bugsnag
+	echo "macOS stress-test complete"

features/stress_test.feature

@@ -0,0 +1,12 @@
+@stress_test
+Feature: Stress test
+
+  Scenario: Triggering error notifications continuously
+    When I start a new shell
+    And I input "cd features/fixtures/macos-stress-test" interactively
+    And I input "make build run" interactively
+    And I wait for the shell to output "macOS stress-test complete" to stdout
+    Then the last interactive command exited successfully
+
+    # This is low, but due to network congestion all we can guarantee in the course of a normal test
+    And I have received at least 1 error

features/support/env.rb

@@ -5,6 +5,14 @@
 
 AfterConfiguration do |_config|
   Maze.config.receive_no_requests_wait = 15
+
+  # Setup a 3 minute timeout for receiving requests is STRESS_TEST env var is set
+  Maze.config.receive_requests_wait = 180 unless ENV['STRESS_TEST'].nil?
+end
+
+# Skip stress tests unless STRESS_TEST env var is set
+Before('@stress_test') do |_scenario|
+  skip_this_scenario('Skipping: Run is not configured for stress tests') if ENV['STRESS_TEST'].nil?
 end
 
 # Additional require MacOS configuration