Merge pull request #174 from bugsnag/PLAT-7734/mazerunner-v6

Update tests to use mazerunner v6

.buildkite/pipeline.yml

@@ -21,11 +21,19 @@ steps:
           run: java-common
     command: './gradlew check test'
 
-  - label: ':docker: Run mazerunner tests'
-    key: 'java-mazerunner-tests'
-    timeout_in_minutes: 60
+  - label: ':docker: Mazerunner tests batch 1'
+    key: 'java-mazerunner-tests-1'
+    timeout_in_minutes: 30
+    plugins:
+      - docker-compose#v3.7.0:
+          run: java-mazerunner
+    command: 'bundle exec maze-runner --exclude=features/[^a-m].*.feature'
+
+  - label: ':docker: Mazerunner tests batch 2'
+    key: 'java-mazerunner-tests-2'
+    timeout_in_minutes: 30
     plugins:
       - docker-compose#v3.7.0:
           pull: java-mazerunner
           run: java-mazerunner
-    command: 'bundle exec bugsnag-maze-runner --verbose'
+    command: 'bundle exec maze-runner --exclude=features/[^n-z].*.feature'

Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-gem 'bugsnag-maze-runner', :git => 'https://github.com/bugsnag/maze-runner', :branch => 'v1'
+gem 'bugsnag-maze-runner', git: 'https://github.com/bugsnag/maze-runner', tag: 'v6.8.0'
 gem 'os'

Gemfile.lock

@@ -1,48 +1,102 @@
 GIT
   remote: https://github.com/bugsnag/maze-runner
-  revision: ba232513f88c18149ab5b4595c9f5a81174bdee5
+  revision: fe12189f83aad154f54221ee0fcd41b483d3c0d1
+  tag: v6.8.0
   specs:
-    bugsnag-maze-runner (1.0.0)
-      cucumber (~> 3.1.0)
-      cucumber-expressions (= 5.0.15)
-      minitest (~> 5.0)
+    bugsnag-maze-runner (6.8.0)
+      appium_lib (~> 11.2.0)
+      bugsnag (~> 6.24)
+      cucumber (~> 7.1)
+      cucumber-expressions (~> 6.0.0)
+      curb (~> 0.9.6)
+      optimist (~> 3.0.1)
       os (~> 1.0.0)
-      rack (~> 2.0.0)
-      rake (~> 12.3.0)
-      test-unit (~> 3.2.0)
+      rake (~> 12.3.3)
+      rubyzip (~> 2.3.2)
+      selenium-webdriver (~> 3.11)
+      test-unit (~> 3.5.2)
+      webrick (~> 1.7.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    backports (3.11.4)
-    builder (3.2.3)
-    cucumber (3.1.0)
-      builder (>= 2.1.2)
-      cucumber-core (~> 3.1.0)
-      cucumber-expressions (~> 5.0.4)
-      cucumber-wire (~> 0.0.1)
-      diff-lcs (~> 1.3)
-      gherkin (~> 5.0)
-      multi_json (>= 1.7.5, < 2.0)
-      multi_test (>= 0.1.2)
-    cucumber-core (3.1.0)
-      backports (>= 3.8.0)
-      cucumber-tag_expressions (~> 1.1.0)
-      gherkin (>= 5.0.0)
-    cucumber-expressions (5.0.15)
-    cucumber-tag_expressions (1.1.1)
-    cucumber-wire (0.0.1)
-    diff-lcs (1.3)
-    gherkin (5.1.0)
-    minitest (5.11.3)
-    multi_json (1.13.1)
+    appium_lib (11.2.0)
+      appium_lib_core (~> 4.1)
+      nokogiri (~> 1.8, >= 1.8.1)
+      tomlrb (~> 1.1)
+    appium_lib_core (4.7.1)
+      faye-websocket (~> 0.11.0)
+      selenium-webdriver (~> 3.14, >= 3.14.1)
+    bugsnag (6.24.1)
+      concurrent-ruby (~> 1.0)
+    builder (3.2.4)
+    childprocess (3.0.0)
+    concurrent-ruby (1.1.9)
+    cucumber (7.1.0)
+      builder (~> 3.2, >= 3.2.4)
+      cucumber-core (~> 10.1, >= 10.1.0)
+      cucumber-create-meta (~> 6.0, >= 6.0.1)
+      cucumber-cucumber-expressions (~> 14.0, >= 14.0.0)
+      cucumber-gherkin (~> 22.0, >= 22.0.0)
+      cucumber-html-formatter (~> 17.0, >= 17.0.0)
+      cucumber-messages (~> 17.1, >= 17.1.1)
+      cucumber-wire (~> 6.2, >= 6.2.0)
+      diff-lcs (~> 1.4, >= 1.4.4)
+      mime-types (~> 3.3, >= 3.3.1)
+      multi_test (~> 0.1, >= 0.1.2)
+      sys-uname (~> 1.2, >= 1.2.2)
+    cucumber-core (10.1.0)
+      cucumber-gherkin (~> 22.0, >= 22.0.0)
+      cucumber-messages (~> 17.1, >= 17.1.1)
+      cucumber-tag-expressions (~> 4.0, >= 4.0.2)
+    cucumber-create-meta (6.0.4)
+      cucumber-messages (~> 17.1, >= 17.1.1)
+      sys-uname (~> 1.2, >= 1.2.2)
+    cucumber-cucumber-expressions (14.0.0)
+    cucumber-expressions (6.0.1)
+    cucumber-gherkin (22.0.0)
+      cucumber-messages (~> 17.1, >= 17.1.1)
+    cucumber-html-formatter (17.0.0)
+      cucumber-messages (~> 17.1, >= 17.1.0)
+    cucumber-messages (17.1.1)
+    cucumber-tag-expressions (4.1.0)
+    cucumber-wire (6.2.0)
+      cucumber-core (~> 10.1, >= 10.1.0)
+      cucumber-cucumber-expressions (~> 14.0, >= 14.0.0)
+      cucumber-messages (~> 17.1, >= 17.1.1)
+    curb (0.9.11)
+    diff-lcs (1.5.0)
+    eventmachine (1.2.7)
+    faye-websocket (0.11.1)
+      eventmachine (>= 0.12.0)
+      websocket-driver (>= 0.5.1)
+    ffi (1.15.4)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.1115)
+    mini_portile2 (2.6.1)
     multi_test (0.1.2)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
+      racc (~> 1.4)
+    optimist (3.0.1)
     os (1.0.0)
-    power_assert (1.1.3)
-    rack (2.0.6)
-    rake (12.3.1)
-    test-unit (3.2.8)
+    power_assert (2.0.1)
+    racc (1.6.0)
+    rake (12.3.3)
+    rubyzip (2.3.2)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
+    sys-uname (1.2.2)
+      ffi (~> 1.1)
+    test-unit (3.5.3)
       power_assert
+    tomlrb (1.3.0)
+    webrick (1.7.0)
+    websocket-driver (0.7.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
 
 PLATFORMS
   ruby

dockerfiles/Dockerfile.java-mazerunner

@@ -1,8 +1,8 @@
-FROM tomcat:7.0
+FROM tomcat:9.0.56-jdk8
 WORKDIR /app
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
-    apt-get install -y -q docker-compose bundler
+    apt-get install -y -q docker-compose bundler libcurl4-openssl-dev
 
 # Force download of gradle zip early to avoid repeating
 # if Docker cache is invalidated by branch changes.
@@ -16,5 +16,5 @@ RUN ./gradlew
 COPY . /app
 
 # Setup mazerunner
-RUN bundle update --bundler
+RUN gem install bundler:1.16.5
 RUN bundle install

features/app_type.feature

@@ -2,31 +2,32 @@ Feature: Reporting app version
 
 Scenario: Test logback appender with app type 'testAppType'
     When I run "LogbackScenario" with logback config "app_type_config.xml"
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Java" notifier
+    And the error payload field "events" is an array with 1 elements
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "LogbackScenario"
     And the event "app.type" equals "testAppType"
 
 Scenario: Test Java app with app type 'testAppType'
     When I run "AppTypeScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Java" notifier
     And the exception "message" equals "AppTypeScenario"
     And the event "app.type" equals "testAppType"
 
 Scenario: Test Spring Boot app with app type 'testAppType'
     When I run spring boot "AppTypeScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
     And the exception "message" equals "AppTypeScenario"
     And the event "app.type" equals "testAppType"
 
 Scenario: Test Spring app with app type 'testAppType'
-    When I run plain Spring "AppTypeScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
+    When I run the plain spring app
+    And I wait for 20 seconds
+    And I navigate to the route "/run-scenario/AppTypeScenario" on port "8080"
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
     And the exception "message" equals "AppTypeScenario"
     And the event "app.type" equals "testAppType"

features/app_version.feature

@@ -2,34 +2,33 @@ Feature: Reporting app version
 
 Scenario: Test Java handled Exception
     When I run "AppVersionScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Java" notifier
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "AppVersionScenario"
     And the event "app.version" equals "1.2.3.abc"
 
 Scenario: Test Spring Boot handled Exception
     When I run spring boot "AppVersionScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "AppVersionScenario"
     And the event "app.version" equals "1.2.3.abc"
 
 Scenario: Test plain Spring handled Exception
     When I run plain Spring "AppVersionScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "AppVersionScenario"
     And the event "app.version" equals "1.2.3.abc"
 
 Scenario: Test logback appender with app version 1.2.3
     When I run "LogbackScenario" with logback config "app_version_config.xml"
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Java" notifier
+    And the error payload field "events" is an array with 1 elements
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "LogbackScenario"
     And the event "app.version" equals "1.2.3"

features/async_method.feature

@@ -2,11 +2,9 @@ Feature: Reporting Unhandled exceptions in an async method
 
 Scenario: Report an exception from a spring boot async method
     When I run spring boot "AsyncMethodScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the request used the Spring notifier
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
+    And the error payload field "events" is an array with 1 elements
     And the event "unhandled" is true
     And the event "severity" equals "error"
     And the exception "errorClass" equals "java.lang.RuntimeException"
@@ -16,27 +14,26 @@ Scenario: Report an exception from a spring boot async method
 
 Scenario: Notify an exception from a spring boot async method
     When I run spring boot "AsyncNotifyScenario" with the defaults
-    Then I should receive 2 requests
-    And the request 0 is a valid for the error reporting API
-    And the exception "errorClass" equals "java.lang.RuntimeException" for request 0
-    And the exception "message" equals "test from before async" for request 0
-    And the event "metaData.thread.controllerMethod" equals "meta data from controller method" for request 0
-    And the event "metaData.thread.inAsyncMethod" is null for request 0
+    Then I wait to receive 2 errors
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
+    And the exception "errorClass" equals "java.lang.RuntimeException"
+    And the exception "message" equals "test from before async"
+    And the event "metaData.thread.controllerMethod" equals "meta data from controller method"
+    And the event "metaData.thread.inAsyncMethod" is null
+    Then I discard the oldest error
 
-    And the request 1 is a valid for the error reporting API
-    And the exception "errorClass" equals "java.lang.RuntimeException" for request 1
-    And the exception "message" equals "test from async" for request 1
-    And the event "metaData.thread.controllerMethod" is null for request 1
-    And the event "metaData.thread.inAsyncMethod" equals "meta data from async method" for request 1
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
+    And the exception "errorClass" equals "java.lang.RuntimeException"
+    And the exception "message" equals "test from async"
+    And the event "metaData.thread.controllerMethod" is null
+    And the event "metaData.thread.inAsyncMethod" equals "meta data from async method"
 
 Scenario: Report an exception from a plain spring async method
     Given I run the plain spring app
     When I navigate to the route "/run-async-task" on port "8080"
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the request used the Spring notifier
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
+    And the error payload field "events" is an array with 1 elements
     And the event "unhandled" is true
     And the event "severity" equals "error"
     And the exception "errorClass" equals "java.lang.RuntimeException"

features/crash_handler.feature

@@ -2,27 +2,24 @@ Feature: Reporting with other exception handlers installed
 
 Scenario: Other uncaught exception handler installed in Java app
     When I run "CrashHandlerScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Java" notifier
+    And the error payload field "events" is an array with 1 elements
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "CrashHandlerScenario"
 
 Scenario: Other uncaught exception handler installed in Spring Boot app
     When I run spring boot "CrashHandlerScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
+    And the error payload field "events" is an array with 1 elements
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "CrashHandlerScenario"
 
 Scenario: Other uncaught exception handler installed in Spring app
     When I run plain Spring "CrashHandlerScenario" with the defaults
-    Then I should receive a request
-    And the request is a valid for the error reporting API
-    And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
-    And the payload field "events" is an array with 1 element
+    And I wait to receive an error
+    And the error is valid for the error reporting API version "4" for the "Bugsnag Spring" notifier
+    And the error payload field "events" is an array with 1 elements
     And the exception "errorClass" equals "java.lang.RuntimeException"
     And the exception "message" equals "CrashHandlerScenario"