[PLAT-8506] Local watchOS E2E testing

Makefile

@@ -137,6 +137,23 @@ e2e_macos:
 	./features/scripts/export_mac_app.sh
 	bundle exec maze-runner --app=macOSTestApp --os=macOS --os-version=11 $(FEATURES)
 
+.PHONY: e2e_watchos
+e2e_watchos: features/fixtures/watchos/Podfile.lock features/fixtures/shared/scenarios/watchos_maze_host.h
+	open --background features/fixtures/watchos/watchOSTestApp.xcworkspace
+ifneq ($(FEATURES),)
+	bundle exec maze-runner --os=watchos $(FEATURES)
+else
+	bundle exec maze-runner --os=watchos --tags @watchos
+endif
+
+features/fixtures/watchos/Podfile.lock: features/fixtures/watchos/Podfile
+	cd features/fixtures/watchos && pod install
+
+.PHONY: features/fixtures/shared/scenarios/watchos_maze_host.h
+features/fixtures/shared/scenarios/watchos_maze_host.h:
+	printf '#define WATCHOS_MAZE_HOST ' > $@
+	ruby -r socket -e 'p Socket.ip_address_list.select{ |a| a.ipv4_private? }[0].ip_address' >> $@
+
 #--------------------------------------------------------------------------
 # Release
 #

TESTING.md

@@ -92,9 +92,45 @@ Build the test iOS fixture:
     ```
    `<udid>` is the device Identifier found under Devices and Simulators in Xcode.
 
+### Running tests on Apple Watch
+
+#### Prerequisites
+
+1. The `xcdebug` command line tool - included in Xcode 13.4 and later.
+
+2. An Apple Watch paired to an iPhone that is connected via USB and visible in Xcode's devices window.
+
+3. The Apple Watch must be on the same WiFi network as the computer executing Maze runner.
+
+#### Running tests
+
+Due to device and tooling constraints, not all tests are suitable for running on Apple Watch.
+
+Tests that support Apple Watch are [tagged](https://cucumber.io/docs/cucumber/api/#tags) with `@watchos`.
+
+1. To run all supported tests:
+    ```shell script
+    make e2e_watchos
+    ```
+
+2. To run an individual test:
+    ```shell script
+    make e2e_watchos FEATURES=features/breadcrumbs.feature:59
+    ```
+
+#### Troubleshooting
+
+E2E testing for Apple Watch relies on the `xcdebug` to build and run the test fixture.
+
+`xcdebug` triggers actions in the Xcode IDE, but does not provide feedback about the success of those actions, so Maze runner will not be able to detect failures.
+
+Pay attention to Xcode's UI to see if there are any build or run failures.
+
+Ensure Xcode's run destination is set to the Apple Watch.
+
 ### Notes
 
 1. Maze Runner supports various other options, as well as all those that Cucumber does. For full details run:
     ```shell script
-    `bundle exec maze-runner --help`
+    bundle exec maze-runner --help
     ```

features/barebone_tests.feature

@@ -3,6 +3,7 @@ Feature: Barebone tests
   Background:
     Given I clear all persistent data
 
+  @watchos
   Scenario: Barebone test: handled errors
     When I run "BareboneTestHandledScenario"
     And I wait to receive a session
@@ -18,14 +19,16 @@ Feature: Barebone tests
     And the event "app.binaryArch" matches "(arm|x86)"
     And the event "app.bundleVersion" equals "12301"
     And the event "app.id" equals the platform-dependent string:
-      | ios   | com.bugsnag.iOSTestApp   |
-      | macos | com.bugsnag.macOSTestApp |
+      | ios     | com.bugsnag.iOSTestApp                                   |
+      | macos   | com.bugsnag.macOSTestApp                                 |
+      | watchos | com.bugsnag.watchOSTestApp.watchkitapp.watchkitextension |
     And the event "app.inForeground" is true
     And the event "app.isLaunching" is true
     And the event "app.releaseStage" equals "development"
     And the event "app.type" equals the platform-dependent string:
-      | ios   | iOS   |
-      | macos | macOS |
+      | ios     | iOS     |
+      | macos   | macOS   |
+      | watchos | watchOS |
     And the event "app.version" equals "12.3"
     And the event "breadcrumbs.0.name" equals "Running BareboneTestHandledScenario"
     And the event "breadcrumbs.1.name" equals "This is super <redacted>"
@@ -35,18 +38,20 @@ Feature: Barebone tests
     And the event "device.locale" is not null
     And the event "device.manufacturer" equals "Apple"
     And the event "device.modelNumber" equals the platform-dependent string:
-      | ios   | @not_null |
-      | macos | @null     |
+      | ios     | @not_null |
+      | macos   | @null     |
+      | watchos | @not_null |
     And on iOS, the event "device.orientation" matches "(face(down|up)|landscape(left|right)|portrait(upsidedown)?)"
     And the event "device.osName" equals the platform-dependent string:
-      | ios   | iOS    |
-      | macos | Mac OS |
+      | ios     | iOS     |
+      | macos   | Mac OS  |
+      | watchos | watchOS |
     And the event "device.osVersion" matches "\d+\.\d+"
     And the event "device.runtimeVersions.clangVersion" is not null
     And the event "device.runtimeVersions.osBuild" is not null
     And the event "device.time" is a timestamp
-    And on iOS, the event "metaData.device.batteryLevel" is a number
-    And on iOS, the event "metaData.device.charging" is a boolean
+    And on !macOS, the event "metaData.device.batteryLevel" is a number
+    And on !macOS, the event "metaData.device.charging" is a boolean
     And the event "metaData.device.simulator" is false
     And the event "metaData.device.timezone" is not null
     And the event "metaData.device.wordSize" is not null
@@ -114,6 +119,7 @@ Feature: Barebone tests
     And the "method" of stack frame 0 matches "BareboneTestHandledScenario"
     And the stacktrace is valid for the event
 
+  @watchos
   Scenario: Barebone test: unhandled error
     When I run "BareboneTestUnhandledErrorScenario" and relaunch the crashed app
     And I set the app to "report" mode
@@ -127,8 +133,9 @@ Feature: Barebone tests
     And the event "app.isLaunching" is true
     And the event "app.releaseStage" equals "development"
     And the event "app.type" equals the platform-dependent string:
-      | ios   | iOS   |
-      | macos | macOS |
+      | ios     | iOS     |
+      | macos   | macOS   |
+      | watchos | watchOS |
     And the event "app.version" equals "12.3"
     And the event "breadcrumbs.0.name" equals "Bugsnag loaded"
     And the event "breadcrumbs.1.name" is null
@@ -140,18 +147,16 @@ Feature: Barebone tests
     And the event "device.manufacturer" equals "Apple"
     And on iOS, the event "device.orientation" matches "(face(down|up)|landscape(left|right)|portrait(upsidedown)?)"
     And the event "device.osName" equals the platform-dependent string:
-      | ios   | iOS    |
-      | macos | Mac OS |
+      | ios     | iOS     |
+      | macos   | Mac OS  |
+      | watchos | watchOS |
     And the event "device.osVersion" matches "\d+\.\d+"
     And the event "device.runtimeVersions.clangVersion" is not null
     And the event "device.runtimeVersions.osBuild" is not null
     And the event "device.time" is a timestamp
-    And on iOS, the event "metaData.device.batteryLevel" is a number
-    And on iOS, the event "metaData.device.charging" is a boolean
+    And on !macOS, the event "metaData.device.batteryLevel" is a number
+    And on !macOS, the event "metaData.device.charging" is a boolean
     And the event "metaData.device.simulator" is false
-    And the event "metaData.error.mach.code_name" equals "KERN_INVALID_ADDRESS"
-    And the event "metaData.error.mach.code" equals "0x1"
-    And the event "metaData.error.mach.exception_name" is not null
     And the event "metaData.lastRunInfo.consecutiveLaunchCrashes" equals 1
     And the event "metaData.lastRunInfo.crashed" is true
     And the event "metaData.lastRunInfo.crashedDuringLaunch" is true
@@ -166,19 +171,19 @@ Feature: Barebone tests
     And the event "severity" equals "error"
     And the event "severityReason.type" equals "unhandledException"
     And the event "severityReason.unhandledOverridden" is null
-    And the event "threads.0.errorReportingThread" is true
-    And the event "threads.0.id" equals "0"
-    And the event "threads.0.state" is not null
-    And the event "threads.0.stacktrace.0.method" matches "(assertionFailure|fatalErrorMessage|<redacted>)"
+    And on !watchOS, the event "threads.0.errorReportingThread" is true
+    And on !watchOS, the event "threads.0.id" equals "0"
+    And on !watchOS, the event "threads.0.state" is not null
+    And on watchOS, the event "threads" is an array with 0 elements
     And the event "unhandled" is true
     And the event "user.email" equals "barfoo@example.com"
     And the event "user.id" equals "barfoo"
     And the event "user.name" equals "Bar Foo"
     And the event contains the following feature flags:
       | featureFlag | variant |
       | Testing     |         |
-    And the exception "errorClass" equals "Fatal error"
-    And the exception "message" equals "Unexpectedly found nil while implicitly unwrapping an Optional value"
+    And the exception "errorClass" equals "NSRangeException"
+    And the exception "message" equals "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray"
     And the exception "type" equals "cocoa"
     And the error payload field "events.0.app.dsymUUIDs" is a non-empty array
     And the error payload field "events.0.app.duration" is a number
@@ -187,9 +192,8 @@ Feature: Barebone tests
     And the error payload field "events.0.device.freeMemory" is an integer
     And the error payload field "events.0.device.model" matches the test device model
     And the error payload field "events.0.device.totalMemory" is an integer
-    And the error payload field "events.0.threads" is a non-empty array
-    And the error payload field "events.0.threads.1" is not null
-    And the "method" of stack frame 0 matches "(assertionFailure|fatalErrorMessage|<redacted>)"
+    And on !watchOS, the error payload field "events.0.threads" is a non-empty array
+    And on !watchOS, the error payload field "events.0.threads.1" is not null
     And the stacktrace is valid for the event
 
   @skip_macos

features/breadcrumbs.feature

@@ -55,6 +55,7 @@ Feature: Attaching a series of notable events leading up to errors
     And the event has a "state" breadcrumb named "Scene Will Enter Foreground"
     And the event has a "state" breadcrumb named "Scene Activated"
 
+  @watchos
   Scenario: Network breadcrumbs
     When I start the document server
     And I run "NetworkBreadcrumbsScenario"
@@ -63,7 +64,7 @@ Feature: Attaching a series of notable events leading up to errors
     And the event "breadcrumbs.0.name" equals "NSURLSession request failed"
     And the event "breadcrumbs.0.type" equals "request"
     And the event "breadcrumbs.0.metaData.method" equals "GET"
-    And the event "breadcrumbs.0.metaData.url" equals "http://bs-local.com:9340/reflect/"
+    And the event "breadcrumbs.0.metaData.url" matches "http://.*:9340/reflect/"
     And the event "breadcrumbs.0.metaData.urlParams.status" equals "444"
     And the event "breadcrumbs.0.metaData.urlParams.password" equals "[REDACTED]"
     And the event "breadcrumbs.0.metaData.status" equals 444
@@ -74,7 +75,7 @@ Feature: Attaching a series of notable events leading up to errors
     And the event "breadcrumbs.1.name" equals "NSURLSession request succeeded"
     And the event "breadcrumbs.1.type" equals "request"
     And the event "breadcrumbs.1.metaData.method" equals "GET"
-    And the event "breadcrumbs.1.metaData.url" equals "http://bs-local.com:9340/reflect/"
+    And the event "breadcrumbs.1.metaData.url" matches "http://.*:9340/reflect/"
     And the event "breadcrumbs.1.metaData.urlParams.delay_ms" equals "3000"
     And the event "breadcrumbs.1.metaData.status" equals 200
     And the event "breadcrumbs.1.metaData.duration" is greater than 0

features/delivery.feature

@@ -3,6 +3,7 @@ Feature: Delivery of errors
   Background:
     Given I clear all persistent data
 
+  @watchos
   Scenario: Delivery is retried after an HTTP 500 error
     When I set the HTTP status code for the next request to 500
     And I run "HandledExceptionScenario"

features/fixtures/shared/scenarios/.gitignore

@@ -0,0 +1 @@
+watchos_maze_host.h

features/fixtures/shared/scenarios/BareboneTestHandledScenario.swift

@@ -42,7 +42,9 @@ class BareboneTestHandledScenario: Scenario {
         config.addMetadata(["Testing": true], section: "Flags")
         config.addMetadata(["password": "123456"], section: "Other")
         config.launchDurationMillis = 0
+#if !os(watchOS)
         config.sendThreads = .unhandledOnly
+#endif
         config.setUser("foobar", withEmail: "foobar@example.com", andName: "Foo Bar")
         config.addMetadata(["group": "users"], section: "user")
         config.addFeatureFlag(name: "Testing")

features/fixtures/shared/scenarios/BareboneTestUnhandledErrorScenario.swift

@@ -30,8 +30,7 @@ class BareboneTestUnhandledErrorScenario: Scenario {
     }
 
     override func run() {
-        // Triggers "Fatal error: Unexpectedly found nil while implicitly unwrapping an Optional value: ..."
-        print(payload.name)
+        NSArray().object(at: 42)
     }
 }
 

features/fixtures/shared/scenarios/NetworkBreadcrumbsScenario.swift

@@ -10,30 +10,35 @@ import Foundation
 
 @available(iOS 10.0, macOS 10.12, *)
 class NetworkBreadcrumbsScenario : Scenario {
-
+
+    lazy var baseURL: URL = {
+        var components = URLComponents(string: Scenario.mazeRunnerURL.absoluteString)!
+        components.port = 9340 // `/reflect` listens on a different port :-((
+        return components.url!
+    }()
+
     override func startBugsnag() {
         config.autoTrackSessions = false;
         config.add(BugsnagNetworkRequestPlugin())
         config.addOnBreadcrumb {
-            return $0.type == .request &&
-                ($0.metadata["url"] as? String)?.hasPrefix("http://bs-local.com:9340") == true
+            ($0.metadata["url"] as? String ?? "").hasPrefix(self.baseURL.absoluteString)
         }
 
         super.startBugsnag()
     }
 
     override func run() {
         // Make some network requests so that automatic network breadcrumbs are left
-        query(address: "http://bs-local.com:9340/reflect/?status=444&password=T0p5ecr3t")
-        query(address: "http://bs-local.com:9340/reflect/?delay_ms=3000")
+        query(string: "/reflect/?status=444&password=T0p5ecr3t")
+        query(string: "/reflect/?delay_ms=3000")
 
         // Send a handled error
         let error = NSError(domain: "NetworkBreadcrumbsScenario", code: 100, userInfo: nil)
         Bugsnag.notifyError(error)
     }
 
-    func query(address: String) {
-        let url = URL(string: address)!
+    func query(string: String) {
+        let url = URL(string: string, relativeTo: baseURL)!
         let semaphore = DispatchSemaphore(value: 0)
 
         let task = URLSession.shared.dataTask(with: url) {(data, response, error) in

features/fixtures/shared/scenarios/Scenario.h

@@ -17,6 +17,8 @@ void markErrorHandledCallback(const BSG_KSCrashReportWriter *writer);
 
 @interface Scenario : NSObject
 
+@property (class, readonly) NSURL *mazeRunnerURL;  
+
 @property (strong, nonatomic, nonnull) BugsnagConfiguration *config;
 
 + (Scenario *)createScenarioNamed:(NSString *)className withConfig:(nullable BugsnagConfiguration *)config;
@@ -42,7 +44,7 @@ void markErrorHandledCallback(const BSG_KSCrashReportWriter *writer);
 
 + (void)clearPersistentData;
 
-+ (void)executeMazeRunnerCommand:(void (^)(NSString *action, NSString *scenarioName, NSString *scenarioMode))preHandler;
++ (void)executeMazeRunnerCommand:(nullable void (^)(NSString *action, NSString *scenarioName, NSString *scenarioMode))preHandler;
 
 @end