Local watchOS E2E testing

Makefile

@@ -137,6 +137,19 @@ e2e_macos:
 	./features/scripts/export_mac_app.sh
 	bundle exec maze-runner --app=macOSTestApp --os=macOS --os-version=11 $(FEATURES)
 
+.PHONY: e2e_watchos
+e2e_watchos: features/fixtures/watchos/Podfile.lock features/fixtures/shared/scenarios/watchos_maze_host.h
+	open --background features/fixtures/watchos/watchOSTestApp.xcworkspace
+	bundle exec maze-runner --os=watchos --tags @watchos
+
+features/fixtures/watchos/Podfile.lock: features/fixtures/watchos/Podfile
+	cd features/fixtures/watchos && pod install
+
+.PHONY: features/fixtures/shared/scenarios/watchos_maze_host.h
+features/fixtures/shared/scenarios/watchos_maze_host.h:
+	printf '#define WATCHOS_MAZE_HOST ' > $@
+	ruby -r socket -e 'p Socket.ip_address_list.select{ |a| a.ipv4_private? }[0].ip_address' >> $@
+
 #--------------------------------------------------------------------------
 # Release
 #

features/barebone_tests.feature

@@ -3,6 +3,7 @@ Feature: Barebone tests
   Background:
     Given I clear all persistent data
 
+  @watchos
   Scenario: Barebone test: handled errors
     When I run "BareboneTestHandledScenario"
     And I wait to receive a session
@@ -18,14 +19,16 @@ Feature: Barebone tests
     And the event "app.binaryArch" matches "(arm|x86)"
     And the event "app.bundleVersion" equals "12301"
     And the event "app.id" equals the platform-dependent string:
-      | ios   | com.bugsnag.iOSTestApp   |
-      | macos | com.bugsnag.macOSTestApp |
+      | ios     | com.bugsnag.iOSTestApp                                   |
+      | macos   | com.bugsnag.macOSTestApp                                 |
+      | watchos | com.bugsnag.watchOSTestApp.watchkitapp.watchkitextension |
     And the event "app.inForeground" is true
     And the event "app.isLaunching" is true
     And the event "app.releaseStage" equals "development"
     And the event "app.type" equals the platform-dependent string:
-      | ios   | iOS   |
-      | macos | macOS |
+      | ios     | iOS     |
+      | macos   | macOS   |
+      | watchos | watchOS |
     And the event "app.version" equals "12.3"
     And the event "breadcrumbs.0.name" equals "Running BareboneTestHandledScenario"
     And the event "breadcrumbs.1.name" equals "This is super <redacted>"
@@ -35,18 +38,20 @@ Feature: Barebone tests
     And the event "device.locale" is not null
     And the event "device.manufacturer" equals "Apple"
     And the event "device.modelNumber" equals the platform-dependent string:
-      | ios   | @not_null |
-      | macos | @null     |
+      | ios     | @not_null |
+      | macos   | @null     |
+      | watchos | @not_null |
     And on iOS, the event "device.orientation" matches "(face(down|up)|landscape(left|right)|portrait(upsidedown)?)"
     And the event "device.osName" equals the platform-dependent string:
-      | ios   | iOS    |
-      | macos | Mac OS |
+      | ios     | iOS     |
+      | macos   | Mac OS  |
+      | watchos | watchOS |
     And the event "device.osVersion" matches "\d+\.\d+"
     And the event "device.runtimeVersions.clangVersion" is not null
     And the event "device.runtimeVersions.osBuild" is not null
     And the event "device.time" is a timestamp
-    And on iOS, the event "metaData.device.batteryLevel" is a number
-    And on iOS, the event "metaData.device.charging" is a boolean
+    And on !macOS, the event "metaData.device.batteryLevel" is a number
+    And on !macOS, the event "metaData.device.charging" is a boolean
     And the event "metaData.device.simulator" is false
     And the event "metaData.device.timezone" is not null
     And the event "metaData.device.wordSize" is not null
@@ -114,6 +119,7 @@ Feature: Barebone tests
     And the "method" of stack frame 0 matches "BareboneTestHandledScenario"
     And the stacktrace is valid for the event
 
+  @watchos
   Scenario: Barebone test: unhandled error
     When I run "BareboneTestUnhandledErrorScenario" and relaunch the crashed app
     And I set the app to "report" mode
@@ -127,8 +133,9 @@ Feature: Barebone tests
     And the event "app.isLaunching" is true
     And the event "app.releaseStage" equals "development"
     And the event "app.type" equals the platform-dependent string:
-      | ios   | iOS   |
-      | macos | macOS |
+      | ios     | iOS     |
+      | macos   | macOS   |
+      | watchos | watchOS |
     And the event "app.version" equals "12.3"
     And the event "breadcrumbs.0.name" equals "Bugsnag loaded"
     And the event "breadcrumbs.1.name" is null
@@ -140,18 +147,16 @@ Feature: Barebone tests
     And the event "device.manufacturer" equals "Apple"
     And on iOS, the event "device.orientation" matches "(face(down|up)|landscape(left|right)|portrait(upsidedown)?)"
     And the event "device.osName" equals the platform-dependent string:
-      | ios   | iOS    |
-      | macos | Mac OS |
+      | ios     | iOS     |
+      | macos   | Mac OS  |
+      | watchos | watchOS |
     And the event "device.osVersion" matches "\d+\.\d+"
     And the event "device.runtimeVersions.clangVersion" is not null
     And the event "device.runtimeVersions.osBuild" is not null
     And the event "device.time" is a timestamp
-    And on iOS, the event "metaData.device.batteryLevel" is a number
-    And on iOS, the event "metaData.device.charging" is a boolean
+    And on !macOS, the event "metaData.device.batteryLevel" is a number
+    And on !macOS, the event "metaData.device.charging" is a boolean
     And the event "metaData.device.simulator" is false
-    And the event "metaData.error.mach.code_name" equals "KERN_INVALID_ADDRESS"
-    And the event "metaData.error.mach.code" equals "0x1"
-    And the event "metaData.error.mach.exception_name" is not null
     And the event "metaData.lastRunInfo.consecutiveLaunchCrashes" equals 1
     And the event "metaData.lastRunInfo.crashed" is true
     And the event "metaData.lastRunInfo.crashedDuringLaunch" is true
@@ -166,19 +171,19 @@ Feature: Barebone tests
     And the event "severity" equals "error"
     And the event "severityReason.type" equals "unhandledException"
     And the event "severityReason.unhandledOverridden" is null
-    And the event "threads.0.errorReportingThread" is true
-    And the event "threads.0.id" equals "0"
-    And the event "threads.0.state" is not null
-    And the event "threads.0.stacktrace.0.method" matches "(assertionFailure|fatalErrorMessage|<redacted>)"
+    And on !watchOS, the event "threads.0.errorReportingThread" is true
+    And on !watchOS, the event "threads.0.id" equals "0"
+    And on !watchOS, the event "threads.0.state" is not null
+    And on watchOS, the event "threads" is an array with 0 elements
     And the event "unhandled" is true
     And the event "user.email" equals "barfoo@example.com"
     And the event "user.id" equals "barfoo"
     And the event "user.name" equals "Bar Foo"
     And the event contains the following feature flags:
       | featureFlag | variant |
       | Testing     |         |
-    And the exception "errorClass" equals "Fatal error"
-    And the exception "message" equals "Unexpectedly found nil while implicitly unwrapping an Optional value"
+    And the exception "errorClass" equals "NSRangeException"
+    And the exception "message" equals "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray"
     And the exception "type" equals "cocoa"
     And the error payload field "events.0.app.dsymUUIDs" is a non-empty array
     And the error payload field "events.0.app.duration" is a number
@@ -187,9 +192,8 @@ Feature: Barebone tests
     And the error payload field "events.0.device.freeMemory" is an integer
     And the error payload field "events.0.device.model" matches the test device model
     And the error payload field "events.0.device.totalMemory" is an integer
-    And the error payload field "events.0.threads" is a non-empty array
-    And the error payload field "events.0.threads.1" is not null
-    And the "method" of stack frame 0 matches "(assertionFailure|fatalErrorMessage|<redacted>)"
+    And on !watchOS, the error payload field "events.0.threads" is a non-empty array
+    And on !watchOS, the error payload field "events.0.threads.1" is not null
     And the stacktrace is valid for the event
 
   @skip_macos

features/fixtures/shared/scenarios/.gitignore

@@ -0,0 +1 @@
+watchos_maze_host.h

features/fixtures/shared/scenarios/BareboneTestHandledScenario.swift

@@ -42,7 +42,9 @@ class BareboneTestHandledScenario: Scenario {
         config.addMetadata(["Testing": true], section: "Flags")
         config.addMetadata(["password": "123456"], section: "Other")
         config.launchDurationMillis = 0
+#if !os(watchOS)
         config.sendThreads = .unhandledOnly
+#endif
         config.setUser("foobar", withEmail: "foobar@example.com", andName: "Foo Bar")
         config.addMetadata(["group": "users"], section: "user")
         config.addFeatureFlag(name: "Testing")

features/fixtures/shared/scenarios/BareboneTestUnhandledErrorScenario.swift

@@ -30,8 +30,7 @@ class BareboneTestUnhandledErrorScenario: Scenario {
     }
 
     override func run() {
-        // Triggers "Fatal error: Unexpectedly found nil while implicitly unwrapping an Optional value: ..."
-        print(payload.name)
+        NSArray().object(at: 42)
     }
 }
 

features/fixtures/shared/scenarios/Scenario.h

@@ -42,7 +42,7 @@ void markErrorHandledCallback(const BSG_KSCrashReportWriter *writer);
 
 + (void)clearPersistentData;
 
-+ (void)executeMazeRunnerCommand:(void (^)(NSString *action, NSString *scenarioName, NSString *scenarioMode))preHandler;
++ (void)executeMazeRunnerCommand:(nullable void (^)(NSString *action, NSString *scenarioName, NSString *scenarioMode))preHandler;
 
 @end
 

features/fixtures/shared/scenarios/Scenario.m

@@ -6,6 +6,20 @@
 
 #import <objc/runtime.h>
 
+#if TARGET_OS_IOS
+#define BASE_URL "http://bs-local.com:9339"
+#define SWIFT_MODULE "iOSTestApp"
+#elif TARGET_OS_OSX
+#define BASE_URL "http://localhost:9339"
+#define SWIFT_MODULE "macOSTestApp"
+#elif TARGET_OS_WATCH
+#import "watchos_maze_host.h"
+#define BASE_URL "http://" WATCHOS_MAZE_HOST ":9339"
+#define SWIFT_MODULE "watchOSTestApp_WatchKit_Extension"
+#else
+#error Unsupported TARGET_OS
+#endif
+
 extern bool bsg_kslog_setLogFilename(const char *filename, bool overwrite);
 
 extern void bsg_i_kslog_logCBasic(const char *fmt, ...) __printflike(1, 2);
@@ -22,7 +36,9 @@ void markErrorHandledCallback(const BSG_KSCrashReportWriter *writer) {
 
 static Scenario *theScenario;
 
+#if !TARGET_OS_WATCH
 static char ksLogPath[PATH_MAX];
+#endif
 
 @implementation Scenario {
     dispatch_block_t _onEventDelivery;
@@ -57,8 +73,7 @@ + (void)load {
 
 + (Scenario *)createScenarioNamed:(NSString *)className withConfig:(BugsnagConfiguration *)config {
     Class class = NSClassFromString(className) ?:
-    NSClassFromString([@"iOSTestApp." stringByAppendingString:className]) ?:
-    NSClassFromString([@"macOSTestApp." stringByAppendingString:className]);
+    NSClassFromString([@SWIFT_MODULE "." stringByAppendingString:className]);
 
     if (!class) {
         [NSException raise:NSInvalidArgumentException format:@"Failed to find scenario class named %@", className];
@@ -77,10 +92,12 @@ - (instancetype)initWithConfig:(BugsnagConfiguration *)config {
             _config = config;
         } else {
             _config = [[BugsnagConfiguration alloc] initWithApiKey:@"12312312312312312312312312312312"];
-            _config.endpoints.notify = @"http://bs-local.com:9339/notify";
-            _config.endpoints.sessions = @"http://bs-local.com:9339/sessions";
+            _config.endpoints.notify = @BASE_URL "/notify";
+            _config.endpoints.sessions = @BASE_URL "/sessions";
         }
+#if !TARGET_OS_WATCH
         _config.enabledErrorTypes.ooms = NO;
+#endif
     }
     return self;
 }
@@ -144,6 +161,7 @@ - (void)requestDidComplete:(NSURLRequest *)request {
 
 + (void)initialize {
     if (self == [Scenario self]) {
+#if !TARGET_OS_WATCH
 #if TARGET_OS_IPHONE
         NSString *logPath = [NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES)[0]
                              stringByAppendingPathComponent:@"kscrash.log"];
@@ -152,7 +170,7 @@ + (void)initialize {
 #endif
         [logPath getFileSystemRepresentation:ksLogPath maxLength:sizeof(ksLogPath)];
         bsg_kslog_setLogFilename(ksLogPath, false);
-        
+#endif
         Method method = class_getInstanceMethod([NSURLSession class], @selector(uploadTaskWithRequest:fromData:completionHandler:));
         NSURLSession_uploadTaskWithRequest_fromData_completionHandler =
         (void *)method_setImplementation(method, (void *)uploadTaskWithRequest_fromData_completionHandler);
@@ -188,14 +206,16 @@ + (void)clearPersistentData {
             NSLog(@"%@", error);
         }
     }
+#if !TARGET_OS_WATCH
     bsg_kslog_setLogFilename(ksLogPath, true);
+#endif
 }
 
 + (void)executeMazeRunnerCommand:(void (^)(NSString *action, NSString *scenarioName, NSString *scenarioMode))preHandler {
     NSLog(@"%s", __PRETTY_FUNCTION__);
     NSURLSession *session = [NSURLSession sessionWithConfiguration:[NSURLSessionConfiguration ephemeralSessionConfiguration]];
 
-    NSURLRequest *request = [NSURLRequest requestWithURL:[NSURL URLWithString:@"http://bs-local.com:9339/command"]];
+    NSURLRequest *request = [NSURLRequest requestWithURL:[NSURL URLWithString:@BASE_URL "/command"]];
     [[session dataTaskWithRequest:request completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
         if (![response isKindOfClass:[NSHTTPURLResponse class]] || [(NSHTTPURLResponse *)response statusCode] != 200) {
             NSLog(@"%s request failed with %@", __PRETTY_FUNCTION__, response ?: error);
@@ -219,9 +239,11 @@ + (void)executeMazeRunnerCommand:(void (^)(NSString *action, NSString *scenarioN
             [self clearPersistentData];
         }
 
-        dispatch_async(dispatch_get_main_queue(), ^{
-            preHandler(action, scenarioName, eventMode);
-        });
+        if (preHandler) {
+            dispatch_async(dispatch_get_main_queue(), ^{
+                preHandler(action, scenarioName, eventMode);
+            });
+        }
 
         dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
             if ([action isEqualToString:@"run_scenario"]) {

features/fixtures/watchos/Podfile

@@ -0,0 +1,13 @@
+platform :watchos, '6.3'
+
+# target 'watchOSTestApp' do
+# end
+
+# target 'watchOSTestApp WatchKit App' do
+# end
+
+target 'watchOSTestApp WatchKit Extension' do
+  use_frameworks!
+  pod 'Bugsnag', path: '../../..'
+  pod 'BugsnagNetworkRequestPlugin', path: '../../..'
+end

features/fixtures/watchos/watchOSTestApp WatchKit App/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}