Merge pull request #1587 from bugsnag/robert/plat-10987_ios_17_testing

iOS 17 testing

.buildkite/pipeline.yml

@@ -11,7 +11,9 @@ steps:
     key: cocoa_fixture
     timeout_in_minutes: 30
     agents:
-      queue: macos-12-arm
+      queue: macos-13-arm
+    env:
+      DEVELOPER_DIR: /Applications/Xcode14.3.app
     artifact_paths:
       - features/fixtures/ios/output/iOSTestApp.ipa
       - features/fixtures/macos/output/macOSTestApp.zip
@@ -26,15 +28,15 @@ steps:
   - label: Static framework and Swift Package Manager builds
     timeout_in_minutes: 10
     agents:
-      queue: macos-12-arm
+      queue: macos-13-arm
     commands:
       - make build_swift
       - make build_ios_static
 
   - label: Carthage
     timeout_in_minutes: 15
     agents:
-      queue: macos-12-arm
+      queue: macos-13-arm
     commands:
       - ./scripts/build-carthage.sh
     plugins:
@@ -91,6 +93,15 @@ steps:
     artifact_paths:
       - logs/*
 
+  - label: iOS 17 unit tests
+    timeout_in_minutes: 10
+    agents:
+      queue: macos-13-arm
+    commands:
+      - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=17.0
+    artifact_paths:
+      - logs/*
+
   - label: iOS 15 unit tests
     timeout_in_minutes: 10
     agents:
@@ -467,6 +478,68 @@ steps:
   #
   # BrowserStack
   #
+  - label: ':browserstack: iOS 17 E2E tests batch 1'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 60
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        service-ports: true
+        command:
+          - --app=@/app/build/ipa_url_bs.txt
+          - --farm=bs
+          - --device=IOS_17
+          - --appium-version=1.18.0
+          - --a11y-locator
+          - --fail-fast
+          - --exclude=features/[e-z].*.feature$
+          - --order=random
+    concurrency: 25
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  - label: ':browserstack: iOS 17 E2E tests batch 2'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 60
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        service-ports: true
+        command:
+          - --app=@/app/build/ipa_url_bs.txt
+          - --farm=bs
+          - --device=IOS_17
+          - --appium-version=1.18.0
+          - --a11y-locator
+          - --fail-fast
+          - --exclude=features/[a-d].*.feature$
+          - --order=random
+    concurrency: 25
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
   - label: ':browserstack: iOS 11 barebone tests'
     skip: "https://smartbear.atlassian.net/browse/PLAT-11154"
     depends_on:

BugsnagNetworkRequestPlugin/BugsnagNetworkRequestPlugin.xcodeproj/project.pbxproj

@@ -888,6 +888,7 @@
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = 372ZUL2ZB7;
 				INFOPLIST_FILE = "../Tests/TestHost-iOS/Info.plist";
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -905,6 +906,7 @@
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = 372ZUL2ZB7;
 				INFOPLIST_FILE = "../Tests/TestHost-iOS/Info.plist";
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -1250,6 +1252,7 @@
 				DYLIB_INSTALL_NAME_BASE = "@rpath";
 				INFOPLIST_FILE = BugsnagNetworkRequestPlugin/Info.plist;
 				INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -1273,6 +1276,7 @@
 				DYLIB_INSTALL_NAME_BASE = "@rpath";
 				INFOPLIST_FILE = BugsnagNetworkRequestPlugin/Info.plist;
 				INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -1293,6 +1297,7 @@
 				GCC_WARN_PEDANTIC = NO;
 				GCC_WARN_UNUSED_PARAMETER = NO;
 				INFOPLIST_FILE = BugsnagNetworkRequestPluginTests/Info.plist;
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -1313,6 +1318,7 @@
 				GCC_WARN_PEDANTIC = NO;
 				GCC_WARN_UNUSED_PARAMETER = NO;
 				INFOPLIST_FILE = BugsnagNetworkRequestPluginTests/Info.plist;
+				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",

Gemfile

@@ -4,7 +4,7 @@ gem 'cocoapods'
 
 # A reference to Maze Runner is only needed for running tests locally and if committed it must be
 # portable for CI, e.g. a specific release.  However, leaving it commented out would mean quicker CI.
-gem 'bugsnag-maze-runner', '~> 8.0'
+gem 'bugsnag-maze-runner', '~> 8.8'
 
 # Use a specific branch
 #gem 'bugsnag-maze-runner', git: 'https://github.com/bugsnag/maze-runner', branch: 'master'

Gemfile.lock

@@ -21,9 +21,9 @@ GEM
       faye-websocket (~> 0.11.0)
       selenium-webdriver (~> 4.2, < 4.6)
     atomos (0.1.3)
-    bugsnag (6.25.2)
+    bugsnag (6.26.0)
       concurrent-ruby (~> 1.0)
-    bugsnag-maze-runner (8.1.4)
+    bugsnag-maze-runner (8.8.0)
       appium_lib (~> 12.0.0)
       appium_lib_core (~> 5.4.0)
       bugsnag (~> 6.24)
@@ -122,7 +122,7 @@ GEM
     ethon (0.16.0)
       ffi (>= 1.15.0)
     eventmachine (1.2.7)
-    faye-websocket (0.11.2)
+    faye-websocket (0.11.3)
       eventmachine (>= 0.12.0)
       websocket-driver (>= 0.5.1)
     ffi (1.15.5)
@@ -140,25 +140,25 @@ GEM
       regexp_parser (~> 2.0)
       simpleidn (~> 0.2)
       uri_template (~> 0.7)
-    mime-types (3.4.1)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.0218.1)
+    mime-types-data (3.2023.1003)
     minitest (5.18.1)
     molinillo (0.8.0)
     multi_test (0.1.2)
     nanaimo (0.3.0)
     nap (1.1.0)
     netrc (0.11.0)
-    nokogiri (1.15.3-x86_64-darwin)
+    nokogiri (1.15.4-x86_64-darwin)
       racc (~> 1.4)
     optimist (3.0.1)
     os (1.0.1)
     power_assert (2.0.3)
     public_suffix (4.0.7)
     racc (1.7.1)
-    rack (2.2.7)
+    rack (2.2.8)
     rake (12.3.3)
-    regexp_parser (2.8.1)
+    regexp_parser (2.8.2)
     rexml (3.2.5)
     ruby-macho (2.5.1)
     rubyzip (2.3.2)
@@ -183,8 +183,8 @@ GEM
     unf_ext (0.0.8.2)
     uri_template (0.7.0)
     webrick (1.7.0)
-    websocket (1.2.9)
-    websocket-driver (0.7.5)
+    websocket (1.2.10)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     xcodeproj (1.22.0)
@@ -198,9 +198,10 @@ GEM
 PLATFORMS
   x86_64-darwin-19
   x86_64-darwin-20
+  x86_64-darwin-21
 
 DEPENDENCIES
-  bugsnag-maze-runner (~> 8.0)
+  bugsnag-maze-runner (~> 8.8)
   cocoapods
 
 BUNDLED WITH

Makefile

@@ -30,7 +30,12 @@ else
  else
   ifeq ($(PLATFORM),iOS)
    SDK?=iphonesimulator
-   DEVICE?=iPhone 8
+   ifeq ($(shell expr $(OS) \>= 17.0), 1)
+	DEVICE?=iPhone 14
+   else
+	DEVICE?=iPhone 8
+   endif
+
    DESTINATION?=platform=iOS Simulator,name=$(DEVICE),OS=$(OS)
    RELEASE_DIR=Release-iphoneos
   else

Tests/BugsnagTests/BSGInternalErrorReporterTests.m

@@ -63,14 +63,17 @@ - (void)testEventWithException {
 
     NSException *exception = nil;
     @try {
-        NSLog(@"%@", @[][0]);
+        [[NSException exceptionWithName:NSRangeException
+                                 reason:@"Something is out of range"
+                               userInfo:nil] raise];
     } @catch (NSException *e) {
         exception = e;
     }
 
     BugsnagEvent *event = [reporter eventWithException:exception diagnostics:nil groupingHash:@"test"];
     XCTAssertEqualObjects(event.errors[0].errorClass, @"NSRangeException");
-    XCTAssertEqualObjects(event.errors[0].errorMessage, @"*** -[__NSArray0 objectAtIndex:]: index 0 beyond bounds for empty NSArray");
+    XCTAssertEqualObjects(event.errors[0].errorMessage, @"Something is out of range");
+
     XCTAssertEqualObjects(event.groupingHash, @"test");
     XCTAssertEqualObjects(event.threads, @[]);
     XCTAssertGreaterThan(event.errors[0].stacktrace.count, 0);

Tests/BugsnagTests/BugsnagStackframeTest.m

@@ -253,6 +253,10 @@ - (void)testRealCallStackSymbols {
             // This frame is not in any known image
             return;
         }
+        if ([stackframe.method isEqualToString: @"start_sim"]) {
+            // This frame is part of the simulator environment
+            return;
+        }
 #endif
         XCTAssertNotNil(stackframe.machoUuid);
         XCTAssertNotNil(stackframe.machoVmAddress);

features/barebone_tests.feature

@@ -68,7 +68,7 @@ Feature: Barebone tests
     And the event "metaData.error.nsexception.userInfo.date" equals "2001-01-01 00:00:00 +0000"
     And the event "metaData.error.nsexception.userInfo.NSUnderlyingError" matches "Error Domain=ErrorDomain Code=0"
     And the event "metaData.error.nsexception.userInfo.scenario" equals "BareboneTestHandledScenario"
-    And the event "metaData.error.reason" equals "-[__NSSingleObjectArrayI objectAtIndex:]: index 1 beyond bounds [0 .. 0]"
+    And the event "metaData.error.reason" equals "Something is out of range"
     And the event "metaData.error.type" equals "nsexception"
     And the event "metaData.usage" is null
     And the event "metaData.user.email" is null
@@ -103,7 +103,7 @@ Feature: Barebone tests
       | notify      | rangeException |
     And the event does not contain the feature flag "nope"
     And the exception "errorClass" equals "NSRangeException"
-    And the exception "message" equals "-[__NSSingleObjectArrayI objectAtIndex:]: index 1 beyond bounds [0 .. 0]"
+    And the exception "message" equals "Something is out of range"
     And the exception "type" equals "cocoa"
     And the error payload field "events.0.app.dsymUUIDs" is a non-empty array
     And the error payload field "events.0.app.duration" is a number
@@ -199,7 +199,7 @@ Feature: Barebone tests
     And the event "metaData.error.nsexception.userInfo.date" equals "2001-01-01 00:00:00 +0000"
     And the event "metaData.error.nsexception.userInfo.NSUnderlyingError" matches "Error Domain=ErrorDomain Code=0"
     And the event "metaData.error.nsexception.userInfo.scenario" equals "BareboneTestUnhandledErrorScenario"
-    And the event "metaData.error.reason" equals "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray"
+    And the event "metaData.error.reason" equals "Something is out of range"
     And the event "metaData.error.type" equals "nsexception"
     And the event "metaData.usage" is null
     And the event "metaData.user.email" is null
@@ -227,7 +227,7 @@ Feature: Barebone tests
       | featureFlag | variant |
       | Testing     |         |
     And the exception "errorClass" equals "NSRangeException"
-    And the exception "message" equals "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray"
+    And the exception "message" equals "Something is out of range"
     And the exception "type" equals "cocoa"
     And the error payload field "events.0.app.dsymUUIDs" is a non-empty array
     And the error payload field "events.0.app.duration" is a number
@@ -244,6 +244,7 @@ Feature: Barebone tests
 
   @skip_macos
   @skip_ios_16 # https://smartbear.atlassian.net/browse/PLAT-9724
+  @skip_ios_17
   Scenario: Barebone test: Out Of Memory
     When I run "OOMScenario"
 

features/delivery.feature

@@ -46,7 +46,7 @@ Feature: Delivery of errors
     And I configure Bugsnag for "OldHandledErrorScenario"
     And I wait to receive an error
     And I wait for the fixture to process the response
-    # The error should now have been deleted 
+    # The error should now have been deleted
     And I kill and relaunch the app
     And I clear the error queue
     And I configure Bugsnag for "OldHandledErrorScenario"
@@ -157,6 +157,7 @@ Feature: Delivery of errors
     And the event "usage.system.stringCharsTruncated" is not null
     And the event "usage.system.stringsTruncated" is not null
 
+  @skip_ios_17
   Scenario Outline: Attempt Delivery On Crash
     When I set the app to "<scenario_mode>" mode
     And I run "AttemptDeliveryOnCrashScenario"
@@ -174,7 +175,31 @@ Feature: Delivery of errors
     And I wait to receive 2 sessions
     Then I should receive no error
     Examples:
-      | scenario_mode   | error_type  | error_class      | message                                                                    |
-      | NSException     | nsexception | NSRangeException | *** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray |
-      | SwiftFatalError | mach        | Fatal error      | Unexpectedly found nil while unwrapping an Optional value                  |
-      | BadAccess       | mach        | EXC_BAD_ACCESS   | Attempted to dereference garbage pointer 0x20.                             |
+      | scenario_mode   | error_type  | error_class      | message                                                   |
+      | NSException     | nsexception | NSRangeException | Something is out of range                                 |
+      | SwiftFatalError | mach        | Fatal error      | Unexpectedly found nil while unwrapping an Optional value |
+      | BadAccess       | mach        | EXC_BAD_ACCESS   | Attempted to dereference garbage pointer 0x20.            |
+
+  @skip_below_ios_17
+  @skip_macos
+  Scenario Outline: Attempt Delivery On Crash iOS 17
+    When I set the app to "<scenario_mode>" mode
+    And I run "AttemptDeliveryOnCrashScenario"
+    And I wait to receive an error
+    Then the error is valid for the error reporting API
+    And the event "context" equals "OnSendError"
+    And the exception "errorClass" equals "<error_class>"
+    And the exception "message" equals "<message>"
+    And the event "metaData.error.type" equals "<error_type>"
+    And the event "unhandled" is true
+    And the event "usage.config.attemptDeliveryOnCrash" is true
+    And I discard the oldest error
+    And I relaunch the app after a crash
+    And I configure Bugsnag for "AttemptDeliveryOnCrashScenario"
+    And I wait to receive 2 sessions
+    Then I should receive no error
+    Examples:
+      | scenario_mode   | error_type  | error_class      | message                                                   |
+      | NSException     | nsexception | NSRangeException | Something is out of range                                 |
+      | SwiftFatalError | mach        | Fatal error      | Unexpectedly found nil while unwrapping an Optional value |
+      | BadAccess       | mach        | EXC_BAD_ACCESS   | Attempted to dereference garbage pointer 0x20.            |