From 7b01aa94bc0cd56227616ee5ba849282c5c35762 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 25 Sep 2024 09:23:35 +0100 Subject: [PATCH 1/7] add test analytics --- .buildkite/pipeline.yml | 23 ++++++++++++----------- docker-compose.yml | 2 ++ features/support/maze.buildkite.cfg | 7 +++++++ 3 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 features/support/maze.buildkite.cfg diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fcd46fec..1134215e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -45,6 +45,10 @@ steps: - "build-expo-apk" timeout_in_minutes: 50 plugins: + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^v51/main|v51/next$$" artifacts#v1.5.0: download: "build/output.apk" docker-compose#v3.9.0: @@ -55,9 +59,6 @@ steps: - --app=build/output.apk - --farm=bs - --device=ANDROID_12 - - --a11y-locator - - --fail-fast - - --retry=2 concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager @@ -67,6 +68,10 @@ steps: - "build-expo-ipa" timeout_in_minutes: 50 plugins: + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^v51/main|v51/next$$" artifacts#v1.5.0: download: "build/output.ipa" docker-compose#v3.9.0: @@ -77,10 +82,6 @@ steps: - --app=build/output.ipa - --farm=bs - --device=IOS_16 - - --a11y-locator - - --fail-fast - - --retry=2 - - --order=random concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager @@ -90,6 +91,10 @@ steps: - "build-expo-ipa" timeout_in_minutes: 50 plugins: + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^v51/main|v51/next$$" artifacts#v1.5.0: download: "build/output.ipa" docker-compose#v3.9.0: @@ -100,10 +105,6 @@ steps: - --app=build/output.ipa - --farm=bs - --device=IOS_14 - - --a11y-locator - - --fail-fast - - --retry=2 - - --order=random concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager diff --git a/docker-compose.yml b/docker-compose.yml index 1d9b5ca8..8accc11a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: DEBUG: BROWSER_STACK_USERNAME: BROWSER_STACK_ACCESS_KEY: + MAZE_NO_FAIL_FAST: networks: default: aliases: @@ -31,6 +32,7 @@ services: volumes: - ./build:/app/build - ./features:/app/features + - ./reports/:/app/reports/ release: build: diff --git a/features/support/maze.buildkite.cfg b/features/support/maze.buildkite.cfg new file mode 100644 index 00000000..49770ff4 --- /dev/null +++ b/features/support/maze.buildkite.cfg @@ -0,0 +1,7 @@ +--format=junit +--out=reports +--format=pretty +--a11y-locator +--fail-fast +--retry=2 +--order=random From 36cb83dc0717864d539b3258a416097c893fb9e4 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 25 Sep 2024 09:47:09 +0100 Subject: [PATCH 2/7] reorder plugins --- .buildkite/pipeline.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1134215e..e0045870 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -45,10 +45,6 @@ steps: - "build-expo-apk" timeout_in_minutes: 50 plugins: - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^v51/main|v51/next$$" artifacts#v1.5.0: download: "build/output.apk" docker-compose#v3.9.0: @@ -59,6 +55,10 @@ steps: - --app=build/output.apk - --farm=bs - --device=ANDROID_12 + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^v51/main|v51/next$$" concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager @@ -68,10 +68,6 @@ steps: - "build-expo-ipa" timeout_in_minutes: 50 plugins: - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^v51/main|v51/next$$" artifacts#v1.5.0: download: "build/output.ipa" docker-compose#v3.9.0: @@ -82,6 +78,10 @@ steps: - --app=build/output.ipa - --farm=bs - --device=IOS_16 + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^v51/main|v51/next$$" concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager @@ -91,10 +91,6 @@ steps: - "build-expo-ipa" timeout_in_minutes: 50 plugins: - test-collector#v1.10.2: - files: "reports/TEST-*.xml" - format: "junit" - branch: "^v51/main|v51/next$$" artifacts#v1.5.0: download: "build/output.ipa" docker-compose#v3.9.0: @@ -105,6 +101,10 @@ steps: - --app=build/output.ipa - --farm=bs - --device=IOS_14 + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^v51/main|v51/next$$" concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager From af92d5e6361cdb026e530e93198c9ca25d7bec33 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 25 Sep 2024 14:02:12 +0100 Subject: [PATCH 3/7] add test analytics --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 8accc11a..19b2bbda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: BUILDKITE_REPO: BUILDKITE_RETRY_COUNT: BUILDKITE_STEP_KEY: + BUILDKITE_ANALYTICS_TOKEN: MAZE_BUGSNAG_API_KEY: DEBUG: BROWSER_STACK_USERNAME: From bbb3e6bafaccd9c9c133419431d9be80e6099aa1 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 25 Sep 2024 15:15:58 +0100 Subject: [PATCH 4/7] update to MR v9 --- Gemfile | 2 +- Gemfile.lock | 33 ++++++++++++++++----------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/Gemfile b/Gemfile index 9a3fdda1..ddaf05da 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,7 @@ source 'https://rubygems.org' gem 'cocoapods', '~> 1.14.3' gem 'fastlane' -gem 'bugsnag-maze-runner', '~>8.0' +gem 'bugsnag-maze-runner', '~>9.0' # Use a branch of Maze Runner #gem 'bugsnag-maze-runner', git: 'https://github.com/bugsnag/maze-runner', branch: 'tms/use-maze-check' diff --git a/Gemfile.lock b/Gemfile.lock index 40530782..9b53bdce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -48,15 +48,15 @@ GEM babosa (1.0.4) base64 (0.2.0) bigdecimal (3.1.7) - bugsnag (6.26.4) + bugsnag (6.27.1) concurrent-ruby (~> 1.0) - bugsnag-maze-runner (8.20.2) + bugsnag-maze-runner (9.14.0) appium_lib (~> 12.0.0) appium_lib_core (~> 5.4.0) bugsnag (~> 6.24) cucumber (~> 7.1) cucumber-expressions (~> 6.0.0) - curb (~> 0.9.6) + curb (~> 1.0.5) dogstatsd-ruby (~> 5.5.0) json_schemer (~> 0.2.24) optimist (~> 3.0.1) @@ -67,7 +67,7 @@ GEM selenium-webdriver (~> 4.0) test-unit (~> 3.5.2) webrick (~> 1.7.0) - builder (3.2.4) + builder (3.3.0) childprocess (4.1.0) claide (1.1.0) cocoapods (1.14.3) @@ -144,7 +144,7 @@ GEM cucumber-wire (6.2.1) cucumber-core (~> 10.1, >= 10.1.0) cucumber-cucumber-expressions (~> 14.0, >= 14.0.0) - curb (0.9.11) + curb (1.0.6) declarative (0.0.20) diff-lcs (1.5.1) digest-crc (0.6.5) @@ -293,7 +293,7 @@ GEM base64 mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0305) + mime-types-data (3.2024.0903) mini_magick (4.12.0) mini_mime (1.1.5) minitest (5.22.3) @@ -307,7 +307,9 @@ GEM naturally (2.2.1) netrc (0.11.0) nkf (0.2.0) - nokogiri (1.16.4-x86_64-darwin) + nokogiri (1.16.7-arm64-darwin) + racc (~> 1.4) + nokogiri (1.16.7-x86_64-darwin) racc (~> 1.4) optimist (3.0.1) optparse (0.5.0) @@ -315,10 +317,10 @@ GEM plist (3.7.1) power_assert (2.0.3) public_suffix (4.0.7) - racc (1.7.3) + racc (1.8.1) rack (2.2.9) rake (12.3.3) - regexp_parser (2.9.0) + regexp_parser (2.9.2) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -343,9 +345,8 @@ GEM simctl (1.6.10) CFPropertyList naturally - simpleidn (0.2.2) - unf (~> 0.1.4) - sys-uname (1.2.3) + simpleidn (0.2.3) + sys-uname (1.3.0) ffi (~> 1.1) terminal-notifier (2.0.0) terminal-table (3.0.2) @@ -363,13 +364,10 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.9.1) unicode-display_width (2.5.0) uri_template (0.7.0) webrick (1.7.0) - websocket (1.2.10) + websocket (1.2.11) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -387,10 +385,11 @@ GEM xcpretty (~> 0.2, >= 0.0.7) PLATFORMS + arm64-darwin-23 x86_64-darwin-19 DEPENDENCIES - bugsnag-maze-runner (~> 8.0) + bugsnag-maze-runner (~> 9.0) cocoapods (~> 1.14.3) fastlane From a5de094ffca2d3c693fc5340dfe3562b93d9a4c9 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 25 Sep 2024 15:50:36 +0100 Subject: [PATCH 5/7] change branches for test analytics plugin --- .buildkite/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e0045870..84eea030 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -58,7 +58,7 @@ steps: test-collector#v1.10.2: files: "reports/TEST-*.xml" format: "junit" - branch: "^v51/main|v51/next$$" + branch: "^v51$" concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager @@ -81,7 +81,7 @@ steps: test-collector#v1.10.2: files: "reports/TEST-*.xml" format: "junit" - branch: "^v51/main|v51/next$$" + branch: "^v51$" concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager @@ -104,7 +104,7 @@ steps: test-collector#v1.10.2: files: "reports/TEST-*.xml" format: "junit" - branch: "^v51/main|v51/next$$" + branch: "^v51$" concurrency: 5 concurrency_group: 'browserstack-app' concurrency_method: eager From 8f693bdb9e2c318afc4524007d091c1fed55266a Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Wed, 25 Sep 2024 16:13:04 +0100 Subject: [PATCH 6/7] change branches for test analytics plugin --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 19b2bbda..0d8fad69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: dockerfile: dockerfiles/Dockerfile.ci expo-maze-runner: - image: 855461928731.dkr.ecr.us-west-1.amazonaws.com/maze-runner-releases:latest-v8-cli + image: 855461928731.dkr.ecr.us-west-1.amazonaws.com/maze-runner-releases:latest-v9-cli environment: BUILDKITE: BUILDKITE_BRANCH: From a91a73ca1f99c3f52a7ce7a240d85e6b78d2ae40 Mon Sep 17 00:00:00 2001 From: Josh Edney Date: Thu, 26 Sep 2024 09:23:53 +0100 Subject: [PATCH 7/7] update options passed to maze runner --- .buildkite/pipeline.yml | 12 ++++++++++++ features/support/maze.buildkite.cfg | 4 ---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 84eea030..8c8a8c5e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -55,6 +55,10 @@ steps: - --app=build/output.apk - --farm=bs - --device=ANDROID_12 + - --a11y-locator + - --fail-fast + - --retry=2 + - --order=random test-collector#v1.10.2: files: "reports/TEST-*.xml" format: "junit" @@ -78,6 +82,10 @@ steps: - --app=build/output.ipa - --farm=bs - --device=IOS_16 + - --a11y-locator + - --fail-fast + - --retry=2 + - --order=random test-collector#v1.10.2: files: "reports/TEST-*.xml" format: "junit" @@ -101,6 +109,10 @@ steps: - --app=build/output.ipa - --farm=bs - --device=IOS_14 + - --a11y-locator + - --fail-fast + - --retry=2 + - --order=random test-collector#v1.10.2: files: "reports/TEST-*.xml" format: "junit" diff --git a/features/support/maze.buildkite.cfg b/features/support/maze.buildkite.cfg index 49770ff4..c5bc1827 100644 --- a/features/support/maze.buildkite.cfg +++ b/features/support/maze.buildkite.cfg @@ -1,7 +1,3 @@ --format=junit --out=reports --format=pretty ---a11y-locator ---fail-fast ---retry=2 ---order=random