chore: retry all soft-fail Android E2E steps, add logging to iOS E2E (#…

…4084) * Allow test app to install external if it wants It is quite large and managing emulator internal space is important * Bump gradle in test app to 6.6, kotlin to current, compile/target SDKs to 29 Includes output of `./gradlew wrapper` to regenerate everything * E2E action no longer needs special cache handling Upstream bug resolved * Increase emulator RAM and disable JNI checking in E2E test This should speed things up a bit, increasing success rate vs timeouts * Add iOS Detox tracing to E2E config Should help diagnose iOS flakiness there * Remove JDK8 config in Android E2E config It is the default on macOS 10.15 runner, so special setup unnecessary * Use Xcode 11.6 (current) for iOS E2E run This is the default, so special configuration not needed, but still need to extract the Xcode version for Detox build cache * Decompose Android E2E tasks, retry all flaky steps
invertase · Aug 14, 2020 · d98b80e · d98b80e
1 parent bff8f9f
commit d98b80e
Show file tree

Hide file tree

Showing 9 changed files with 120 additions and 102 deletions.
diff --git a/.github/workflows/tests_e2e.yml b/.github/workflows/tests_e2e.yml
@@ -25,17 +25,6 @@ jobs:
 
       - uses: actions/setup-node@v2-beta
 
-      # Yarn cache directory on macOS GH Action virtual environment isn't empty;
-      #  -> https://github.com/actions/virtual-environments/issues/427
-      #  -> https://github.com/actions/cache/issues/187
-      - name: Clean Yarn cache directory
-        run: yarn cache clean
-
-      # Gradle cache directory on macOS GH Action virtual environment isn't empty;
-      #  -> https://github.com/actions/virtual-environments/issues/427
-      - name: Clean Gradle cache directory
-        run: rm -rf ~/.gradle/caches
-
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
@@ -58,67 +47,112 @@ jobs:
           export DETOX_DISABLE_POSTINSTALL=1
           yarn --no-audit --prefer-offline
 
-      - name: Setup JDK8
-        uses: actions/setup-java@v1
-        with:
-          java-version: '8'
-          architecture: 'x64'
+      - name: Verify JDK 1.8
+        # OpenJDK1.8 is the default in GitHub macOS 10.15 runner, setup is not required
+        # Run a check that exits with error unless it is 1.8 version to future-proof against unexpected upgrades
+        run: java -fullversion 2>&1 | grep '1.8'
+        shell: bash
 
       - name: Build Android App
         run: cd tests/android && ./gradlew assembleDebug assembleAndroidTest -DtestBuildType=debug -Dorg.gradle.daemon=false
 
-      - name: Download Emulator Image
+      - name: Pre-fetch Javascript bundle
+        # Prebuild the bundle so that's fast when the app starts.
         run: |
-          echo "y" | $ANDROID_HOME/tools/bin/sdkmanager --install "system-images;android-28;google_apis;x86_64"
-          echo "no" | $ANDROID_HOME/tools/bin/avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-28;google_apis;x86_64' -g google_apis
-          $ANDROID_HOME/emulator/emulator -list-avds
+          nohup yarn run tests:packager:jet &
+          printf 'Waiting for packager to come online'
+          until curl --output /dev/null --silent --head --fail http://localhost:8081/status; do
+            printf '.'
+            sleep 2
+          done
+          echo "Packager is online!"
+          curl --output /dev/null --silent --head --fail "http://localhost:8081/index.bundle?platform=android&dev=true&minify=false&inlineSourceMap=true"
 
+      - name: Download Emulator Image
+        # This can fail on network request, wrap with retry
+        uses: nick-invision/retry@v1
+        with:
+          timeout_minutes: 10
+          retry_wait_seconds: 60
+          max_attempts: 3
+          command: echo "y" | $ANDROID_HOME/tools/bin/sdkmanager --install "system-images;android-28;google_apis;x86_64"
+
+      - name: Create Emulator
+        run: echo "no" | $ANDROID_HOME/tools/bin/avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-28;google_apis;x86_64' -g google_apis
+
+      # These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes
       - name: Start Android Emulator
-        timeout-minutes: 15
+        id: emu1
+        timeout-minutes: 5
         continue-on-error: true
         run: |
           echo "Starting emulator"
-          nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim &
+          nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 &
           $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
+
+      - name: Start Android Emulator Retry 1
+        id: emu2
+        if: steps.emu1.outcome=='failure'
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          echo "Starting emulator, second attempt"
           $ANDROID_HOME/platform-tools/adb devices
-          $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
-          $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
-          $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
-          echo "Emulator started"
+          sudo killall -9 qemu-system-x86_64-headless || true
+          sleep 2
+          nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 &
+          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
+
+      - name: Start Android Emulator Retry 2
+        id: emu3
+        if: steps.emu2.outcome=='failure'
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          echo "Starting emulator, third attempt"
+          $ANDROID_HOME/platform-tools/adb devices
+          sudo killall -9 qemu-system-x86_64-headless || true
+          sleep 2
+          nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 &
+          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
+
+      - name: Emulator Status
+        if: always()
+        run: |
+          if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then
+             echo "Emulator Started"
+          else
+             exit 1
+          fi
 
       - name: Detox Test
-        timeout-minutes: 40
-        # Prebuild the bundle so that's fast when the app starts.
         # Detox uses Espresso to choreograph steps in reaction to UI events, so we need to send a stream of taps.
-        # But if you send them before the app is ready, there is a chance react-native comes alive with an ACTION_UP and no ACTION_DOWN, which hangs everything.
-        # So we wait 180 seconds, which seems to be enough for the app to come up.
+        timeout-minutes: 40
         run: |
-          nohup yarn run tests:packager:jet &
-          printf 'Waiting for packager to come online'
-          until curl --output /dev/null --silent --head --fail http://localhost:8081/status; do
-            printf '.'
-            sleep 2
-          done
-          echo "Packager is online!"
-          curl --output /dev/null --silent --head --fail "http://localhost:8081/index.bundle?platform=android&dev=true&minify=false&inlineSourceMap=true"
-          nohup sh -c "sleep 180 && until false; do $ANDROID_HOME/platform-tools/adb shell input tap 100 800; sleep 0.1; done" &
+          $ANDROID_HOME/platform-tools/adb devices
+          $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
+          $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
+          $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
+          nohup sh -c "until false; do $ANDROID_HOME/platform-tools/adb shell input tap 100 800; sleep 0.2; done" &
           nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" &
-          cd tests
-          ./node_modules/.bin/nyc ./node_modules/.bin/detox test --loglevel trace --configuration android.emu.debug --cleanup
+          yarn tests:android:test-cover --cleanup
         shell: bash
 
       - name: Submit Coverage
-        run: |
-          ./node_modules/.bin/codecov
-        shell: bash
+        # This can fail on timeouts etc, wrap with retry
+        uses: nick-invision/retry@v1
+        with:
+          timeout_minutes: 10
+          retry_wait_seconds: 60
+          max_attempts: 3
+          command: ./node_modules/.bin/codecov
 
       - name: Compress Emulator Log
         if: always()
-        run: |
-            gzip -9 adb-log.txt
+        run: gzip -9 adb-log.txt
         shell: bash
 
-      - name: Upload Logs
+      - name: Upload Emulator Log
         uses: actions/upload-artifact@v2
         if: always()
         with:
@@ -131,12 +165,15 @@ jobs:
   ios:
     name: iOS
     runs-on: macos-latest
+    # TODO matrix across APIs, at least 10 and 13 (lowest to highest)
     timeout-minutes: 35
     env:
       # Use the latest version of xcode for rnfirebase
       # See link below for available versions;
       #  -> https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md#xcode
-      DEVELOPER_DIR: /Applications/Xcode_11.4.app
+      # TODO they update image default quickly, version is only used to differentiate detox cache, should fetch dynamically so we don't have to worry about it:
+      #       `xcodebuild -version|tail -1|cut -f3 -d' '` gets the build code, then put that as output for a step and then use that output in detox cache key differentiator
+      DEVELOPER_DIR: /Applications/Xcode_11.6.app
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     steps:
       - uses: actions/checkout@v2
@@ -145,12 +182,6 @@ jobs:
 
       - uses: actions/setup-node@v2-beta
 
-      # Yarn cache directory on macOS GH Action virtual environment isn't empty;
-      #  -> https://github.com/actions/virtual-environments/issues/427
-      #  -> https://github.com/actions/cache/issues/187
-      - name: Clean Yarn cache directory
-        run: yarn cache clean
-
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
@@ -212,7 +243,7 @@ jobs:
           echo "Packager is online!"
 
           cd tests
-          ./node_modules/.bin/nyc ./node_modules/.bin/detox test --configuration ios.ci --cleanup
+          ./node_modules/.bin/nyc ./node_modules/.bin/detox test --loglevel trace --debug-synchronization 200 --configuration ios.ci --cleanup
         shell: bash
 
       - name: Submit Coverage

diff --git a/package.json b/package.json
@@ -25,7 +25,7 @@
     "tests:android:test": "cd tests && ./node_modules/.bin/detox test --configuration android.emu.debug",
     "tests:android:test:debug": "cd tests && ./node_modules/.bin/detox test --configuration android.emu.debug --inspect",
     "tests:android:test-reuse": "cd tests && ./node_modules/.bin/detox test --configuration android.emu.debug --reuse",
-    "tests:android:test-cover": "cd tests && ./node_modules/.bin/nyc ./node_modules/.bin/detox test --configuration android.emu.debug",
+    "tests:android:test-cover": "cd tests && ./node_modules/.bin/nyc ./node_modules/.bin/detox test --loglevel trace --configuration android.emu.debug",
     "tests:android:test-cover-reuse": "cd tests && ./node_modules/.bin/nyc ./node_modules/.bin/detox test --configuration android.emu.debug --reuse",
     "tests:ios:build": "cd tests && ./node_modules/.bin/detox build --configuration ios.sim.debug",
     "tests:ios:build-release": "cd tests && ./node_modules/.bin/detox build --configuration ios.sim.release",
@@ -99,4 +99,4 @@
       "**/superstruct/**"
     ]
   }
-}
+}
diff --git a/tests/android/app/src/main/AndroidManifest.xml b/tests/android/app/src/main/AndroidManifest.xml
@@ -1,5 +1,6 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
   xmlns:tools="http://schemas.android.com/tools"
+  android:installLocation="auto"
   package="com.invertase.testing">
 
   <uses-permission android:name="android.permission.INTERNET" />

diff --git a/tests/android/build.gradle b/tests/android/build.gradle
@@ -1,5 +1,5 @@
 buildscript {
-  ext.kotlinVersion = '1.3.31'
+  ext.kotlinVersion = '1.3.70'
 
   repositories {
     google()
@@ -8,7 +8,7 @@ buildscript {
   }
   dependencies {
     classpath 'com.google.gms:google-services:4.3.3'
-    classpath 'com.android.tools.build:gradle:4.0.0'
+    classpath 'com.android.tools.build:gradle:4.0.1'
     classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
     classpath 'com.google.firebase:perf-plugin:1.3.0'
     classpath 'com.google.firebase:firebase-crashlytics-gradle:2.2.0'
@@ -41,10 +41,10 @@ allprojects {
 subprojects {
   task listAllDependencies(type: DependencyReportTask) {}
   ext {
-    compileSdk = 28
+    compileSdk = 29
     buildTools = "29.0.2"
     minSdk = 21
-    targetSdk = 28
+    targetSdk = 29
   }
 
   afterEvaluate { project ->

diff --git a/tests/android/gradle/wrapper/gradle-wrapper.jar b/tests/android/gradle/wrapper/gradle-wrapper.jar
diff --git a/tests/android/gradle/wrapper/gradle-wrapper.properties b/tests/android/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/tests/android/gradlew b/tests/android/gradlew
@@ -7,7 +7,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -82,6 +82,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
+
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -125,10 +126,11 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
@@ -154,19 +156,19 @@ if $cygwin ; then
         else
             eval `echo args$i`="\"$arg\""
         fi
-        i=$((i+1))
+        i=`expr $i + 1`
     done
     case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
     esac
 fi
 
@@ -175,14 +177,9 @@ save () {
     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
     echo " "
 }
-APP_ARGS=$(save "$@")
+APP_ARGS=`save "$@"`
 
 # Collect all arguments for the java command, following the shell quoting and substitution rules
 eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
-# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
-  cd "$(dirname "$0")"
-fi
-
 exec "$JAVACMD" "$@"