Trim and truncate test failure stack traces for both orchestrator and…

… classic/non-orchestrator modes. This change should clean up test failure reporting by: - Remove test runner framework related stack frames - Truncate stack traces to a 64KB size when running under orchestrator to attempt to avoid binder transaction limits. This limit is already enforced when running in classic/non-orchestrator mode JUnit 4.13 has a really nice getTrimmedStackTrace feature, but androidx.test is fixed to 4.12 for the time being. So as a temporary workaround, copy the relevant JUnit change junit-team/junit4#1028 into this project. Fixes #729, and hopefully #269 PiperOrigin-RevId: 329797783
android · Sep 16, 2020 · d1f9226 · d1f9226
1 parent 4bd7957
commit d1f9226
Show file tree

Hide file tree

Showing 10 changed files with 364 additions and 49 deletions.
diff --git a/...unit_runner/java/androidx/test/internal/runner/listener/InstrumentationResultPrinter.java b/...unit_runner/java/androidx/test/internal/runner/listener/InstrumentationResultPrinter.java
@@ -19,6 +19,7 @@
 import android.os.Bundle;
 import androidx.annotation.VisibleForTesting;
 import android.util.Log;
+import androidx.test.services.events.internal.StackTrimmer;
 import java.io.PrintStream;
 import org.junit.internal.TextListener;
 import org.junit.runner.Description;
@@ -45,8 +46,6 @@ public class InstrumentationResultPrinter extends InstrumentationRunListener {
 
   private static final String TAG = "InstrumentationResultPrinter";
 
-  @VisibleForTesting static final int MAX_TRACE_SIZE = 64 * 1024;
-
   /**
    * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER},
    * identifies AndroidJUnitRunner as the source of the report. This is sent with all status
@@ -175,21 +174,12 @@ public void testAssumptionFailure(Failure failure) {
   }
 
   private void reportFailure(Failure failure) {
-    String trace = failure.getTrace();
-    if (trace.length() > MAX_TRACE_SIZE) {
-      // Since AJUR needs to report failures back to AM via a binder IPC, we need to make sure that
-      // we don't exceed the Binder transaction limit - which is 1MB per process.
-      Log.w(
-          TAG,
-          String.format("Stack trace too long, trimmed to first %s characters.", MAX_TRACE_SIZE));
-      trace = trace.substring(0, MAX_TRACE_SIZE) + "\n";
-    }
+    String trace = StackTrimmer.getTrimmedStackTrace(failure);
     testResult.putString(REPORT_KEY_STACK, trace);
     // pretty printing
     testResult.putString(
         Instrumentation.REPORT_KEY_STREAMRESULT,
-        String.format(
-            "\nError in %s:\n%s", failure.getDescription().getDisplayName(), failure.getTrace()));
+        String.format("\nError in %s:\n%s", failure.getDescription().getDisplayName(), trace));
   }
 
   @Override

diff --git a/runner/android_junit_runner/java/androidx/test/orchestrator/junit/ParcelableFailure.java b/runner/android_junit_runner/java/androidx/test/orchestrator/junit/ParcelableFailure.java
@@ -19,9 +19,10 @@
 import android.os.Parcel;
 import android.os.Parcelable;
 import android.util.Log;
+import androidx.test.services.events.internal.StackTrimmer;
 import org.junit.runner.notification.Failure;
 
-/** Parcelable imitation of a JUnit ParcelableFailure */
+/** Parcelable imitation of a JUnit Failure */
 public final class ParcelableFailure implements Parcelable {
 
   private static final String TAG = "ParcelableFailure";
@@ -33,7 +34,7 @@ public final class ParcelableFailure implements Parcelable {
 
   public ParcelableFailure(Failure failure) {
     this.description = new ParcelableDescription(failure.getDescription());
-    this.trace = failure.getTrace();
+    this.trace = StackTrimmer.getTrimmedStackTrace(failure);
   }
 
   private ParcelableFailure(Parcel in) {

diff --git a/...vatests/androidx/test/internal/events/client/OrchestratedInstrumentationListenerTest.java b/...vatests/androidx/test/internal/events/client/OrchestratedInstrumentationListenerTest.java
@@ -16,10 +16,11 @@
 
 package androidx.test.internal.events.client;
 
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
+import static com.google.common.truth.Truth.assertThat;
 import static org.mockito.Mockito.verify;
 
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.services.events.internal.StackTrimmer;
 import androidx.test.services.events.run.TestAssumptionFailureEvent;
 import androidx.test.services.events.run.TestFailureEvent;
 import androidx.test.services.events.run.TestFinishedEvent;
@@ -39,10 +40,9 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
-import org.robolectric.RobolectricTestRunner;
 
 /** Unit tests for {@link OrchestratedInstrumentationListener}. */
-@RunWith(RobolectricTestRunner.class)
+@RunWith(AndroidJUnit4.class)
 public class OrchestratedInstrumentationListenerTest {
   @Mock TestRunEventService testRunEventService;
 
@@ -83,7 +83,7 @@ public void testRunFinished() throws TestEventClientException {
     verify(testRunEventService).send(argument.capture());
 
     TestRunFinishedEvent event = (TestRunFinishedEvent) argument.getValue();
-    assertThat(event.count, is(1));
+    assertThat(event.count).isEqualTo(1);
   }
 
   @Test
@@ -151,12 +151,12 @@ public void testIgnored() throws TestEventClientException {
 
   private static void compareDescription(
       TestRunEventWithTestCase event, Description jUnitDescription) {
-    assertThat(event.testCase.className, is(jUnitDescription.getClassName()));
-    assertThat(event.testCase.methodName, is(jUnitDescription.getMethodName()));
+    assertThat(event.testCase.className).isEqualTo(jUnitDescription.getClassName());
+    assertThat(event.testCase.methodName).isEqualTo(jUnitDescription.getMethodName());
   }
 
   private static void compareFailure(TestFailureEvent event, Failure jUnitFailure) {
-    assertThat(event.failure.stackTrace, is(jUnitFailure.getTrace()));
+    assertThat(event.failure.stackTrace).isEqualTo(StackTrimmer.getTrimmedStackTrace(jUnitFailure));
     compareDescription(event, jUnitFailure.getDescription());
   }
 }
diff --git a/...er/javatests/androidx/test/internal/runner/listener/InstrumentationResultPrinterTest.java b/...er/javatests/androidx/test/internal/runner/listener/InstrumentationResultPrinterTest.java
@@ -16,7 +16,6 @@
 
 package androidx.test.internal.runner.listener;
 
-import static androidx.test.internal.runner.listener.InstrumentationResultPrinter.MAX_TRACE_SIZE;
 import static androidx.test.internal.runner.listener.InstrumentationResultPrinter.REPORT_KEY_STACK;
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertTrue;
@@ -63,27 +62,6 @@ public void sendStatus(int code, Bundle bundle) {
     assertTrue(resultBundle[0].containsKey(REPORT_KEY_STACK));
   }
 
-  @Test
-  public void verifyFailureStackTraceIsTruncated() throws Exception {
-    InstrumentationResultPrinter intrResultPrinter = new InstrumentationResultPrinter();
-    intrResultPrinter.testNum = 1;
-
-    Failure testFailure = new Failure(Description.EMPTY, new Exception(getVeryLargeString()));
-    intrResultPrinter.testFailure(testFailure);
-
-    int testResultTraceLength =
-        intrResultPrinter.testResult.getString(REPORT_KEY_STACK).length() - 1;
-    assertTrue(
-        String.format(
-            "The stack trace length: %s, exceeds the max: %s",
-            testResultTraceLength, MAX_TRACE_SIZE),
-        testResultTraceLength <= MAX_TRACE_SIZE);
-  }
-
-  private static String getVeryLargeString() {
-    return new String(new char[1000000]);
-  }
-
   @Test
   public void verifyFailureDescriptionPropagatedToStartAndFinishMethods() throws Exception {
     Description[] descriptions = new Description[2];
@@ -101,7 +79,7 @@ public void testFinished(Description description) throws Exception {
         };
 
     Description d = Description.createTestDescription(this.getClass(), "Failure Description");
-    Failure testFailure = new Failure(d, new Exception(getVeryLargeString()));
+    Failure testFailure = new Failure(d, new Exception());
     intrResultPrinter.testFailure(testFailure);
 
     assertEquals(d, descriptions[0]);

diff --git a/...android_junit_runner/javatests/androidx/test/orchestrator/junit/BundleJUnitUtilsTest.java b/...android_junit_runner/javatests/androidx/test/orchestrator/junit/BundleJUnitUtilsTest.java
@@ -23,6 +23,7 @@
 import android.os.Parcel;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import androidx.test.orchestrator.SampleJUnitTest;
+import androidx.test.services.events.internal.StackTrimmer;
 import org.junit.Test;
 import org.junit.runner.Description;
 import org.junit.runner.Result;
@@ -57,7 +58,7 @@ public void fromFailure() {
         BundleJUnitUtils.getFailure(
             parcelBundle(BundleJUnitUtils.getBundleFromFailure(jUnitFailure)));
 
-    assertThat(parcelableFailure.getTrace(), is(jUnitFailure.getTrace()));
+    assertThat(parcelableFailure.getTrace(), is(StackTrimmer.getTrimmedStackTrace(jUnitFailure)));
     compareDescription(parcelableFailure.getDescription(), jUnitFailure.getDescription());
   }
 
@@ -122,7 +123,7 @@ private static void compareDescription(
   }
 
   private static void compareFailure(ParcelableFailure parcelableFailure, Failure jUnitFailure) {
-    assertThat(parcelableFailure.getTrace(), is(jUnitFailure.getTrace()));
+    assertThat(parcelableFailure.getTrace(), is(StackTrimmer.getTrimmedStackTrace(jUnitFailure)));
     compareDescription(parcelableFailure.getDescription(), jUnitFailure.getDescription());
   }
 

diff --git a/services/events/java/androidx/test/services/events/ParcelableConverter.java b/services/events/java/androidx/test/services/events/ParcelableConverter.java
@@ -21,6 +21,7 @@
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import android.util.Log;
+import androidx.test.services.events.internal.StackTrimmer;
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Array;
 import java.lang.reflect.Method;
@@ -99,7 +100,7 @@ public static FailureInfo getFailure(@NonNull Failure junitFailure) throws TestE
     return new FailureInfo(
         junitFailure.getMessage(),
         junitFailure.getTestHeader(),
-        junitFailure.getTrace(),
+        StackTrimmer.getTrimmedStackTrace(junitFailure),
         getTestCaseFromDescription(junitFailure.getDescription()));
   }
 

diff --git a/services/events/java/androidx/test/services/events/internal/StackTrimmer.java b/services/events/java/androidx/test/services/events/internal/StackTrimmer.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.test.services.events.internal;
+
+import androidx.annotation.VisibleForTesting;
+import android.util.Log;
+import org.junit.runner.notification.Failure;
+
+/** A utility for JUnit failure stack traces */
+public final class StackTrimmer {
+
+  private static final String TAG = "StackTrimmer";
+
+  @VisibleForTesting static final int MAX_TRACE_SIZE = 64 * 1024;
+
+  private StackTrimmer() {}
+
+  /**
+   * Returns the stack trace, trimming to remove frames from the test runner, and truncating if its
+   * too large.
+   */
+  public static String getTrimmedStackTrace(Failure failure) {
+    // TODO(b/128614857): switch to JUnit 4.13 Failure.getTrimmedTrace once its available
+    String trace = Throwables.getTrimmedStackTrace(failure.getException());
+    if (trace.length() > MAX_TRACE_SIZE) {
+      // Since AJUR needs to report failures back to AM via a binder IPC, we need to make sure that
+      // we don't exceed the Binder transaction limit - which is 1MB per process.
+      Log.w(
+          TAG,
+          String.format("Stack trace too long, trimmed to first %s characters.", MAX_TRACE_SIZE));
+      trace = trace.substring(0, MAX_TRACE_SIZE) + "\n";
+    }
+    return trace;
+  }
+}