diff --git a/src/main/java/com/bugsnag/android/ErrorStore.java b/src/main/java/com/bugsnag/android/ErrorStore.java
index cbfd96be2d..e74fa901d4 100644
--- a/src/main/java/com/bugsnag/android/ErrorStore.java
+++ b/src/main/java/com/bugsnag/android/ErrorStore.java
@@ -3,6 +3,8 @@
 import java.io.File;
 import java.io.FileWriter;
 import java.io.Writer;
+import java.util.Arrays;
+import java.util.List;
 
 import android.content.Context;
 
@@ -80,9 +82,16 @@ void write(Error error) {
 
         // Limit number of saved errors to prevent disk space issues
         File exceptionDir = new File(path);
-        if (exceptionDir.isDirectory() && exceptionDir.listFiles().length >= MAX_STORED_ERRORS) {
-            Logger.warn("Discarding error without saving to disk as stored error limit reached");
-            return;
+        if (exceptionDir.isDirectory()) {
+            File[] files = exceptionDir.listFiles();
+            if (files.length >= MAX_STORED_ERRORS) {
+                // Sort files then delete the first one (oldest timestamp)
+                Arrays.sort(files);
+                Logger.warn(String.format("Discarding oldest error as stored error limit reached (%s)", files[0].getPath()));
+                if (!files[0].delete()) {
+                    files[0].deleteOnExit();
+                }
+            }
         }
 
         String filename = String.format("%s%d.json", path, System.currentTimeMillis());