diff --git a/lib/bugsnag/integrations/railtie.rb b/lib/bugsnag/integrations/railtie.rb
index 763ab7f16..20dc6347f 100644
--- a/lib/bugsnag/integrations/railtie.rb
+++ b/lib/bugsnag/integrations/railtie.rb
@@ -67,6 +67,14 @@ class Railtie < ::Rails::Railtie
       rescue
         app.config.middleware.use Bugsnag::Rack
       end
+    rescue FrozenError
+      # This can happen when running RSpec if there is a crash after Rails has
+      # started booting but before we've added our middleware. If we don't ignore
+      # this error then the stacktrace blames Bugsnag, which isn't accurate as
+      # the middleware will only be frozen if an earlier error occurs
+      # See this comment for more info:
+      # https://github.com/thoughtbot/factory_bot_rails/issues/303#issuecomment-434560625
+      Bugsnag.configuration.warn("Unable to add Bugsnag::Rack middleware as the middleware stack is frozen")
     end
 
     ##