diff --git a/pkg/storage/chunk/client/gcp/gcs_object_client_test.go b/pkg/storage/chunk/client/gcp/gcs_object_client_test.go
index 560c2e5297ef..ac3e7a77dcd6 100644
--- a/pkg/storage/chunk/client/gcp/gcs_object_client_test.go
+++ b/pkg/storage/chunk/client/gcp/gcs_object_client_test.go
@@ -246,7 +246,7 @@ func fakeHTTPRespondingServer(t *testing.T, code int) *httptest.Server {
 }
 
 func fakeSleepingServer(t *testing.T, responseSleep, connectSleep time.Duration, closeOnNew, closeOnActive bool) *httptest.Server {
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// sleep on response to mimic server overload
 		time.Sleep(responseSleep)
 	}))
@@ -264,6 +264,6 @@ func fakeSleepingServer(t *testing.T, responseSleep, connectSleep time.Duration,
 		}
 	}
 	t.Cleanup(server.Close)
-
+	server.Start()
 	return server
 }