diff --git a/go.sum b/go.sum
index 31fd50bcc..9c1c97052 100644
--- a/go.sum
+++ b/go.sum
@@ -363,7 +363,6 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17Xtb
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5 h1:knEvP4R5v5b2T107/Q6VzB0C8/6T7NXB/V7Vl1FtQsg=
 github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 h1:DB3NTM0ilba/6sW+vccdEnP10bVvrVunDwWvRa0hSKc=
 github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
diff --git a/pkg/restore/backoff.go b/pkg/restore/backoff.go
index 21048dd13..a84014c11 100644
--- a/pkg/restore/backoff.go
+++ b/pkg/restore/backoff.go
@@ -15,18 +15,11 @@ import (
 var (
 	errEpochNotMatch       = errors.NewNoStackError("epoch not match")
 	errKeyNotInRegion      = errors.NewNoStackError("key not in region")
-	errRegionNotFound      = errors.NewNoStackError("region not found")
-	errResp                = errors.NewNoStackError("response error")
 	errRewriteRuleNotFound = errors.NewNoStackError("rewrite rule not found")
 	errRangeIsEmpty        = errors.NewNoStackError("range is empty")
 	errGrpc                = errors.NewNoStackError("gRPC error")
-
-	// TODO: add `error` field to `DownloadResponse` for distinguish the errors of gRPC
-	// and the errors of request
-	errBadFormat      = errors.NewNoStackError("bad format")
-	errWrongKeyPrefix = errors.NewNoStackError("wrong key prefix")
-	errFileCorrupted  = errors.NewNoStackError("file corrupted")
-	errCannotRead     = errors.NewNoStackError("cannot read externel storage")
+	errDownloadFailed      = errors.NewNoStackError("download sst failed")
+	errIngestFailed        = errors.NewNoStackError("ingest sst failed")
 )
 
 const (
@@ -67,7 +60,7 @@ func newDownloadSSTBackoffer() utils.Backoffer {
 
 func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 	switch errors.Cause(err) {
-	case errResp, errGrpc, errEpochNotMatch:
+	case errGrpc, errEpochNotMatch, errIngestFailed:
 		bo.delayTime = 2 * bo.delayTime
 		bo.attempt--
 	case errRangeIsEmpty, errRewriteRuleNotFound:
diff --git a/pkg/restore/backoff_test.go b/pkg/restore/backoff_test.go
index 11accedd2..a07c0839b 100644
--- a/pkg/restore/backoff_test.go
+++ b/pkg/restore/backoff_test.go
@@ -37,7 +37,7 @@ func (s *testBackofferSuite) TestImporterBackoffer(c *C) {
 		case 0:
 			return errGrpc
 		case 1:
-			return errResp
+			return errEpochNotMatch
 		case 2:
 			return errRangeIsEmpty
 		}
@@ -54,8 +54,8 @@ func (s *testBackofferSuite) TestImporterBackoffer(c *C) {
 	}
 	err = utils.WithRetry(context.Background(), func() error {
 		defer func() { counter++ }()
-		return errResp
+		return errEpochNotMatch
 	}, &backoffer)
 	c.Assert(counter, Equals, 10)
-	c.Assert(err, Equals, errResp)
+	c.Assert(err, Equals, errEpochNotMatch)
 }
diff --git a/pkg/restore/import.go b/pkg/restore/import.go
index fec07a870..405af050c 100644
--- a/pkg/restore/import.go
+++ b/pkg/restore/import.go
@@ -6,7 +6,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
-	"strings"
 	"sync"
 	"time"
 
@@ -272,14 +271,12 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *RewriteRul
 					//      2. retry ingest
 					errIngest = errors.AddStack(errEpochNotMatch)
 					break ingestRetry
-				case errPb.RegionNotFound != nil:
-					errIngest = errors.AddStack(errRegionNotFound)
-					break ingestRetry
 				case errPb.KeyNotInRegion != nil:
 					errIngest = errors.AddStack(errKeyNotInRegion)
 					break ingestRetry
 				default:
-					errIngest = errors.Errorf("ingest error %s", errPb)
+					// Other errors like `ServerIsBusy`, `RegionNotFound`, etc. should be retryable
+					errIngest = errors.Annotatef(errIngestFailed, "ingest error %s", errPb)
 					break ingestRetry
 				}
 			}
@@ -346,7 +343,10 @@ func (importer *FileImporter) downloadSST(
 	for _, peer := range regionInfo.Region.GetPeers() {
 		resp, err = importer.importClient.DownloadSST(importer.ctx, peer.GetStoreId(), req)
 		if err != nil {
-			return nil, extractDownloadSSTError(err)
+			return nil, errors.Annotatef(errGrpc, "%s", err)
+		}
+		if resp.GetError() != nil {
+			return nil, errors.Annotate(errDownloadFailed, resp.GetError().GetMessage())
 		}
 		if resp.GetIsEmpty() {
 			return nil, errors.Trace(errRangeIsEmpty)
@@ -395,7 +395,10 @@ func (importer *FileImporter) downloadRawKVSST(
 	for _, peer := range regionInfo.Region.GetPeers() {
 		resp, err = importer.importClient.DownloadSST(importer.ctx, peer.GetStoreId(), req)
 		if err != nil {
-			return nil, extractDownloadSSTError(err)
+			return nil, errors.Annotatef(errGrpc, "%s", err)
+		}
+		if resp.GetError() != nil {
+			return nil, errors.Annotate(errDownloadFailed, resp.GetError().GetMessage())
 		}
 		if resp.GetIsEmpty() {
 			return nil, errors.Trace(errRangeIsEmpty)
@@ -439,18 +442,3 @@ func checkRegionEpoch(new, old *RegionInfo) bool {
 	}
 	return false
 }
-
-func extractDownloadSSTError(e error) error {
-	err := errGrpc
-	switch {
-	case strings.Contains(e.Error(), "bad format"):
-		err = errBadFormat
-	case strings.Contains(e.Error(), "wrong prefix"):
-		err = errWrongKeyPrefix
-	case strings.Contains(e.Error(), "corrupted"):
-		err = errFileCorrupted
-	case strings.Contains(e.Error(), "Cannot read"):
-		err = errCannotRead
-	}
-	return errors.Annotatef(err, "%s", e)
-}
diff --git a/tests/br_move_backup/run.sh b/tests/br_move_backup/run.sh
index 43f27a9af..b85d25823 100755
--- a/tests/br_move_backup/run.sh
+++ b/tests/br_move_backup/run.sh
@@ -32,6 +32,15 @@ run_sql "DROP TABLE $DB.$TABLE;"
 # change backup path
 mv $TEST_DIR/$DB $TEST_DIR/another$DB
 
+# restore table with old path
+echo "restore with old path start..."
+run_br restore table --db $DB --table $TABLE -s "local://$TEST_DIR/$DB" --pd $PD_ADDR || restore_old_fail=1
+
+if [ "$restore_old_fail" -ne "1" ];then
+    echo "TEST: [$TEST_NAME] test restore with old path failed!"
+    exit 1
+fi
+
 # restore table
 echo "restore start..."
 run_br restore table --db $DB --table $TABLE -s "local://$TEST_DIR/another$DB" --pd $PD_ADDR