log-backup: truncate metadata streamingly (#38870) (#38972)

close #38869, ref #38939, ref #38940
pingcap · Nov 9, 2022 · d662237 · d662237
1 parent 274d9c5
commit d662237
Show file tree

Hide file tree

Showing 5 changed files with 2,198 additions and 311 deletions.
diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel
@@ -149,7 +149,6 @@ go_test(
         "//testkit/testsetup",
         "//types",
         "//util/codec",
-        "//util/mathutil",
         "@com_github_fsouza_fake_gcs_server//fakestorage",
         "@com_github_golang_protobuf//proto",
         "@com_github_pingcap_errors//:errors",

diff --git a/br/pkg/restore/log_client_test.go b/br/pkg/restore/log_client_test.go
@@ -300,12 +300,19 @@ func testReadFromMetadataWithVersion(t *testing.T, m metaMaker) {
 
 		meta := new(StreamMetadataSet)
 		meta.Helper = stream.NewMetadataHelper()
-		meta.LoadUntil(ctx, loc, c.untilTS)
+		meta.LoadUntilAndCalculateShiftTS(ctx, loc, c.untilTS)
 
 		var metas []*backuppb.Metadata
-		for _, m := range meta.metadata {
+		for path := range meta.metadataInfos {
+			data, err := loc.ReadFile(ctx, path)
+			require.NoError(t, err)
+
+			m, err := meta.Helper.ParseToMetadataHard(data)
+			require.NoError(t, err)
+
 			metas = append(metas, m)
 		}
+
 		actualStoreIDs := make([]int64, 0, len(metas))
 		for _, meta := range metas {
 			actualStoreIDs = append(actualStoreIDs, meta.StoreId)