Skip to content

Commit

Permalink
perf: hide GetObjectResumableUploadOffset interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@constwz
constwz committed Jan 24, 2024
1 parent fcbe7c4 commit 9f501a0
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 119 deletions.
8 changes: 4 additions & 4 deletions client/api_object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ type IObjectClient interface {
ComputeHashRoots(reader io.Reader, isSerial bool) ([][]byte, int64, storageTypes.RedundancyType, error)
CreateFolder(ctx context.Context, bucketName, objectName string, opts types.CreateObjectOptions) (string, error)
GetObjectUploadProgress(ctx context.Context, bucketName, objectName string) (string, error)
GetObjectResumableUploadOffset(ctx context.Context, bucketName, objectName string) (uint64, error)
//GetObjectResumableUploadOffset(ctx context.Context, bucketName, objectName string) (uint64, error)
ListObjectsByObjectID(ctx context.Context, objectIds []uint64, opts types.EndPointOptions) (types.ListObjectsByObjectIDResponse, error)
ListObjectPolicies(ctx context.Context, objectName, bucketName string, actionType uint32, opts types.ListObjectPoliciesOptions) (types.ListObjectPoliciesResponse, error)
}
Expand Down Expand Up @@ -319,7 +319,7 @@ func (c *Client) putObjectResumable(ctx context.Context, bucketName, objectName
return err
}

offset, err := c.GetObjectResumableUploadOffset(ctx, bucketName, objectName)
offset, err := c.getObjectResumableUploadOffset(ctx, bucketName, objectName)
if err != nil {
return err
}
Expand Down Expand Up @@ -1069,8 +1069,8 @@ func (c *Client) GetObjectUploadProgress(ctx context.Context, bucketName, object
return status.ObjectInfo.ObjectStatus.String(), nil
}

// GetObjectResumableUploadOffset return the status of object including the uploading progress
func (c *Client) GetObjectResumableUploadOffset(ctx context.Context, bucketName, objectName string) (uint64, error) {
// getObjectResumableUploadOffset return the status of object including the uploading progress
func (c *Client) getObjectResumableUploadOffset(ctx context.Context, bucketName, objectName string) (uint64, error) {
status, err := c.HeadObject(ctx, bucketName, objectName)
if err != nil {
return 0, err
Expand Down
230 changes: 115 additions & 115 deletions e2e/e2e_storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,121 +458,121 @@ func (s *StorageTestSuite) TruncateDownloadTempFileToLessPartsize() {
s.Require().NoError(err)
}

func (s *StorageTestSuite) Test_Resumable_Upload_And_Download() {
// 1) create big object without putobject
bucketName, objectName, buffer := s.createBigObjectWithoutPutObject()

s.T().Log("---> Resumable PutObject <---")
partSize16MB := uint64(1024 * 1024 * 16)
// 2) put a big object, the secondary segment will error, then resumable upload
client.UploadSegmentHooker = UploadErrorHooker
err := s.Client.PutObject(s.ClientContext, bucketName, objectName, int64(buffer.Len()),
bytes.NewReader(buffer.Bytes()), types.PutObjectOptions{PartSize: partSize16MB})
s.Require().ErrorContains(err, "UploadErrorHooker")
client.UploadSegmentHooker = client.DefaultUploadSegment
offset, err := s.Client.GetObjectResumableUploadOffset(s.ClientContext, bucketName, objectName)
s.Require().NoError(err)
s.Require().Equal(offset, partSize16MB)

err = s.Client.PutObject(s.ClientContext, bucketName, objectName, int64(buffer.Len()),
bytes.NewReader(buffer.Bytes()), types.PutObjectOptions{PartSize: partSize16MB})
s.Require().NoError(err)

s.WaitSealObject(bucketName, objectName)

// 3) FGetObjectResumable compare with FGetObject
fileName := "test-file-" + storageTestUtil.GenRandomObjectName()
defer os.Remove(fileName)
err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, fileName, types.GetObjectOptions{PartSize: 32 * 1024 * 1024})
s.T().Logf("---> object file :%s <---", fileName)
s.Require().NoError(err)

fGetObjectFileName := "test-file-" + storageTestUtil.GenRandomObjectName()
defer os.Remove(fGetObjectFileName)
s.T().Logf("---> object file :%s <---", fGetObjectFileName)
err = s.Client.FGetObject(s.ClientContext, bucketName, objectName, fGetObjectFileName, types.GetObjectOptions{})
s.Require().NoError(err)

isSame, err := types.CompareFiles(fileName, fGetObjectFileName)
s.Require().True(isSame)
s.Require().NoError(err)

// 4) Resumabledownload, download a file with default checkpoint
client.DownloadSegmentHooker = DownloadErrorHooker
resumableDownloadFile := storageTestUtil.GenRandomObjectName()
defer os.Remove(resumableDownloadFile)
s.T().Logf("---> Resumable download Create newfile:%s, <---", resumableDownloadFile)

err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
s.Require().ErrorContains(err, "DownloadErrorHooker")
client.DownloadSegmentHooker = client.DefaultDownloadSegmentHook

err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
s.Require().NoError(err)
// download success, checkpoint file has been deleted

isSame, err = types.CompareFiles(resumableDownloadFile, fGetObjectFileName)
s.Require().True(isSame)
s.Require().NoError(err)

// when the downloaded file size is less than a part size
client.DownloadSegmentHooker = DownloadErrorHooker
resumableDownloadLessPartFile := storageTestUtil.GenRandomObjectName()
defer os.Remove(resumableDownloadLessPartFile)
s.T().Logf("---> Resumable download for less part size , Create newfile:%s, <---", resumableDownloadLessPartFile)

err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadLessPartFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
s.Require().ErrorContains(err, "DownloadErrorHooker")

s.TruncateDownloadTempFileToLessPartsize()

client.DownloadSegmentHooker = client.DefaultDownloadSegmentHook

err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadLessPartFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
s.Require().NoError(err)
// download success, checkpoint file has been deleted

isSame, err = types.CompareFiles(resumableDownloadLessPartFile, fGetObjectFileName)
s.Require().True(isSame)
s.Require().NoError(err)

// 5) Resumabledownload, download a file with range
s.T().Logf("---> Resumabledownload, download a file with range <---")
rangeOptions := types.GetObjectOptions{Range: "bytes=1000-94131999", PartSize: partSize16MB}
resumableDownloadWithRangeFile := "test-file-" + storageTestUtil.GenRandomObjectName()
defer os.Remove(resumableDownloadWithRangeFile)
err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadWithRangeFile, rangeOptions)
s.T().Logf("---> object file :%s <---", resumableDownloadWithRangeFile)
s.Require().NoError(err)

fGetObjectWithRangeFile := "test-file-" + storageTestUtil.GenRandomObjectName()
defer os.Remove(fGetObjectWithRangeFile)
s.T().Logf("---> object file :%s <---", fGetObjectWithRangeFile)
err = s.Client.FGetObject(s.ClientContext, bucketName, objectName, fGetObjectWithRangeFile, rangeOptions)
s.Require().NoError(err)

isSame, err = types.CompareFiles(resumableDownloadWithRangeFile, fGetObjectWithRangeFile)
s.Require().True(isSame)
s.Require().NoError(err)

// 6) Resumabledownload, download a file with range and Truncate
s.T().Logf("---> Resumabledownload, download a file with range and Truncate <---")
rDownloadTruncateFile := "test-file-" + storageTestUtil.GenRandomObjectName()
defer os.Remove(rDownloadTruncateFile)
client.DownloadSegmentHooker = DownloadErrorHooker
err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, rDownloadTruncateFile, rangeOptions)
s.T().Logf("---> object file :%s <---", rDownloadTruncateFile)
s.Require().ErrorContains(err, "DownloadErrorHooker")
s.TruncateDownloadTempFileToLessPartsize()

client.DownloadSegmentHooker = client.DefaultDownloadSegmentHook
err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, rDownloadTruncateFile, rangeOptions)
s.Require().NoError(err)

isSame, err = types.CompareFiles(rDownloadTruncateFile, fGetObjectWithRangeFile)
s.Require().True(isSame)
s.Require().NoError(err)
}
//func (s *StorageTestSuite) Test_Resumable_Upload_And_Download() {
// // 1) create big object without putobject
// bucketName, objectName, buffer := s.createBigObjectWithoutPutObject()
//
// s.T().Log("---> Resumable PutObject <---")
// partSize16MB := uint64(1024 * 1024 * 16)
// // 2) put a big object, the secondary segment will error, then resumable upload
// client.UploadSegmentHooker = UploadErrorHooker
// err := s.Client.PutObject(s.ClientContext, bucketName, objectName, int64(buffer.Len()),
// bytes.NewReader(buffer.Bytes()), types.PutObjectOptions{PartSize: partSize16MB})
// s.Require().ErrorContains(err, "UploadErrorHooker")
// client.UploadSegmentHooker = client.DefaultUploadSegment
// offset, err := s.Client.getObjectResumableUploadOffset(s.ClientContext, bucketName, objectName)
// s.Require().NoError(err)
// s.Require().Equal(offset, partSize16MB)
//
// err = s.Client.PutObject(s.ClientContext, bucketName, objectName, int64(buffer.Len()),
// bytes.NewReader(buffer.Bytes()), types.PutObjectOptions{PartSize: partSize16MB})
// s.Require().NoError(err)
//
// s.WaitSealObject(bucketName, objectName)
//
// // 3) FGetObjectResumable compare with FGetObject
// fileName := "test-file-" + storageTestUtil.GenRandomObjectName()
// defer os.Remove(fileName)
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, fileName, types.GetObjectOptions{PartSize: 32 * 1024 * 1024})
// s.T().Logf("---> object file :%s <---", fileName)
// s.Require().NoError(err)
//
// fGetObjectFileName := "test-file-" + storageTestUtil.GenRandomObjectName()
// defer os.Remove(fGetObjectFileName)
// s.T().Logf("---> object file :%s <---", fGetObjectFileName)
// err = s.Client.FGetObject(s.ClientContext, bucketName, objectName, fGetObjectFileName, types.GetObjectOptions{})
// s.Require().NoError(err)
//
// isSame, err := types.CompareFiles(fileName, fGetObjectFileName)
// s.Require().True(isSame)
// s.Require().NoError(err)
//
// // 4) Resumabledownload, download a file with default checkpoint
// client.DownloadSegmentHooker = DownloadErrorHooker
// resumableDownloadFile := storageTestUtil.GenRandomObjectName()
// defer os.Remove(resumableDownloadFile)
// s.T().Logf("---> Resumable download Create newfile:%s, <---", resumableDownloadFile)
//
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
// s.Require().ErrorContains(err, "DownloadErrorHooker")
// client.DownloadSegmentHooker = client.DefaultDownloadSegmentHook
//
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
// s.Require().NoError(err)
// // download success, checkpoint file has been deleted
//
// isSame, err = types.CompareFiles(resumableDownloadFile, fGetObjectFileName)
// s.Require().True(isSame)
// s.Require().NoError(err)
//
// // when the downloaded file size is less than a part size
// client.DownloadSegmentHooker = DownloadErrorHooker
// resumableDownloadLessPartFile := storageTestUtil.GenRandomObjectName()
// defer os.Remove(resumableDownloadLessPartFile)
// s.T().Logf("---> Resumable download for less part size , Create newfile:%s, <---", resumableDownloadLessPartFile)
//
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadLessPartFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
// s.Require().ErrorContains(err, "DownloadErrorHooker")
//
// s.TruncateDownloadTempFileToLessPartsize()
//
// client.DownloadSegmentHooker = client.DefaultDownloadSegmentHook
//
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadLessPartFile, types.GetObjectOptions{PartSize: 16 * 1024 * 1024})
// s.Require().NoError(err)
// // download success, checkpoint file has been deleted
//
// isSame, err = types.CompareFiles(resumableDownloadLessPartFile, fGetObjectFileName)
// s.Require().True(isSame)
// s.Require().NoError(err)
//
// // 5) Resumabledownload, download a file with range
// s.T().Logf("---> Resumabledownload, download a file with range <---")
// rangeOptions := types.GetObjectOptions{Range: "bytes=1000-94131999", PartSize: partSize16MB}
// resumableDownloadWithRangeFile := "test-file-" + storageTestUtil.GenRandomObjectName()
// defer os.Remove(resumableDownloadWithRangeFile)
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, resumableDownloadWithRangeFile, rangeOptions)
// s.T().Logf("---> object file :%s <---", resumableDownloadWithRangeFile)
// s.Require().NoError(err)
//
// fGetObjectWithRangeFile := "test-file-" + storageTestUtil.GenRandomObjectName()
// defer os.Remove(fGetObjectWithRangeFile)
// s.T().Logf("---> object file :%s <---", fGetObjectWithRangeFile)
// err = s.Client.FGetObject(s.ClientContext, bucketName, objectName, fGetObjectWithRangeFile, rangeOptions)
// s.Require().NoError(err)
//
// isSame, err = types.CompareFiles(resumableDownloadWithRangeFile, fGetObjectWithRangeFile)
// s.Require().True(isSame)
// s.Require().NoError(err)
//
// // 6) Resumabledownload, download a file with range and Truncate
// s.T().Logf("---> Resumabledownload, download a file with range and Truncate <---")
// rDownloadTruncateFile := "test-file-" + storageTestUtil.GenRandomObjectName()
// defer os.Remove(rDownloadTruncateFile)
// client.DownloadSegmentHooker = DownloadErrorHooker
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, rDownloadTruncateFile, rangeOptions)
// s.T().Logf("---> object file :%s <---", rDownloadTruncateFile)
// s.Require().ErrorContains(err, "DownloadErrorHooker")
// s.TruncateDownloadTempFileToLessPartsize()
//
// client.DownloadSegmentHooker = client.DefaultDownloadSegmentHook
// err = s.Client.FGetObjectResumable(s.ClientContext, bucketName, objectName, rDownloadTruncateFile, rangeOptions)
// s.Require().NoError(err)
//
// isSame, err = types.CompareFiles(rDownloadTruncateFile, fGetObjectWithRangeFile)
// s.Require().True(isSame)
// s.Require().NoError(err)
//}

func (s *StorageTestSuite) Test_Upload_Object_With_Tampering_Content() {
bucketName := storageTestUtil.GenRandomBucketName()
Expand Down

0 comments on commit 9f501a0

Please sign in to comment.