Modify/fmt leaf hash calculation

code/go/0chain.net/blobbercore/allocation/common_test.go

@@ -2,12 +2,10 @@ package allocation
 
 import (
 	"bytes"
-	"encoding/json"
 	"io"
 	"mime/multipart"
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
-	"github.com/0chain/gosdk/core/util"
 )
 
 type MockFileStore struct {
@@ -23,11 +21,11 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string,
 	b := bytes.NewBuffer(make([]byte, 0))
 	n, _ := io.Copy(b, infile)
 	return &filestore.FileOutputData{
-		Name:        fileData.Name,
-		Path:        fileData.Path,
-		MerkleRoot:  "",
-		ContentHash: fileData.Hash,
-		Size:        n,
+		Name:            fileData.Name,
+		Path:            fileData.Path,
+		FixedMerkleRoot: "",
+		ValidationRoot:  fileData.ValidationRoot,
+		Size:            n,
 	}, nil
 }
 
@@ -43,12 +41,12 @@ func (mfs *MockFileStore) DeleteFile(allocID string, contentHash string) error {
 	return nil
 }
 
-func (mfs *MockFileStore) GetFileBlock(allocID string, fileData *filestore.FileInputData, blockNum int64, numBlocks int64) ([]byte, error) {
+func (mfs *MockFileStore) GetFileBlock(rin *filestore.ReadBlockInput) (*filestore.FileDownloadResponse, error) {
 	return nil, nil
 }
 
-func (mfs *MockFileStore) GetBlocksMerkleTreeForChallenge(allocID string, fileData *filestore.FileInputData, blockoffset int) (json.RawMessage, util.MerkleTreeI, error) {
-	return nil, nil, nil
+func (mfs *MockFileStore) GetBlocksMerkleTreeForChallenge(cir *filestore.ChallengeReadBlockInput) (*filestore.ChallengeResponse, error) {
+	return nil, nil
 }
 
 func (mfs *MockFileStore) GetTotalTempFileSizes() (s uint64) {

code/go/0chain.net/blobbercore/allocation/copyfilechange.go

@@ -111,8 +111,10 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
 
 func (rf *CopyFileChange) processCopyRefs(
 	ctx context.Context, srcRef, destRef *reference.Ref,
-	allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (
-	fileRefs []*reference.Ref, err error) {
+	allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string,
+) (
+	fileRefs []*reference.Ref, err error,
+) {
 
 	newRef := *srcRef
 	newRef.ID = 0

code/go/0chain.net/blobbercore/allocation/copyfilechange_test.go

@@ -71,70 +71,70 @@ func TestBlobberCore_CopyFile(t *testing.T) {
 				mocket.Catcher.NewMock().WithQuery(query).WithReply(
 					[]map[string]interface{}{
 						{
-							"id":             1,
-							"level":          0,
-							"lookup_hash":    "lookup_hash_root",
-							"path":           "/",
-							"name":           "/",
-							"allocation_id":  alloc.ID,
-							"parent_path":    "",
-							"content_hash":   "",
-							"thumbnail_size": 00,
-							"thumbnail_hash": "",
-							"type":           reference.DIRECTORY,
-							"created_at":     common.Now() - 3600,
-							"updated_at":     common.Now() - 1800,
+							"id":              1,
+							"level":           0,
+							"lookup_hash":     "lookup_hash_root",
+							"path":            "/",
+							"name":            "/",
+							"allocation_id":   alloc.ID,
+							"parent_path":     "",
+							"validation_root": "",
+							"thumbnail_size":  00,
+							"thumbnail_hash":  "",
+							"type":            reference.DIRECTORY,
+							"created_at":      common.Now() - 3600,
+							"updated_at":      common.Now() - 1800,
 						},
 						{
-							"id":             2,
-							"level":          1,
-							"lookup_hash":    "lookup_hash",
-							"path":           "/orig.txt",
-							"name":           "orig.txt",
-							"allocation_id":  alloc.ID,
-							"parent_path":    "/",
-							"content_hash":   "content_hash",
-							"thumbnail_size": 00,
-							"thumbnail_hash": "",
-							"type":           reference.FILE,
-							"created_at":     common.Now() - 3600,
-							"updated_at":     common.Now() - 1800,
+							"id":              2,
+							"level":           1,
+							"lookup_hash":     "lookup_hash",
+							"path":            "/orig.txt",
+							"name":            "orig.txt",
+							"allocation_id":   alloc.ID,
+							"parent_path":     "/",
+							"validation_root": "validation_root",
+							"thumbnail_size":  00,
+							"thumbnail_hash":  "",
+							"type":            reference.FILE,
+							"created_at":      common.Now() - 3600,
+							"updated_at":      common.Now() - 1800,
 						},
 					},
 				)
 
-				q2 := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","file_meta_hash","path_hash","content_hash","merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","allocation_root","level","created_at","updated_at" FROM "reference_objects" WHERE ((allocation_id=$1 AND parent_path=$2) OR (parent_path = $3 AND allocation_id = $4)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path`
+				q2 := `SELECT "id","allocation_id","type","name","path","parent_path","size","hash","file_meta_hash","path_hash","validation_root","fixed_merkle_root","actual_file_size","actual_file_hash","chunk_size","lookup_hash","thumbnail_hash","allocation_root","level","created_at","updated_at" FROM "reference_objects" WHERE ((allocation_id=$1 AND parent_path=$2) OR (parent_path = $3 AND allocation_id = $4)) AND "reference_objects"."deleted_at" IS NULL ORDER BY path`
 				mocket.Catcher.NewMock().WithQuery(q2).WithReply(
 					[]map[string]interface{}{
 						{
-							"id":             1,
-							"level":          1,
-							"lookup_hash":    "lookup_hash_root",
-							"path":           "/",
-							"name":           "/",
-							"allocation_id":  alloc.ID,
-							"parent_path":    "",
-							"content_hash":   "",
-							"thumbnail_size": 00,
-							"thumbnail_hash": "",
-							"type":           reference.DIRECTORY,
-							"created_at":     common.Now() - 3600,
-							"updated_at":     common.Now() - 1800,
+							"id":              1,
+							"level":           1,
+							"lookup_hash":     "lookup_hash_root",
+							"path":            "/",
+							"name":            "/",
+							"allocation_id":   alloc.ID,
+							"parent_path":     "",
+							"validation_root": "",
+							"thumbnail_size":  00,
+							"thumbnail_hash":  "",
+							"type":            reference.DIRECTORY,
+							"created_at":      common.Now() - 3600,
+							"updated_at":      common.Now() - 1800,
 						},
 						{
-							"id":             2,
-							"level":          2,
-							"lookup_hash":    "lookup_hash",
-							"path":           "/orig.txt",
-							"name":           "orig.txt",
-							"allocation_id":  alloc.ID,
-							"parent_path":    "/",
-							"content_hash":   "content_hash",
-							"thumbnail_size": 00,
-							"thumbnail_hash": "",
-							"type":           reference.FILE,
-							"created_at":     common.Now() - 3600,
-							"updated_at":     common.Now() - 1800,
+							"id":              2,
+							"level":           2,
+							"lookup_hash":     "lookup_hash",
+							"path":            "/orig.txt",
+							"name":            "orig.txt",
+							"allocation_id":   alloc.ID,
+							"parent_path":     "/",
+							"validation_root": "validation_root",
+							"thumbnail_size":  00,
+							"thumbnail_hash":  "",
+							"type":            reference.FILE,
+							"created_at":      common.Now() - 3600,
+							"updated_at":      common.Now() - 1800,
 						},
 					},
 				)

code/go/0chain.net/blobbercore/allocation/deletefilechange.go

@@ -65,24 +65,24 @@ func (nf *DeleteFileChange) DeleteTempFile() error {
 func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error {
 	db := datastore.GetStore().GetTransaction(ctx)
 	type Result struct {
-		Id            string
-		ContentHash   string
-		ThumbnailHash string
+		Id             string
+		ValidationRoot string
+		ThumbnailHash  string
 	}
 
 	limitCh := make(chan struct{}, 10)
 	wg := &sync.WaitGroup{}
 	var results []Result
 	err := db.Model(&reference.Ref{}).Unscoped().
-		Select("id", "content_hash", "thumbnail_hash").
+		Select("id", "validation_root", "thumbnail_hash").
 		Where("allocation_id=? AND path LIKE ? AND type=? AND deleted_at is not NULL",
 			nf.AllocationID, nf.Path+"%", reference.FILE).
 		FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error {
 
 			for _, res := range results {
 				var count int64
 				tx.Model(&reference.Ref{}).
-					Where("allocation_id=? AND content_hash=?", nf.AllocationID, res.ContentHash).
+					Where("allocation_id=? AND validation_root=?", nf.AllocationID, res.ValidationRoot).
 					Count(&count)
 
 				db.Model(&reference.CommitMetaTxn{}).Delete("ref_id = ?", res.Id)
@@ -101,10 +101,10 @@ func (nf *DeleteFileChange) CommitToFileStore(ctx context.Context) error {
 					}()
 
 					if count == 0 {
-						err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ContentHash)
+						err := filestore.GetFileStore().DeleteFile(nf.AllocationID, res.ValidationRoot)
 						if err != nil {
 							logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()),
-								zap.String("content_hash", res.ContentHash))
+								zap.String("validation_root", res.ValidationRoot))
 						}
 					}
 

code/go/0chain.net/blobbercore/allocation/file_changer_base.go

@@ -17,6 +17,8 @@ type BaseFileChanger struct {
 	//client side:
 	Path string `json:"filepath" validation:"required"`
 	//client side:
+	ActualFileHashSignature string `json:"actual_file_hash_signature,omitempty" validation:"required"`
+	//client side:
 	ActualHash string `json:"actual_hash,omitempty" validation:"required"`
 	//client side:
 	ActualSize int64 `json:"actual_size,omitempty" validation:"required"`
@@ -29,13 +31,15 @@ type BaseFileChanger struct {
 	MimeType string `json:"mimetype,omitempty"`
 	//client side:
 	//client side:
-	MerkleRoot string `json:"merkle_root,omitempty"`
+	FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"`
 
 	//server side: update them by ChangeProcessor
 	AllocationID string `json:"allocation_id"`
 	//client side:
-	Hash string `json:"content_hash,omitempty"`
-	Size int64  `json:"size"`
+	ValidationRootSignature string `json:"validation_root_signature,omitempty"`
+	//client side:
+	ValidationRoot string `json:"validation_root,omitempty"`
+	Size           int64  `json:"size"`
 	//server side:
 	ThumbnailHash     string `json:"thumbnail_content_hash,omitempty"`
 	ThumbnailSize     int64  `json:"thumbnail_size"`
@@ -57,13 +61,13 @@ func (fc *BaseFileChanger) DeleteTempFile() error {
 	fileInputData := &filestore.FileInputData{}
 	fileInputData.Name = fc.Filename
 	fileInputData.Path = fc.Path
-	fileInputData.Hash = fc.Hash
+	fileInputData.ValidationRoot = fc.ValidationRoot
 	err := filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData)
 	if fc.ThumbnailSize > 0 {
 		fileInputData := &filestore.FileInputData{}
 		fileInputData.Name = fc.ThumbnailFilename
 		fileInputData.Path = fc.Path
-		fileInputData.Hash = fc.ThumbnailHash
+		fileInputData.ThumbnailHash = fc.ThumbnailHash
 		err = filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData)
 	}
 	return err
@@ -73,8 +77,8 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error {
 	fileInputData := &filestore.FileInputData{}
 	fileInputData.Name = fc.Filename
 	fileInputData.Path = fc.Path
-	fileInputData.Hash = fc.Hash
-	fileInputData.MerkleRoot = fc.MerkleRoot
+	fileInputData.ValidationRoot = fc.ValidationRoot
+	fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot
 	fileInputData.ChunkSize = fc.ChunkSize
 	_, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData)
 	if err != nil {
@@ -84,7 +88,7 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context) error {
 		fileInputData := &filestore.FileInputData{}
 		fileInputData.Name = fc.ThumbnailFilename
 		fileInputData.Path = fc.Path
-		fileInputData.Hash = fc.ThumbnailHash
+		fileInputData.ThumbnailHash = fc.ThumbnailHash
 		fileInputData.ChunkSize = fc.ChunkSize
 		fileInputData.IsThumbnail = true
 		_, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData)

code/go/0chain.net/blobbercore/allocation/file_changer_update.go

@@ -78,15 +78,17 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
 	if fileRef.ThumbnailHash != "" && fileRef.ThumbnailHash != nf.ThumbnailHash {
 		nf.deleteHash[fileRef.ThumbnailHash] = true
 	}
-	if fileRef.ContentHash != "" && fileRef.ContentHash != nf.Hash {
-		nf.deleteHash[fileRef.ContentHash] = true
+	if fileRef.ValidationRoot != "" && fileRef.ValidationRoot != nf.ValidationRoot {
+		nf.deleteHash[fileRef.ValidationRoot] = true
 	}
 	fileRef.ActualFileHash = nf.ActualHash
+	fileRef.ActualFileHashSignature = nf.ActualFileHashSignature
 	fileRef.ActualFileSize = nf.ActualSize
 	fileRef.MimeType = nf.MimeType
-	fileRef.ContentHash = nf.Hash
+	fileRef.ValidationRootSignature = nf.ValidationRootSignature
+	fileRef.ValidationRoot = nf.ValidationRoot
 	fileRef.CustomMeta = nf.CustomMeta
-	fileRef.MerkleRoot = nf.MerkleRoot
+	fileRef.FixedMerkleRoot = nf.FixedMerkleRoot
 	fileRef.AllocationRoot = allocationRoot
 	fileRef.Size = nf.Size
 	fileRef.ThumbnailHash = nf.ThumbnailHash
@@ -108,18 +110,18 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
 
 func (nf *UpdateFileChanger) CommitToFileStore(ctx context.Context) error {
 	db := datastore.GetStore().GetTransaction(ctx)
-	for contenthash := range nf.deleteHash {
+	for hash := range nf.deleteHash {
 		var count int64
 		err := db.Table((&reference.Ref{}).TableName()).
 			Where(
-				db.Where(&reference.Ref{ThumbnailHash: contenthash}).
-					Or(&reference.Ref{ContentHash: contenthash})).
+				db.Where(&reference.Ref{ThumbnailHash: hash}).
+					Or(&reference.Ref{ValidationRoot: hash})).
 			Where(&reference.Ref{AllocationID: nf.AllocationID}).
 			Count(&count).Error
 
 		if err == nil && count == 0 {
-			logging.Logger.Info("Deleting content file", zap.String("content_hash", contenthash))
-			if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, contenthash); err != nil {
+			logging.Logger.Info("Deleting content file", zap.String("validation_root", hash))
+			if err := filestore.GetFileStore().DeleteFile(nf.AllocationID, hash); err != nil {
 				logging.Logger.Error("FileStore_DeleteFile", zap.String("allocation_id", nf.AllocationID), zap.Error(err))
 			}
 		}

code/go/0chain.net/blobbercore/allocation/file_changer_upload.go

@@ -84,28 +84,30 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *Allocation
 	}
 
 	newFile := &reference.Ref{
-		ActualFileHash:      nf.ActualHash,
-		ActualFileSize:      nf.ActualSize,
-		AllocationID:        dirRef.AllocationID,
-		ContentHash:         nf.Hash,
-		CustomMeta:          nf.CustomMeta,
-		MerkleRoot:          nf.MerkleRoot,
-		Name:                nf.Filename,
-		Path:                nf.Path,
-		ParentPath:          dirRef.Path,
-		Type:                reference.FILE,
-		Size:                nf.Size,
-		MimeType:            nf.MimeType,
-		AllocationRoot:      allocationRoot,
-		ThumbnailHash:       nf.ThumbnailHash,
-		ThumbnailSize:       nf.ThumbnailSize,
-		ActualThumbnailHash: nf.ActualThumbnailHash,
-		ActualThumbnailSize: nf.ActualThumbnailSize,
-		EncryptedKey:        nf.EncryptedKey,
-		ChunkSize:           nf.ChunkSize,
-		CreatedAt:           ts,
-		UpdatedAt:           ts,
-		HashToBeComputed:    true,
+		ActualFileHash:          nf.ActualHash,
+		ActualFileHashSignature: nf.ActualFileHashSignature,
+		ActualFileSize:          nf.ActualSize,
+		AllocationID:            dirRef.AllocationID,
+		ValidationRoot:          nf.ValidationRoot,
+		ValidationRootSignature: nf.ValidationRootSignature,
+		CustomMeta:              nf.CustomMeta,
+		FixedMerkleRoot:         nf.FixedMerkleRoot,
+		Name:                    nf.Filename,
+		Path:                    nf.Path,
+		ParentPath:              dirRef.Path,
+		Type:                    reference.FILE,
+		Size:                    nf.Size,
+		MimeType:                nf.MimeType,
+		AllocationRoot:          allocationRoot,
+		ThumbnailHash:           nf.ThumbnailHash,
+		ThumbnailSize:           nf.ThumbnailSize,
+		ActualThumbnailHash:     nf.ActualThumbnailHash,
+		ActualThumbnailSize:     nf.ActualThumbnailSize,
+		EncryptedKey:            nf.EncryptedKey,
+		ChunkSize:               nf.ChunkSize,
+		CreatedAt:               ts,
+		UpdatedAt:               ts,
+		HashToBeComputed:        true,
 	}
 
 	fileID, ok := fileIDMeta[newFile.Path]