feat(hash): new result hash

Dockerfile

@@ -1,6 +1,7 @@
 FROM ubuntu:18.04 as builder
 
-ENV LIBVIPS_VERSION 8.7.3
+# ENV LIBVIPS_VERSION 8.7.3
+ENV LIBVIPS_VERSION 8.6.2
 ENV DEP_VERSION v0.5.0
 ENV GOLANG_VERSION 1.11
 

go.mod

@@ -1,8 +1,7 @@
-module github.com/Aldor007/mort
+module github.com/aldor007/mort
 
 require (
 	github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272
-	github.com/aldor007/mort v0.12.0
 	github.com/aldor007/stow v0.0.0-20190109171535-d59ae5fea5cf
 	github.com/aws/aws-sdk-go v1.16.14
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973

pkg/object/file_object_test.go

@@ -162,6 +162,42 @@ func TestNewFileObjecWithNestedParent(t *testing.T) {
 	assert.Equal(t, "/parent.jpg", parent.Parent.Key, "parent of parent should have correct path")
 }
 
+func TestNewFileObjecWithNestedParentHash(t *testing.T) {
+	mortConfig := config.GetInstance()
+	mortConfig.Load("testdata/bucket-transform-hash.yml")
+	obj, err := NewFileObject(pathToURL("/bucket/width/bucket/height/bucket/parent.jpg"), mortConfig)
+
+	assert.Nil(t, err, "Unexpected to have error when parsing path")
+
+	assert.NotNil(t, obj, "obj should be nil")
+
+	assert.True(t, obj.HasParent(), "obj should have parent")
+
+	parent := obj.Parent
+
+	assert.True(t, parent.HasParent(), "parent should have parent")
+
+	assert.Equal(t, "/6ca/hei/height-bucket-parent.jpg-6ca0dabe9909875a", obj.Key)
+}
+
+func TestNewFileObjecWithNestedParentHashParent(t *testing.T) {
+	mortConfig := config.GetInstance()
+	mortConfig.Load("testdata/bucket-transform-hashParent.yml")
+	obj, err := NewFileObject(pathToURL("/bucket/width/bucket/height/bucket/parent.jpg"), mortConfig)
+
+	assert.Nil(t, err, "Unexpected to have error when parsing path")
+
+	assert.NotNil(t, obj, "obj should be nil")
+
+	assert.True(t, obj.HasParent(), "obj should have parent")
+
+	parent := obj.Parent
+
+	assert.True(t, parent.HasParent(), "parent should have parent")
+
+	assert.Equal(t, "/parent.jpg/545d03ded959a5c0", obj.Key)
+}
+
 func TestNewFileObjectQueryResize(t *testing.T) {
 	mortConfig := &config.Config{}
 	mortConfig.Load("testdata/bucket-transform-query-parent-storage.yml")

pkg/object/preset.go

@@ -70,10 +70,10 @@ func decodePreset(_ *url.URL, bucketConfig config.Bucket, obj *FileObject) (stri
 		parent = "/" + parent
 	}
 
-	if bucketConfig.Transform.ResultKey == "hash" {
-		obj.Key = hashKey(obj.Transforms.Hash(), subMatchMap["parent"])
-		obj.allowChangeKey = false
-	}
+	//if bucketConfig.Transform.ResultKey == "hash" {
+	//	obj.Key = hashKey(obj.Transforms.Hash(), subMatchMap["parent"])
+	//	obj.allowChangeKey = false
+	//}
 
 	return parent, err
 }

pkg/object/query.go

@@ -24,7 +24,6 @@ func decodeQuery(url *url.URL, bucketConfig config.Bucket, obj *FileObject) (str
 			parent = "/" + path.Join(trans.ParentBucket, obj.Key)
 		}
 
-		obj.Key = hashKey(obj.Transforms.Hash(), parent)
 		return parent, err
 	}
 

pkg/object/testdata/bucket-transform-hash.yml

@@ -0,0 +1,33 @@
+buckets:
+    bucket:
+        transform:
+            path: "\\/(?P<presetName>[a-z0-9_]+)\\/(?P<parent>.*)"
+            kind: "presets"
+            resultKey: "hash"
+            presets:
+                blog_small:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            width: 100
+                            height: 100
+                            mode: outbound
+                width:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            width: 100
+                            mode: outbound
+                height:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            height: 100
+                            mode: outbound
+        storages:
+            basic:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+            transform:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"

pkg/object/testdata/bucket-transform-hashParent.yml

@@ -0,0 +1,33 @@
+buckets:
+    bucket:
+        transform:
+            path: "\\/(?P<presetName>[a-z0-9_]+)\\/(?P<parent>.*)"
+            kind: "presets"
+            resultKey: "hashParent"
+            presets:
+                blog_small:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            width: 100
+                            height: 100
+                            mode: outbound
+                width:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            width: 100
+                            mode: outbound
+                height:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            height: 100
+                            mode: outbound
+        storages:
+            basic:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+            transform:
+                kind: "local"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"

pkg/object/uri.go

@@ -2,9 +2,9 @@ package object
 
 import (
 	"bytes"
+	"encoding/hex"
 	"errors"
 	"github.com/aldor007/mort/pkg/config"
-	"hash"
 	"net/url"
 	"strconv"
 	"strings"
@@ -63,8 +63,11 @@ func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error {
 			obj.CheckParent = bucketConfig.Transform.CheckParent
 			if obj.Transforms.NotEmpty {
 				obj.Storage = bucketConfig.Storages.Transform()
-				if obj.allowChangeKey == true && bucketConfig.Transform.ResultKey == "hash" {
-					obj.Key = hashKey(obj.Transforms.Hash(), parentObj.key)
+				switch bucketConfig.Transform.ResultKey {
+				case "hash":
+					obj.Key = hashKey(obj)
+				case "hashParent":
+					obj.Key = hashKeyParent(obj)
 				}
 			} else {
 				obj.Storage = bucketConfig.Storages.Basic()
@@ -80,10 +83,10 @@ func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error {
 	return errors.New("unknown bucket")
 }
 
-func hashKey(h hash.Hash64, suffix string) string {
-	hashB := []byte(strconv.FormatUint(uint64(h.Sum64()), 16))
+func hashKey(obj *FileObject) string {
+	hashB := []byte(strconv.FormatUint(uint64(obj.Transforms.Hash().Sum64()), 16))
 	buf := bufPool.Get().(*bytes.Buffer)
-	safePath := strings.Replace(suffix, "/", "-", -1)
+	safePath := strings.Replace(obj.Parent.key, "/", "-", -1)
 	sliceRange := 3
 
 	if l := len(safePath); l < 3 {
@@ -99,7 +102,32 @@ func hashKey(h hash.Hash64, suffix string) string {
 	buf.WriteString(safePath)
 	buf.WriteByte('-')
 	buf.Write(hashB)
-	defer bufPool.Put(buf)
+	bufPool.Put(buf)
+	return buf.String()
+}
+
+func hashKeyParent(obj *FileObject) string {
+	var currObj *FileObject
+	currObj = obj.Parent
+	transHash := currObj.Transforms.Hash()
+	buf := bufPool.Get().(*bytes.Buffer)
+	for currObj.HasParent() {
+		buf.Reset()
+		hSum := currObj.Transforms.Hash().Sum(nil)
+		buf.WriteString(currObj.Key)
+		buf.Write(hSum)
+		transHash.Sum(buf.Bytes())
+		currObj = currObj.Parent
+	}
+	hashB := transHash.Sum(nil)
+	safePath := strings.Replace(currObj.key, "/", "-", -1)
+
+	buf.Reset()
+	buf.WriteByte('/')
+	buf.WriteString(safePath)
+	buf.WriteByte('/')
+	buf.WriteString(hex.EncodeToString(hashB))
+	bufPool.Put(buf)
 	return buf.String()
 }
 

pkg/processor/plugins/compress_test.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"compress/gzip"
 	"github.com/aldor007/mort/pkg/response"
-	"github.com/stretchr/testify/assert"
 	brEnc "github.com/google/brotli/go/cbrotli"
+	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v2"
 	"net/http"
 	"net/http/httptest"
@@ -245,7 +245,7 @@ func TestCompressBrotliType(t *testing.T) {
 
 	var buf bytes.Buffer
 
-	br := brEnc.NewWriter(&buf, brEnc.WriterOptions{Quality:4})
+	br := brEnc.NewWriter(&buf, brEnc.WriterOptions{Quality: 4})
 	br.Write(body)
 	br.Close()
 

pkg/storage/storage.go

@@ -84,7 +84,6 @@ func Head(obj *object.FileObject) *response.Response {
 	item, err := client.Item(key)
 	if err != nil {
 		if err == stow.ErrNotFound {
-
 			monitoring.Log().Info("Storage/Head item response", obj.LogData(zap.Int("sc", 404))...)
 			return response.NewString(404, notFound)
 		}
@@ -412,7 +411,7 @@ func prepareMetadata(obj *object.FileObject, metaHeaders http.Header) map[string
 		case "s3":
 			keyLower := strings.ToLower(k)
 			if strings.HasPrefix(keyLower, "x-amz-meta") || keyLower == "content-type" {
-				metadata[strings.Replace(strings.ToLower(k), "x-amz-meta-", "", 1)] = v[0]
+				metadata[strings.Replace(keyLower, "x-amz-meta-", "", 1)] = v[0]
 			}
 		default:
 			keyLower := strings.ToLower(k)
@@ -445,10 +444,11 @@ func parseMetadata(obj *object.FileObject, metadata map[string]interface{}, res
 			switch k {
 			case "cache-control", "content-type":
 				res.Set(k, v.(string))
+			default:
+				res.Set(strings.Join([]string{"x-amz-meta", k}, "-"), v.(string))
 
 			}
 
-			res.Set(strings.Join([]string{"x-amz-meta", k}, "-"), v.(string))
 		}
 	}