From bb207a016ecbe59735689aaa02f5565856892351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Tue, 7 Nov 2017 01:14:21 +0200 Subject: [PATCH 1/5] Add LFS object verification step after upload --- modules/lfs/content_store.go | 9 +++++++++ modules/lfs/server.go | 39 ++++++++++++++++++++++++++++++++++++ routers/routes/routes.go | 1 + 3 files changed, 49 insertions(+) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 3e1b2f345b8e..ba9919bcfe6a 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -82,6 +82,15 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { return true } +// Verify returns true if the object exists in the content store and size is correct. +func (s *ContentStore) Verify(meta *models.LFSMetaObject) bool { + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) + if fi, err := os.Stat(path); os.IsNotExist(err) || fi.Size() != meta.Size { + return false + } + return true +} + func transformKey(key string) string { if len(key) < 5 { return key diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 1a15aba0014b..e81c4dba147a 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -69,6 +69,11 @@ func (v *RequestVars) ObjectLink() string { return fmt.Sprintf("%s%s/%s/info/lfs/objects/%s", setting.AppURL, v.User, v.Repo, v.Oid) } +// VerifyLink builds a URL for verifying the object. +func (v *RequestVars) VerifyLink() string { + return fmt.Sprintf("%s%s/%s/info/lfs/verify", setting.AppURL, v.User, v.Repo) +} + // link provides a structure used to build a hypermedia representation of an HTTP link. type link struct { Href string `json:"href"` @@ -320,6 +325,35 @@ func PutHandler(ctx *context.Context) { logRequest(ctx.Req, 200) } +// VerifyHandler verify oid and it's size from the content store +func VerifyHandler(ctx *context.Context) { + + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return + } + + if !ContentMatcher(ctx.Req) { + writeStatus(ctx, 400) + return + } + + rv := unpack(ctx) + + meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true) + if meta == nil { + return + } + + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + if !contentStore.Verify(meta) { + writeStatus(ctx, 404) + return + } + + logRequest(ctx.Req, 200) +} + // Represent takes a RequestVars and Meta and turns it into a Representation suitable // for json encoding func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { @@ -347,6 +381,11 @@ func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload boo rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} } + if upload && !download { + // Force client side verify action while gitea lacks proper server side verification + rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: header} + } + return rep } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index f1c9f184899a..5a76dddb6659 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -681,6 +681,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/objects/:oid/:filename", lfs.ObjectOidHandler) m.Any("/objects/:oid", lfs.ObjectOidHandler) m.Post("/objects", lfs.PostHandler) + m.Post("/verify", lfs.VerifyHandler) m.Any("/*", func(ctx *context.Context) { ctx.Handle(404, "", nil) }) From 4b448a3f88ccf2042b610a8c5ea1f4e22c57689a Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Tue, 7 Nov 2017 08:29:24 +0200 Subject: [PATCH 2/5] Fix file verification condition and small refactor --- modules/lfs/content_store.go | 5 +++-- modules/lfs/server.go | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index ba9919bcfe6a..9c33220b2f3d 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -1,13 +1,14 @@ package lfs import ( - "code.gitea.io/gitea/models" "crypto/sha256" "encoding/hex" "errors" "io" "os" "path/filepath" + + "code.gitea.io/gitea/models" ) var ( @@ -85,7 +86,7 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { // Verify returns true if the object exists in the content store and size is correct. func (s *ContentStore) Verify(meta *models.LFSMetaObject) bool { path := filepath.Join(s.BasePath, transformKey(meta.Oid)) - if fi, err := os.Stat(path); os.IsNotExist(err) || fi.Size() != meta.Size { + if fi, err := os.Stat(path); os.IsNotExist(err) || err == nil && fi.Size() != meta.Size { return false } return true diff --git a/modules/lfs/server.go b/modules/lfs/server.go index e81c4dba147a..5be8c31110c3 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "path" "regexp" "strconv" "strings" @@ -15,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/dgrijalva/jwt-go" "gopkg.in/macaron.v1" ) @@ -66,12 +68,12 @@ type ObjectError struct { // ObjectLink builds a URL linking to the object. func (v *RequestVars) ObjectLink() string { - return fmt.Sprintf("%s%s/%s/info/lfs/objects/%s", setting.AppURL, v.User, v.Repo, v.Oid) + return path.Join(setting.AppURL, v.User, v.Repo, "info/lfs/objects", v.Oid) } // VerifyLink builds a URL for verifying the object. func (v *RequestVars) VerifyLink() string { - return fmt.Sprintf("%s%s/%s/info/lfs/verify", setting.AppURL, v.User, v.Repo) + return path.Join(setting.AppURL, v.User, v.Repo, "info/lfs/verify") } // link provides a structure used to build a hypermedia representation of an HTTP link. @@ -325,7 +327,7 @@ func PutHandler(ctx *context.Context) { logRequest(ctx.Req, 200) } -// VerifyHandler verify oid and it's size from the content store +// VerifyHandler verify oid and its size from the content store func VerifyHandler(ctx *context.Context) { if !setting.LFS.StartServer { From e554e611bd133be9053c67a067ec1969bff92f44 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Tue, 7 Nov 2017 09:22:27 +0200 Subject: [PATCH 3/5] Fix URLs --- modules/lfs/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 5be8c31110c3..aba9550202f4 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -68,12 +68,12 @@ type ObjectError struct { // ObjectLink builds a URL linking to the object. func (v *RequestVars) ObjectLink() string { - return path.Join(setting.AppURL, v.User, v.Repo, "info/lfs/objects", v.Oid) + return setting.AppURL + path.Join(v.User, v.Repo, "info/lfs/objects", v.Oid) } // VerifyLink builds a URL for verifying the object. func (v *RequestVars) VerifyLink() string { - return path.Join(setting.AppURL, v.User, v.Repo, "info/lfs/verify") + return setting.AppURL + path.Join(v.User, v.Repo, "info/lfs/verify") } // link provides a structure used to build a hypermedia representation of an HTTP link. From da21016292c90f4967671505928e2dc20cc5a5c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Wed, 8 Nov 2017 00:19:53 +0200 Subject: [PATCH 4/5] Remove newline and return status 422 on failed verification --- modules/lfs/server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/lfs/server.go b/modules/lfs/server.go index aba9550202f4..b88e6183b95c 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -329,7 +329,6 @@ func PutHandler(ctx *context.Context) { // VerifyHandler verify oid and its size from the content store func VerifyHandler(ctx *context.Context) { - if !setting.LFS.StartServer { writeStatus(ctx, 404) return @@ -349,7 +348,7 @@ func VerifyHandler(ctx *context.Context) { contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} if !contentStore.Verify(meta) { - writeStatus(ctx, 404) + writeStatus(ctx, 422) return } From b54628009c587b6d9e97718ff73f1274cee4e808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Wed, 8 Nov 2017 00:29:32 +0200 Subject: [PATCH 5/5] Better error hadling --- modules/lfs/content_store.go | 13 +++++++++---- modules/lfs/server.go | 8 +++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 9c33220b2f3d..895b51b8ba98 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -84,12 +84,17 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { } // Verify returns true if the object exists in the content store and size is correct. -func (s *ContentStore) Verify(meta *models.LFSMetaObject) bool { +func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { path := filepath.Join(s.BasePath, transformKey(meta.Oid)) - if fi, err := os.Stat(path); os.IsNotExist(err) || err == nil && fi.Size() != meta.Size { - return false + + fi, err := os.Stat(path) + if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size { + return false, nil + } else if err != nil { + return false, err } - return true + + return true, nil } func transformKey(key string) string { diff --git a/modules/lfs/server.go b/modules/lfs/server.go index b88e6183b95c..121aeea9eb44 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -347,7 +347,13 @@ func VerifyHandler(ctx *context.Context) { } contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} - if !contentStore.Verify(meta) { + ok, err := contentStore.Verify(meta) + if err != nil { + ctx.Resp.WriteHeader(500) + fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) + return + } + if !ok { writeStatus(ctx, 422) return }