From dc9d46cd61ed7661400d06ddd62ebddb7a44cf6d Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 28 Jul 2022 16:57:44 +0200 Subject: [PATCH 01/43] Added first draft for a raw json local file storage. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 client/filejsonstore/filejsonstore.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go new file mode 100644 index 00000000..1a963112 --- /dev/null +++ b/client/filejsonstore/filejsonstore.go @@ -0,0 +1,95 @@ +package filejsonstore + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +type FileJSONStore struct { + f *os.File + baseDir string +} + +func New(baseDir string) (*FileJSONStore, error) { + return newImpl(baseDir, true) +} + +func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { + var f = FileJSONStore{ + baseDir: baseDir, + } + var err error + + if f.f, err = os.Open(baseDir); err != nil { + pe, ok := err.(*os.PathError) + fmt.Println(ok) + fmt.Println(pe.Err) + if ok && errors.Is(pe.Err, os.ErrNotExist) && recurse { + // Create the directory + err = os.Mkdir(baseDir, 0750) + if err == nil { + return newImpl(baseDir, false) + } + return nil, err + } else { + return nil, err + } + } + + if stat, err := f.f.Stat(); err != nil { + f.f.Close() + return nil, fmt.Errorf("failed to stat file %s: %w", + baseDir, err) + } else { + if !stat.IsDir() { + f.f.Close() + return nil, fmt.Errorf("can not open %s, not a directory", + baseDir) + } + } + + return &f, nil +} + +func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { + names, err := f.f.Readdirnames(0) + + if err != nil { + return nil, err + } + + meta := map[string]json.RawMessage{} + for _, name := range names { + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + b, err := os.ReadFile(p) + if err != nil { + return nil, err + } + meta[name] = b + } + + return meta, nil +} + +func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + return os.WriteFile(p, meta, 0640) +} + +func (f *FileJSONStore) DeleteMeta(name string) error { + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + return os.Remove(p) +} + +func (f *FileJSONStore) Close() error { + if f == nil { + return nil + } + if f.f != nil { + return f.f.Close() + } + return nil +} From c41bd2d9adc1ddaea2e41d67023f54f961201a26 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 28 Jul 2022 16:58:23 +0200 Subject: [PATCH 02/43] Ignore emacs temporary files Signed-off-by: Fredrik Skogman --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 64b8f8a8..39e7149c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ cmd/tuf/tuf cmd/tuf-client/tuf-client .vscode +*~ From 9fdf33211bc51850741257d2fe902f5ba9002c3e Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 13:35:49 +0200 Subject: [PATCH 03/43] moved isMetaFile to util directory for reuse in other packages. Signed-off-by: Fredrik Skogman --- local_store.go | 17 ++--------------- util/util.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/local_store.go b/local_store.go index 9fd95dc5..86aed2c9 100644 --- a/local_store.go +++ b/local_store.go @@ -222,19 +222,6 @@ func (f *fileSystemStore) stagedDir() string { return filepath.Join(f.dir, "staged") } -func isMetaFile(e os.DirEntry) (bool, error) { - if e.IsDir() || filepath.Ext(e.Name()) != ".json" { - return false, nil - } - - info, err := e.Info() - if err != nil { - return false, err - } - - return info.Mode().IsRegular(), nil -} - func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { // Build a map of metadata names (e.g. root.json) to their full paths // (whether in the committed repo dir, or in the staged repo dir). @@ -247,7 +234,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range committed { - imf, err := isMetaFile(e) + imf, err := util.IsMetaFile(e) if err != nil { return nil, err } @@ -264,7 +251,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range staged { - imf, err := isMetaFile(e) + imf, err := util.IsMetaFile(e) if err != nil { return nil, err } diff --git a/util/util.go b/util/util.go index 0878c823..dfcb67f3 100644 --- a/util/util.go +++ b/util/util.go @@ -331,3 +331,17 @@ func AtomicallyWriteFile(filename string, data []byte, perm os.FileMode) error { return nil } + +// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. +func IsMetaFile(e os.DirEntry) (bool, error) { + if e.IsDir() || filepath.Ext(e.Name()) != ".json" { + return false, nil + } + + info, err := e.Info() + if err != nil { + return false, err + } + + return info.Mode().IsRegular(), nil +} From 1acc7bf0bd536434ead0263f5980a3f9c5a23e7d Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 13:36:49 +0200 Subject: [PATCH 04/43] Added unit tests and refactored code to match the local store for repository side. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 61 +++++----- client/filejsonstore/filejsonstore_test.go | 132 +++++++++++++++++++++ 2 files changed, 165 insertions(+), 28 deletions(-) create mode 100644 client/filejsonstore/filejsonstore_test.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 1a963112..e48ce3c1 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -6,10 +6,13 @@ import ( "fmt" "os" "path/filepath" + + "github.com/theupdateframework/go-tuf/util" ) +var ErrNotJSON = errors.New("file is not in JSON format") + type FileJSONStore struct { - f *os.File baseDir string } @@ -23,39 +26,33 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { } var err error - if f.f, err = os.Open(baseDir); err != nil { + // Does the directory exist? + fi, err := os.Stat(baseDir) + if err != nil { pe, ok := err.(*os.PathError) - fmt.Println(ok) - fmt.Println(pe.Err) if ok && errors.Is(pe.Err, os.ErrNotExist) && recurse { // Create the directory + // user: rwx + // group: r-x + // other: --- err = os.Mkdir(baseDir, 0750) if err == nil { return newImpl(baseDir, false) } - return nil, err - } else { - return nil, err } + return nil, err } - if stat, err := f.f.Stat(); err != nil { - f.f.Close() - return nil, fmt.Errorf("failed to stat file %s: %w", - baseDir, err) - } else { - if !stat.IsDir() { - f.f.Close() - return nil, fmt.Errorf("can not open %s, not a directory", - baseDir) - } + if !fi.IsDir() { + return nil, fmt.Errorf("can not open %s, not a directory", + baseDir) } return &f, nil } func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { - names, err := f.f.Readdirnames(0) + names, err := os.ReadDir(f.baseDir) if err != nil { return nil, err @@ -63,33 +60,41 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { meta := map[string]json.RawMessage{} for _, name := range names { - p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + p := filepath.FromSlash(filepath.Join(f.baseDir, name.Name())) + if ok, err := util.IsMetaFile(name); !ok { + continue + } else if err != nil { + return nil, err + } b, err := os.ReadFile(p) if err != nil { return nil, err } - meta[name] = b + meta[name.Name()] = b } return meta, nil } func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { + if filepath.Ext(name) != ".json" { + return ErrNotJSON + } + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) - return os.WriteFile(p, meta, 0640) + // user: rw- + // group: r-- + // other: --- + err := util.AtomicallyWriteFile(p, meta, 0640) + return err } func (f *FileJSONStore) DeleteMeta(name string) error { p := filepath.FromSlash(filepath.Join(f.baseDir, name)) - return os.Remove(p) + err := os.Remove(p) + return err } func (f *FileJSONStore) Close() error { - if f == nil { - return nil - } - if f.f != nil { - return f.f.Close() - } return nil } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go new file mode 100644 index 00000000..65fc73a8 --- /dev/null +++ b/client/filejsonstore/filejsonstore_test.go @@ -0,0 +1,132 @@ +package filejsonstore + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/check.v1" +) + +type RawJSONStoreSuite struct{} + +var _ = check.Suite(&RawJSONStoreSuite{}) + +// Hook up gocheck into the "go test" runnder +func Test(t *testing.T) { check.TestingT(t) } + +func (RawJSONStoreSuite) TestNewOk(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Assert path does not exist + fi, err := os.Stat(p) + c.Assert(fi, check.IsNil) + c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) + + // Create implementation + s, err := New(p) + c.Assert(err, check.IsNil) + c.Assert(s, check.NotNil) + + // Assert path does exist and is a directory + fi, err = os.Stat(p) + c.Assert(fi, check.NotNil) + c.Assert(err, check.IsNil) + c.Assert(fi.IsDir(), check.Equals, true) +} + +func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Create an empty file + f, err := os.Create(p) + c.Assert(err, check.IsNil) + f.Close() + + // Create implementation + s, err := New(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.NotNil) + found := strings.Contains(err.Error(), ", not a directory") + c.Assert(found, check.Equals, true) +} + +func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) +} + +func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + expected := map[string]json.RawMessage{ + "file1.json": []byte{0xf1, 0xe1, 0xd1}, + "file2.json": []byte{0xf2, 0xe2, 0xd2}, + "file3.json": []byte{0xf3, 0xe3, 0xd3}, + } + + for k, v := range expected { + s.SetMeta(k, v) + } + + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 3) + c.Assert(md, check.DeepEquals, expected) + + // Nuke items + count := 3 + for k := range expected { + err = s.DeleteMeta(k) + count-- + c.Assert(err, check.IsNil) + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, count) + } + + md, err = s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) +} + +func (RawJSONStoreSuite) TestNoJSON(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + files := []string{ + "file.xml", + "file", + "", + } + for _, f := range files { + err := s.SetMeta(f, []byte{}) + c.Assert(err, check.Equals, ErrNotJSON) + } + +} + +func (RawJSONStoreSuite) TestClose(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + err = s.Close() + c.Assert(err, check.IsNil) +} From c527e56034fcd9d6d404e9094addd7b789fd1327 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 14:09:34 +0200 Subject: [PATCH 05/43] Added test case for non json file in metadata directory. Changed package to client to align with leveldb storage. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 2 +- client/filejsonstore/filejsonstore_test.go | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index e48ce3c1..15943886 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -1,4 +1,4 @@ -package filejsonstore +package client import ( "encoding/json" diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 65fc73a8..5577aab6 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -1,4 +1,4 @@ -package filejsonstore +package client import ( "encoding/json" @@ -101,6 +101,22 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { c.Assert(md, check.HasLen, 0) } +func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + // Create a file which does not end with '.json' + fp := filepath.FromSlash(filepath.Join(p, "meta.xml")) + os.WriteFile(fp, []byte{}, 0644) + + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) +} + func (RawJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") From b7dbd08e640e71ee04dfe98aaf87178089860802 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 15:33:37 +0200 Subject: [PATCH 06/43] Use os.MkdirAll when creating metadata cache. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 15943886..c2c7a42e 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -35,7 +35,7 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { // user: rwx // group: r-x // other: --- - err = os.Mkdir(baseDir, 0750) + err = os.MkdirAll(baseDir, 0750) if err == nil { return newImpl(baseDir, false) } From 2dc5d0e3adb5a9d82d8b10319cef24d6d800e8a2 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Mon, 1 Aug 2022 08:35:45 +0200 Subject: [PATCH 07/43] More consistent naming, added comments and a unit test. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 21 ++++++++++++++++- client/filejsonstore/filejsonstore_test.go | 27 ++++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index c2c7a42e..fb9493e7 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -10,13 +10,22 @@ import ( "github.com/theupdateframework/go-tuf/util" ) +// ErrNotJSON is returned when a metadata operation is attempted to be +// performed against a file that does not seem to be a JSON file +// (e.g. does not end in .json, case sensitive). var ErrNotJSON = errors.New("file is not in JSON format") +// FileJSONStore represents a local metadata cache relying on raw JSON files +// as retrieved from the remote repository. type FileJSONStore struct { baseDir string } -func New(baseDir string) (*FileJSONStore, error) { +// NewFileJSONStore returns a new metadata cache, implemented using raw JSON +// files, stored in a directory provided by the client. +// If the provided directory does not exist on disk, it will be created. +// The provided metadata cache is not safe for concurrent access. +func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { return newImpl(baseDir, true) } @@ -51,6 +60,7 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { return &f, nil } +// GetMeta returns the currently cached set of metadata files. func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { names, err := os.ReadDir(f.baseDir) @@ -76,6 +86,8 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { return meta, nil } +// SetMeta stores a metadata file in the cache. If the metadata file exist, +// it will be overwritten. func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { if filepath.Ext(name) != ".json" { return ErrNotJSON @@ -89,12 +101,19 @@ func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { return err } +// DeleteMeta deletes a metadata file from the cache. +// If the file does not exist, an *os.PathError is returned. func (f *FileJSONStore) DeleteMeta(name string) error { + if filepath.Ext(name) != ".json" { + return ErrNotJSON + } + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) err := os.Remove(p) return err } +// Close closes the metadata cache. This is a no-op. func (f *FileJSONStore) Close() error { return nil } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 5577aab6..7a6a139c 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -28,7 +28,7 @@ func (RawJSONStoreSuite) TestNewOk(c *check.C) { c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) // Create implementation - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(err, check.IsNil) c.Assert(s, check.NotNil) @@ -49,7 +49,7 @@ func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { f.Close() // Create implementation - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.IsNil) c.Assert(err, check.NotNil) found := strings.Contains(err.Error(), ", not a directory") @@ -59,7 +59,7 @@ func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) md, err := s.GetMeta() c.Assert(err, check.IsNil) @@ -69,7 +69,7 @@ func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) expected := map[string]json.RawMessage{ "file1.json": []byte{0xf1, 0xe1, 0xd1}, "file2.json": []byte{0xf2, 0xe2, 0xd2}, @@ -104,7 +104,7 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) @@ -120,7 +120,7 @@ func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { func (RawJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) @@ -139,10 +139,23 @@ func (RawJSONStoreSuite) TestNoJSON(c *check.C) { func (RawJSONStoreSuite) TestClose(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) err = s.Close() c.Assert(err, check.IsNil) } + +func (RawJSONStoreSuite) TestDelete(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + err = s.DeleteMeta("not_json.yml") + c.Assert(err, check.Equals, ErrNotJSON) + err = s.DeleteMeta("non_existing.json") + c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) +} From ad4974bf2c8eb05d650578aebc0feac04b4e5de0 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 4 Aug 2022 08:12:46 +0200 Subject: [PATCH 08/43] Added a localstore wrapper for concurrent access. Signed-off-by: Fredrik Skogman --- client/concurrentlocalstore.go | 49 +++++++++++++++++++ client/concurrentlocalstore_test.go | 55 ++++++++++++++++++++++ client/filejsonstore/filejsonstore.go | 4 +- client/filejsonstore/filejsonstore_test.go | 3 +- 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 client/concurrentlocalstore.go create mode 100644 client/concurrentlocalstore_test.go diff --git a/client/concurrentlocalstore.go b/client/concurrentlocalstore.go new file mode 100644 index 00000000..f37f7aa1 --- /dev/null +++ b/client/concurrentlocalstore.go @@ -0,0 +1,49 @@ +package client + +import ( + "encoding/json" + "sync" +) + +// ConcurrentLocalStore provides a LocalStore over an existing LocalStore +// implementation that is safe for concurrent access. +type ConcurrentLocalStore struct { + mtx sync.RWMutex + store LocalStore +} + +// NewConcurrentLocalStore returns a wrapped LocalStore that is safe for +// concurrent access. +func NewConcurrentLocalStore(s LocalStore) *ConcurrentLocalStore { + return &ConcurrentLocalStore{ + store: s, + } +} + +// GetMeta returns all known targets. +func (c *ConcurrentLocalStore) GetMeta() (map[string]json.RawMessage, error) { + c.mtx.RLock() + defer c.mtx.RUnlock() + return c.store.GetMeta() +} + +// SetMeta updates the lcoal store with a new target. +func (c *ConcurrentLocalStore) SetMeta(name string, meta json.RawMessage) error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.store.SetMeta(name, meta) +} + +// DeleteMeta remves a target from the cache. +func (c *ConcurrentLocalStore) DeleteMeta(name string) error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.store.DeleteMeta(name) +} + +// Close closes the local store. +func (c *ConcurrentLocalStore) Close() error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.store.Close() +} diff --git a/client/concurrentlocalstore_test.go b/client/concurrentlocalstore_test.go new file mode 100644 index 00000000..4c883f23 --- /dev/null +++ b/client/concurrentlocalstore_test.go @@ -0,0 +1,55 @@ +package client + +import ( + "encoding/json" + "testing" + + "gopkg.in/check.v1" +) + +type ConcurrentStoreSuite struct{} + +var _ = check.Suite(&ConcurrentStoreSuite{}) + +// Hook up gocheck into the "go test" runnder +func ConcurrentTest(t *testing.T) { check.TestingT(t) } + +func (ConcurrentStoreSuite) TestOperations(c *check.C) { + mem := MemoryLocalStore() + store := NewConcurrentLocalStore(mem) + + c.Assert(store, check.NotNil) + expected := map[string]json.RawMessage{ + "file1.json": []byte{0xf1, 0xe1, 0xd1}, + "file2.json": []byte{0xf2, 0xe2, 0xd2}, + "file3.json": []byte{0xf3, 0xe3, 0xd3}, + } + + for k, v := range expected { + err := store.SetMeta(k, v) + c.Assert(err, check.IsNil) + } + + md, err := store.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 3) + c.Assert(md, check.DeepEquals, expected) + + // Nuke items + count := 3 + for k := range expected { + err = store.DeleteMeta(k) + count-- + c.Assert(err, check.IsNil) + md, err := store.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, count) + } + + md, err = store.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) + + err = store.Close() + c.Assert(err, check.IsNil) +} diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index fb9493e7..06a73abd 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -24,7 +24,9 @@ type FileJSONStore struct { // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. -// The provided metadata cache is not safe for concurrent access. +// The provided metadata cache is not safe for concurrent access, if +// concurrent access safety is requires, wrap local store in a +// ConcurrentLocalStore. func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { return newImpl(baseDir, true) } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 7a6a139c..a6d7cfca 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -77,7 +77,8 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { } for k, v := range expected { - s.SetMeta(k, v) + err := s.SetMeta(k, v) + c.Assert(err, check.IsNil) } md, err := s.GetMeta() From 61c73238f437fc90853e146840169437432bb6c3 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 3 Aug 2022 13:43:12 +0200 Subject: [PATCH 09/43] Update client/filejsonstore/filejsonstore_test.go Fixed spelling error found during review. Co-authored-by: Joshua Lock Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index a6d7cfca..778f4808 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -15,7 +15,7 @@ type RawJSONStoreSuite struct{} var _ = check.Suite(&RawJSONStoreSuite{}) -// Hook up gocheck into the "go test" runnder +// Hook up gocheck into the "go test" runner func Test(t *testing.T) { check.TestingT(t) } func (RawJSONStoreSuite) TestNewOk(c *check.C) { From 4714ff1a346d8830bc4baad8656a146301f792d7 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 4 Aug 2022 08:44:57 +0200 Subject: [PATCH 10/43] Added tests to make sure returned struct implements LocalStore interface. Signed-off-by: Fredrik Skogman --- client/concurrentlocalstore_test.go | 4 +++- client/filejsonstore/filejsonstore_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/client/concurrentlocalstore_test.go b/client/concurrentlocalstore_test.go index 4c883f23..c4824f5d 100644 --- a/client/concurrentlocalstore_test.go +++ b/client/concurrentlocalstore_test.go @@ -15,8 +15,10 @@ var _ = check.Suite(&ConcurrentStoreSuite{}) func ConcurrentTest(t *testing.T) { check.TestingT(t) } func (ConcurrentStoreSuite) TestOperations(c *check.C) { + var store LocalStore + mem := MemoryLocalStore() - store := NewConcurrentLocalStore(mem) + store = NewConcurrentLocalStore(mem) c.Assert(store, check.NotNil) expected := map[string]json.RawMessage{ diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 778f4808..e256f50d 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -9,6 +9,8 @@ import ( "testing" "gopkg.in/check.v1" + + "github.com/theupdateframework/go-tuf/client" ) type RawJSONStoreSuite struct{} @@ -160,3 +162,14 @@ func (RawJSONStoreSuite) TestDelete(c *check.C) { err = s.DeleteMeta("non_existing.json") c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } + +func (RawJSONStoreSuite) TestInterface(c *check.C) { + var localStore client.LocalStore + var err error + + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + localStore, err = NewFileJSONStore(p) + c.Assert(localStore, check.NotNil) + c.Assert(err, check.IsNil) +} From 40488485a978ac76464206c4c31f857b240bc053 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Tue, 9 Aug 2022 17:54:30 +0200 Subject: [PATCH 11/43] Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 06a73abd..3272b04e 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -10,8 +10,8 @@ import ( "github.com/theupdateframework/go-tuf/util" ) -// ErrNotJSON is returned when a metadata operation is attempted to be -// performed against a file that does not seem to be a JSON file +// ErrNotJSON is returned when a metadata operation is attempted +// against a file that does not seem to be a JSON file // (e.g. does not end in .json, case sensitive). var ErrNotJSON = errors.New("file is not in JSON format") From 1d074f8caa7bb251990e76226e16c5ec19d9ed12 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Tue, 9 Aug 2022 17:54:55 +0200 Subject: [PATCH 12/43] Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index e256f50d..130767bc 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -88,7 +88,7 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { c.Assert(md, check.HasLen, 3) c.Assert(md, check.DeepEquals, expected) - // Nuke items + // Delete all items count := 3 for k := range expected { err = s.DeleteMeta(k) From 18f7ce711a5d5fe4d33e32942edb61c1aec6acc9 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 10 Aug 2022 08:13:14 +0200 Subject: [PATCH 13/43] Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 3272b04e..dd13651f 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -65,7 +65,6 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { // GetMeta returns the currently cached set of metadata files. func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { names, err := os.ReadDir(f.baseDir) - if err != nil { return nil, err } From 800da609e41bd8f56323df50ff580f36f252c858 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 10 Aug 2022 08:13:59 +0200 Subject: [PATCH 14/43] Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 130767bc..8fb4ece6 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -136,7 +136,6 @@ func (RawJSONStoreSuite) TestNoJSON(c *check.C) { err := s.SetMeta(f, []byte{}) c.Assert(err, check.Equals, ErrNotJSON) } - } func (RawJSONStoreSuite) TestClose(c *check.C) { From 0627a275b11e3d8cea48e0dce0c9fd1ca821cfed Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 10 Aug 2022 10:55:40 +0200 Subject: [PATCH 15/43] Made FileJSONStore safe for concurrent access. Moved IsMetaFile to new pkg, internal/fsutil Permission bits are validated when access the cache. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 82 +++++++++++++------- client/filejsonstore/filejsonstore_test.go | 88 +++++++++++++++++++--- internal/fsutil/fsutil.go | 39 ++++++++++ local_store.go | 5 +- util/util.go | 14 ---- 5 files changed, 174 insertions(+), 54 deletions(-) create mode 100644 internal/fsutil/fsutil.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index dd13651f..61442800 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "path/filepath" + "sync" + "github.com/theupdateframework/go-tuf/internal/fsutil" "github.com/theupdateframework/go-tuf/util" ) @@ -15,23 +17,33 @@ import ( // (e.g. does not end in .json, case sensitive). var ErrNotJSON = errors.New("file is not in JSON format") +// ErrToPermissive is returned when the metadata directory, or a metadata +// file has too permissive file mode bits set +var ErrTooPermissive = errors.New("permissions are too permissive") + +const ( + // user: rwx + // group: r-x + // other: --- + dirCreateMode = os.FileMode(0750) + // user: rw- + // group: r-- + // other: --- + fileCreateMode = os.FileMode(0640) +) + // FileJSONStore represents a local metadata cache relying on raw JSON files // as retrieved from the remote repository. type FileJSONStore struct { + mtx sync.RWMutex baseDir string } // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. -// The provided metadata cache is not safe for concurrent access, if -// concurrent access safety is requires, wrap local store in a -// ConcurrentLocalStore. +// The provided metadata cache is afe for concurrent access func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { - return newImpl(baseDir, true) -} - -func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { var f = FileJSONStore{ baseDir: baseDir, } @@ -41,22 +53,24 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { fi, err := os.Stat(baseDir) if err != nil { pe, ok := err.(*os.PathError) - if ok && errors.Is(pe.Err, os.ErrNotExist) && recurse { + if ok && errors.Is(pe.Err, os.ErrNotExist) { // Create the directory - // user: rwx - // group: r-x - // other: --- - err = os.MkdirAll(baseDir, 0750) - if err == nil { - return newImpl(baseDir, false) + if err = os.MkdirAll(baseDir, dirCreateMode); err != nil { + return nil, err } + } else { + return nil, err + } + } else { + // Verify that it is a directory + if !fi.IsDir() { + return nil, fmt.Errorf("can not open %s, not a directory", + baseDir) + } + // Verify file mode is not too permissive. + if err = fsutil.EnsurePermission(fi, dirCreateMode); err != nil { + return nil, ErrTooPermissive } - return nil, err - } - - if !fi.IsDir() { - return nil, fmt.Errorf("can not open %s, not a directory", - baseDir) } return &f, nil @@ -64,6 +78,9 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { // GetMeta returns the currently cached set of metadata files. func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { + f.mtx.RLock() + defer f.mtx.RUnlock() + names, err := os.ReadDir(f.baseDir) if err != nil { return nil, err @@ -72,11 +89,21 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { meta := map[string]json.RawMessage{} for _, name := range names { p := filepath.FromSlash(filepath.Join(f.baseDir, name.Name())) - if ok, err := util.IsMetaFile(name); !ok { + ok, err := fsutil.IsMetaFile(name) + if err != nil { + return nil, err + } + if !ok { continue - } else if err != nil { + } + // Verify permissions + info, err := name.Info() + if err != nil { return nil, err } + if err = fsutil.EnsurePermission(info, fileCreateMode); err != nil { + return nil, ErrTooPermissive + } b, err := os.ReadFile(p) if err != nil { return nil, err @@ -90,21 +117,24 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { // SetMeta stores a metadata file in the cache. If the metadata file exist, // it will be overwritten. func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { + f.mtx.Lock() + defer f.mtx.Unlock() + if filepath.Ext(name) != ".json" { return ErrNotJSON } p := filepath.FromSlash(filepath.Join(f.baseDir, name)) - // user: rw- - // group: r-- - // other: --- - err := util.AtomicallyWriteFile(p, meta, 0640) + err := util.AtomicallyWriteFile(p, meta, fileCreateMode) return err } // DeleteMeta deletes a metadata file from the cache. // If the file does not exist, an *os.PathError is returned. func (f *FileJSONStore) DeleteMeta(name string) error { + f.mtx.Lock() + defer f.mtx.Unlock() + if filepath.Ext(name) != ".json" { return ErrNotJSON } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 8fb4ece6..dfc54a62 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -13,14 +13,14 @@ import ( "github.com/theupdateframework/go-tuf/client" ) -type RawJSONStoreSuite struct{} +type FileJSONStoreSuite struct{} -var _ = check.Suite(&RawJSONStoreSuite{}) +var _ = check.Suite(&FileJSONStoreSuite{}) // Hook up gocheck into the "go test" runner func Test(t *testing.T) { check.TestingT(t) } -func (RawJSONStoreSuite) TestNewOk(c *check.C) { +func (FileJSONStoreSuite) TestNewOk(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") @@ -41,7 +41,7 @@ func (RawJSONStoreSuite) TestNewOk(c *check.C) { c.Assert(fi.IsDir(), check.Equals, true) } -func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { +func (FileJSONStoreSuite) TestNewFileExists(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") @@ -58,7 +58,37 @@ func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { c.Assert(found, check.Equals, true) } -func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { +func (FileJSONStoreSuite) TestNewDirectoryExists(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + err := os.Mkdir(p, 0750) + c.Assert(err, check.IsNil) + + // Create implementation + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + // Modify the directory permission and try again + err = os.Chmod(p, 0751) + s, err = NewFileJSONStore(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.Equals, ErrTooPermissive) +} + +func (FileJSONStoreSuite) TestNewNoCreate(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Clear the write bit for the user + err := os.Chmod(tmp, 0551) + s, err := NewFileJSONStore(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.NotNil) +} + +func (FileJSONStoreSuite) TestGetMetaEmpty(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -68,7 +98,20 @@ func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { c.Assert(md, check.HasLen, 0) } -func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { +func (FileJSONStoreSuite) TestGetNoDirectory(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + + err = os.Remove(p) + c.Assert(err, check.IsNil) + + md, err := s.GetMeta() + c.Assert(md, check.IsNil) + c.Assert(err, check.NotNil) +} + +func (FileJSONStoreSuite) TestMetadataOperations(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -104,7 +147,7 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { c.Assert(md, check.HasLen, 0) } -func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { +func (FileJSONStoreSuite) TestGetNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -113,14 +156,32 @@ func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { // Create a file which does not end with '.json' fp := filepath.FromSlash(filepath.Join(p, "meta.xml")) - os.WriteFile(fp, []byte{}, 0644) + err = os.WriteFile(fp, []byte{}, 0644) + c.Assert(err, check.IsNil) md, err := s.GetMeta() c.Assert(err, check.IsNil) c.Assert(md, check.HasLen, 0) } -func (RawJSONStoreSuite) TestNoJSON(c *check.C) { +func (FileJSONStoreSuite) TestGetTooPermissive(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + // Create a file which does not end with '.json' + fp := filepath.FromSlash(filepath.Join(p, "meta.json")) + err = os.WriteFile(fp, []byte{}, 0644) + c.Assert(err, check.IsNil) + + md, err := s.GetMeta() + c.Assert(md, check.IsNil) + c.Assert(err, check.Equals, ErrTooPermissive) +} + +func (FileJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -138,7 +199,7 @@ func (RawJSONStoreSuite) TestNoJSON(c *check.C) { } } -func (RawJSONStoreSuite) TestClose(c *check.C) { +func (FileJSONStoreSuite) TestClose(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -149,7 +210,7 @@ func (RawJSONStoreSuite) TestClose(c *check.C) { c.Assert(err, check.IsNil) } -func (RawJSONStoreSuite) TestDelete(c *check.C) { +func (FileJSONStoreSuite) TestDelete(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -162,7 +223,7 @@ func (RawJSONStoreSuite) TestDelete(c *check.C) { c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } -func (RawJSONStoreSuite) TestInterface(c *check.C) { +func (FileJSONStoreSuite) TestInterface(c *check.C) { var localStore client.LocalStore var err error @@ -172,3 +233,6 @@ func (RawJSONStoreSuite) TestInterface(c *check.C) { c.Assert(localStore, check.NotNil) c.Assert(err, check.IsNil) } + +// test 3 file exist and is too permissive +// test 4 remote base dir after create diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go new file mode 100644 index 00000000..98883952 --- /dev/null +++ b/internal/fsutil/fsutil.go @@ -0,0 +1,39 @@ +// Package fsutil defiens a set of internal utility functions used to +// interact with the file system. +package fsutil + +import ( + "errors" + "io/fs" + "os" + "path/filepath" +) + +var ErrPermission = errors.New("unexpected permission") + +// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. +func IsMetaFile(e os.DirEntry) (bool, error) { + if e.IsDir() || filepath.Ext(e.Name()) != ".json" { + return false, nil + } + + info, err := e.Info() + if err != nil { + return false, err + } + + return info.Mode().IsRegular(), nil +} + +// EnsurePemissions tests the provided file info to make sure the +// permission bits matches the provided. +func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { + // Clear all bits which are note related to the permission. + mode := fi.Mode() & fs.ModePerm + mask := ^perm + if (mode & mask) != 0 { + return ErrPermission + } + + return nil +} diff --git a/local_store.go b/local_store.go index 86aed2c9..1409f13f 100644 --- a/local_store.go +++ b/local_store.go @@ -14,6 +14,7 @@ import ( "github.com/theupdateframework/go-tuf/data" "github.com/theupdateframework/go-tuf/encrypted" + "github.com/theupdateframework/go-tuf/internal/fsutil" "github.com/theupdateframework/go-tuf/internal/sets" "github.com/theupdateframework/go-tuf/pkg/keys" "github.com/theupdateframework/go-tuf/util" @@ -234,7 +235,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range committed { - imf, err := util.IsMetaFile(e) + imf, err := fsutil.IsMetaFile(e) if err != nil { return nil, err } @@ -251,7 +252,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range staged { - imf, err := util.IsMetaFile(e) + imf, err := fsutil.IsMetaFile(e) if err != nil { return nil, err } diff --git a/util/util.go b/util/util.go index dfcb67f3..0878c823 100644 --- a/util/util.go +++ b/util/util.go @@ -331,17 +331,3 @@ func AtomicallyWriteFile(filename string, data []byte, perm os.FileMode) error { return nil } - -// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. -func IsMetaFile(e os.DirEntry) (bool, error) { - if e.IsDir() || filepath.Ext(e.Name()) != ".json" { - return false, nil - } - - info, err := e.Info() - if err != nil { - return false, err - } - - return info.Mode().IsRegular(), nil -} From 83c0eab250002efead52520d00367d7eea8f18d1 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 11 Aug 2022 15:42:46 +0200 Subject: [PATCH 16/43] Spelling error. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 61442800..90cfe24f 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -42,7 +42,7 @@ type FileJSONStore struct { // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. -// The provided metadata cache is afe for concurrent access +// The provided metadata cache is safe for concurrent access. func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { var f = FileJSONStore{ baseDir: baseDir, From 0cbf33235c7ee150778c3991f9f84af05e44bb4c Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 12 Aug 2022 07:37:51 +0200 Subject: [PATCH 17/43] Updates based on PR comments. Removed a test for satisfying an interface, and replaced with a compile time check. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 3 +++ client/filejsonstore/filejsonstore_test.go | 16 ---------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 90cfe24f..c80d8ede 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sync" + "github.com/theupdateframework/go-tuf/client" "github.com/theupdateframework/go-tuf/internal/fsutil" "github.com/theupdateframework/go-tuf/util" ) @@ -39,6 +40,8 @@ type FileJSONStore struct { baseDir string } +var _ client.LocalStore = (*FileJSONStore)(nil) + // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index dfc54a62..1fdc4e8b 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -9,8 +9,6 @@ import ( "testing" "gopkg.in/check.v1" - - "github.com/theupdateframework/go-tuf/client" ) type FileJSONStoreSuite struct{} @@ -222,17 +220,3 @@ func (FileJSONStoreSuite) TestDelete(c *check.C) { err = s.DeleteMeta("non_existing.json") c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } - -func (FileJSONStoreSuite) TestInterface(c *check.C) { - var localStore client.LocalStore - var err error - - tmp := c.MkDir() - p := filepath.Join(tmp, "tuf_raw.db") - localStore, err = NewFileJSONStore(p) - c.Assert(localStore, check.NotNil) - c.Assert(err, check.IsNil) -} - -// test 3 file exist and is too permissive -// test 4 remote base dir after create From 7a935d6f4cfbd9e16995644232781e88d450ba3a Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 28 Jul 2022 16:57:44 +0200 Subject: [PATCH 18/43] Added first draft for a raw json local file storage. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 client/filejsonstore/filejsonstore.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go new file mode 100644 index 00000000..1a963112 --- /dev/null +++ b/client/filejsonstore/filejsonstore.go @@ -0,0 +1,95 @@ +package filejsonstore + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +type FileJSONStore struct { + f *os.File + baseDir string +} + +func New(baseDir string) (*FileJSONStore, error) { + return newImpl(baseDir, true) +} + +func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { + var f = FileJSONStore{ + baseDir: baseDir, + } + var err error + + if f.f, err = os.Open(baseDir); err != nil { + pe, ok := err.(*os.PathError) + fmt.Println(ok) + fmt.Println(pe.Err) + if ok && errors.Is(pe.Err, os.ErrNotExist) && recurse { + // Create the directory + err = os.Mkdir(baseDir, 0750) + if err == nil { + return newImpl(baseDir, false) + } + return nil, err + } else { + return nil, err + } + } + + if stat, err := f.f.Stat(); err != nil { + f.f.Close() + return nil, fmt.Errorf("failed to stat file %s: %w", + baseDir, err) + } else { + if !stat.IsDir() { + f.f.Close() + return nil, fmt.Errorf("can not open %s, not a directory", + baseDir) + } + } + + return &f, nil +} + +func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { + names, err := f.f.Readdirnames(0) + + if err != nil { + return nil, err + } + + meta := map[string]json.RawMessage{} + for _, name := range names { + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + b, err := os.ReadFile(p) + if err != nil { + return nil, err + } + meta[name] = b + } + + return meta, nil +} + +func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + return os.WriteFile(p, meta, 0640) +} + +func (f *FileJSONStore) DeleteMeta(name string) error { + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + return os.Remove(p) +} + +func (f *FileJSONStore) Close() error { + if f == nil { + return nil + } + if f.f != nil { + return f.f.Close() + } + return nil +} From c0cc5fcacf0d9443f1997048f988ff2a050fcac4 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 28 Jul 2022 16:58:23 +0200 Subject: [PATCH 19/43] Ignore emacs temporary files Signed-off-by: Fredrik Skogman --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 64b8f8a8..39e7149c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ cmd/tuf/tuf cmd/tuf-client/tuf-client .vscode +*~ From ad8f340de1b04ec789ee6a8ec37c908826707129 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 13:35:49 +0200 Subject: [PATCH 20/43] moved isMetaFile to util directory for reuse in other packages. Signed-off-by: Fredrik Skogman --- local_store.go | 17 ++--------------- util/util.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/local_store.go b/local_store.go index 9fd95dc5..86aed2c9 100644 --- a/local_store.go +++ b/local_store.go @@ -222,19 +222,6 @@ func (f *fileSystemStore) stagedDir() string { return filepath.Join(f.dir, "staged") } -func isMetaFile(e os.DirEntry) (bool, error) { - if e.IsDir() || filepath.Ext(e.Name()) != ".json" { - return false, nil - } - - info, err := e.Info() - if err != nil { - return false, err - } - - return info.Mode().IsRegular(), nil -} - func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { // Build a map of metadata names (e.g. root.json) to their full paths // (whether in the committed repo dir, or in the staged repo dir). @@ -247,7 +234,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range committed { - imf, err := isMetaFile(e) + imf, err := util.IsMetaFile(e) if err != nil { return nil, err } @@ -264,7 +251,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range staged { - imf, err := isMetaFile(e) + imf, err := util.IsMetaFile(e) if err != nil { return nil, err } diff --git a/util/util.go b/util/util.go index 0878c823..dfcb67f3 100644 --- a/util/util.go +++ b/util/util.go @@ -331,3 +331,17 @@ func AtomicallyWriteFile(filename string, data []byte, perm os.FileMode) error { return nil } + +// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. +func IsMetaFile(e os.DirEntry) (bool, error) { + if e.IsDir() || filepath.Ext(e.Name()) != ".json" { + return false, nil + } + + info, err := e.Info() + if err != nil { + return false, err + } + + return info.Mode().IsRegular(), nil +} From d1956d6ce5db774086a5c859ca1c2a80a2cff623 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 13:36:49 +0200 Subject: [PATCH 21/43] Added unit tests and refactored code to match the local store for repository side. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 61 +++++----- client/filejsonstore/filejsonstore_test.go | 132 +++++++++++++++++++++ 2 files changed, 165 insertions(+), 28 deletions(-) create mode 100644 client/filejsonstore/filejsonstore_test.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 1a963112..e48ce3c1 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -6,10 +6,13 @@ import ( "fmt" "os" "path/filepath" + + "github.com/theupdateframework/go-tuf/util" ) +var ErrNotJSON = errors.New("file is not in JSON format") + type FileJSONStore struct { - f *os.File baseDir string } @@ -23,39 +26,33 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { } var err error - if f.f, err = os.Open(baseDir); err != nil { + // Does the directory exist? + fi, err := os.Stat(baseDir) + if err != nil { pe, ok := err.(*os.PathError) - fmt.Println(ok) - fmt.Println(pe.Err) if ok && errors.Is(pe.Err, os.ErrNotExist) && recurse { // Create the directory + // user: rwx + // group: r-x + // other: --- err = os.Mkdir(baseDir, 0750) if err == nil { return newImpl(baseDir, false) } - return nil, err - } else { - return nil, err } + return nil, err } - if stat, err := f.f.Stat(); err != nil { - f.f.Close() - return nil, fmt.Errorf("failed to stat file %s: %w", - baseDir, err) - } else { - if !stat.IsDir() { - f.f.Close() - return nil, fmt.Errorf("can not open %s, not a directory", - baseDir) - } + if !fi.IsDir() { + return nil, fmt.Errorf("can not open %s, not a directory", + baseDir) } return &f, nil } func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { - names, err := f.f.Readdirnames(0) + names, err := os.ReadDir(f.baseDir) if err != nil { return nil, err @@ -63,33 +60,41 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { meta := map[string]json.RawMessage{} for _, name := range names { - p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + p := filepath.FromSlash(filepath.Join(f.baseDir, name.Name())) + if ok, err := util.IsMetaFile(name); !ok { + continue + } else if err != nil { + return nil, err + } b, err := os.ReadFile(p) if err != nil { return nil, err } - meta[name] = b + meta[name.Name()] = b } return meta, nil } func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { + if filepath.Ext(name) != ".json" { + return ErrNotJSON + } + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) - return os.WriteFile(p, meta, 0640) + // user: rw- + // group: r-- + // other: --- + err := util.AtomicallyWriteFile(p, meta, 0640) + return err } func (f *FileJSONStore) DeleteMeta(name string) error { p := filepath.FromSlash(filepath.Join(f.baseDir, name)) - return os.Remove(p) + err := os.Remove(p) + return err } func (f *FileJSONStore) Close() error { - if f == nil { - return nil - } - if f.f != nil { - return f.f.Close() - } return nil } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go new file mode 100644 index 00000000..65fc73a8 --- /dev/null +++ b/client/filejsonstore/filejsonstore_test.go @@ -0,0 +1,132 @@ +package filejsonstore + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/check.v1" +) + +type RawJSONStoreSuite struct{} + +var _ = check.Suite(&RawJSONStoreSuite{}) + +// Hook up gocheck into the "go test" runnder +func Test(t *testing.T) { check.TestingT(t) } + +func (RawJSONStoreSuite) TestNewOk(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Assert path does not exist + fi, err := os.Stat(p) + c.Assert(fi, check.IsNil) + c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) + + // Create implementation + s, err := New(p) + c.Assert(err, check.IsNil) + c.Assert(s, check.NotNil) + + // Assert path does exist and is a directory + fi, err = os.Stat(p) + c.Assert(fi, check.NotNil) + c.Assert(err, check.IsNil) + c.Assert(fi.IsDir(), check.Equals, true) +} + +func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Create an empty file + f, err := os.Create(p) + c.Assert(err, check.IsNil) + f.Close() + + // Create implementation + s, err := New(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.NotNil) + found := strings.Contains(err.Error(), ", not a directory") + c.Assert(found, check.Equals, true) +} + +func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) +} + +func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + expected := map[string]json.RawMessage{ + "file1.json": []byte{0xf1, 0xe1, 0xd1}, + "file2.json": []byte{0xf2, 0xe2, 0xd2}, + "file3.json": []byte{0xf3, 0xe3, 0xd3}, + } + + for k, v := range expected { + s.SetMeta(k, v) + } + + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 3) + c.Assert(md, check.DeepEquals, expected) + + // Nuke items + count := 3 + for k := range expected { + err = s.DeleteMeta(k) + count-- + c.Assert(err, check.IsNil) + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, count) + } + + md, err = s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) +} + +func (RawJSONStoreSuite) TestNoJSON(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + files := []string{ + "file.xml", + "file", + "", + } + for _, f := range files { + err := s.SetMeta(f, []byte{}) + c.Assert(err, check.Equals, ErrNotJSON) + } + +} + +func (RawJSONStoreSuite) TestClose(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + err = s.Close() + c.Assert(err, check.IsNil) +} From 678b7f9c30f449f4a9b9a569b836001f61e168d1 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 14:09:34 +0200 Subject: [PATCH 22/43] Added test case for non json file in metadata directory. Changed package to client to align with leveldb storage. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 2 +- client/filejsonstore/filejsonstore_test.go | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index e48ce3c1..15943886 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -1,4 +1,4 @@ -package filejsonstore +package client import ( "encoding/json" diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 65fc73a8..5577aab6 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -1,4 +1,4 @@ -package filejsonstore +package client import ( "encoding/json" @@ -101,6 +101,22 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { c.Assert(md, check.HasLen, 0) } +func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := New(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + // Create a file which does not end with '.json' + fp := filepath.FromSlash(filepath.Join(p, "meta.xml")) + os.WriteFile(fp, []byte{}, 0644) + + md, err := s.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) +} + func (RawJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") From 207fb70a2ad0a5a3f593889d36e663fcee8e44d8 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 29 Jul 2022 15:33:37 +0200 Subject: [PATCH 23/43] Use os.MkdirAll when creating metadata cache. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 15943886..c2c7a42e 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -35,7 +35,7 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { // user: rwx // group: r-x // other: --- - err = os.Mkdir(baseDir, 0750) + err = os.MkdirAll(baseDir, 0750) if err == nil { return newImpl(baseDir, false) } From 7916ddf9654bd875590de375aaf09693b8400bf4 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Mon, 1 Aug 2022 08:35:45 +0200 Subject: [PATCH 24/43] More consistent naming, added comments and a unit test. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 21 ++++++++++++++++- client/filejsonstore/filejsonstore_test.go | 27 ++++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index c2c7a42e..fb9493e7 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -10,13 +10,22 @@ import ( "github.com/theupdateframework/go-tuf/util" ) +// ErrNotJSON is returned when a metadata operation is attempted to be +// performed against a file that does not seem to be a JSON file +// (e.g. does not end in .json, case sensitive). var ErrNotJSON = errors.New("file is not in JSON format") +// FileJSONStore represents a local metadata cache relying on raw JSON files +// as retrieved from the remote repository. type FileJSONStore struct { baseDir string } -func New(baseDir string) (*FileJSONStore, error) { +// NewFileJSONStore returns a new metadata cache, implemented using raw JSON +// files, stored in a directory provided by the client. +// If the provided directory does not exist on disk, it will be created. +// The provided metadata cache is not safe for concurrent access. +func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { return newImpl(baseDir, true) } @@ -51,6 +60,7 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { return &f, nil } +// GetMeta returns the currently cached set of metadata files. func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { names, err := os.ReadDir(f.baseDir) @@ -76,6 +86,8 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { return meta, nil } +// SetMeta stores a metadata file in the cache. If the metadata file exist, +// it will be overwritten. func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { if filepath.Ext(name) != ".json" { return ErrNotJSON @@ -89,12 +101,19 @@ func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { return err } +// DeleteMeta deletes a metadata file from the cache. +// If the file does not exist, an *os.PathError is returned. func (f *FileJSONStore) DeleteMeta(name string) error { + if filepath.Ext(name) != ".json" { + return ErrNotJSON + } + p := filepath.FromSlash(filepath.Join(f.baseDir, name)) err := os.Remove(p) return err } +// Close closes the metadata cache. This is a no-op. func (f *FileJSONStore) Close() error { return nil } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 5577aab6..7a6a139c 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -28,7 +28,7 @@ func (RawJSONStoreSuite) TestNewOk(c *check.C) { c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) // Create implementation - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(err, check.IsNil) c.Assert(s, check.NotNil) @@ -49,7 +49,7 @@ func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { f.Close() // Create implementation - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.IsNil) c.Assert(err, check.NotNil) found := strings.Contains(err.Error(), ", not a directory") @@ -59,7 +59,7 @@ func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) md, err := s.GetMeta() c.Assert(err, check.IsNil) @@ -69,7 +69,7 @@ func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) expected := map[string]json.RawMessage{ "file1.json": []byte{0xf1, 0xe1, 0xd1}, "file2.json": []byte{0xf2, 0xe2, 0xd2}, @@ -104,7 +104,7 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) @@ -120,7 +120,7 @@ func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { func (RawJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) @@ -139,10 +139,23 @@ func (RawJSONStoreSuite) TestNoJSON(c *check.C) { func (RawJSONStoreSuite) TestClose(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") - s, err := New(p) + s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) err = s.Close() c.Assert(err, check.IsNil) } + +func (RawJSONStoreSuite) TestDelete(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + err = s.DeleteMeta("not_json.yml") + c.Assert(err, check.Equals, ErrNotJSON) + err = s.DeleteMeta("non_existing.json") + c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) +} From ae6b3d8d9800d1f39f5d162110b7d4f1274bfe2c Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 4 Aug 2022 08:12:46 +0200 Subject: [PATCH 25/43] Added a localstore wrapper for concurrent access. Signed-off-by: Fredrik Skogman --- client/concurrentlocalstore.go | 49 +++++++++++++++++++ client/concurrentlocalstore_test.go | 55 ++++++++++++++++++++++ client/filejsonstore/filejsonstore.go | 4 +- client/filejsonstore/filejsonstore_test.go | 3 +- 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 client/concurrentlocalstore.go create mode 100644 client/concurrentlocalstore_test.go diff --git a/client/concurrentlocalstore.go b/client/concurrentlocalstore.go new file mode 100644 index 00000000..f37f7aa1 --- /dev/null +++ b/client/concurrentlocalstore.go @@ -0,0 +1,49 @@ +package client + +import ( + "encoding/json" + "sync" +) + +// ConcurrentLocalStore provides a LocalStore over an existing LocalStore +// implementation that is safe for concurrent access. +type ConcurrentLocalStore struct { + mtx sync.RWMutex + store LocalStore +} + +// NewConcurrentLocalStore returns a wrapped LocalStore that is safe for +// concurrent access. +func NewConcurrentLocalStore(s LocalStore) *ConcurrentLocalStore { + return &ConcurrentLocalStore{ + store: s, + } +} + +// GetMeta returns all known targets. +func (c *ConcurrentLocalStore) GetMeta() (map[string]json.RawMessage, error) { + c.mtx.RLock() + defer c.mtx.RUnlock() + return c.store.GetMeta() +} + +// SetMeta updates the lcoal store with a new target. +func (c *ConcurrentLocalStore) SetMeta(name string, meta json.RawMessage) error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.store.SetMeta(name, meta) +} + +// DeleteMeta remves a target from the cache. +func (c *ConcurrentLocalStore) DeleteMeta(name string) error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.store.DeleteMeta(name) +} + +// Close closes the local store. +func (c *ConcurrentLocalStore) Close() error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.store.Close() +} diff --git a/client/concurrentlocalstore_test.go b/client/concurrentlocalstore_test.go new file mode 100644 index 00000000..4c883f23 --- /dev/null +++ b/client/concurrentlocalstore_test.go @@ -0,0 +1,55 @@ +package client + +import ( + "encoding/json" + "testing" + + "gopkg.in/check.v1" +) + +type ConcurrentStoreSuite struct{} + +var _ = check.Suite(&ConcurrentStoreSuite{}) + +// Hook up gocheck into the "go test" runnder +func ConcurrentTest(t *testing.T) { check.TestingT(t) } + +func (ConcurrentStoreSuite) TestOperations(c *check.C) { + mem := MemoryLocalStore() + store := NewConcurrentLocalStore(mem) + + c.Assert(store, check.NotNil) + expected := map[string]json.RawMessage{ + "file1.json": []byte{0xf1, 0xe1, 0xd1}, + "file2.json": []byte{0xf2, 0xe2, 0xd2}, + "file3.json": []byte{0xf3, 0xe3, 0xd3}, + } + + for k, v := range expected { + err := store.SetMeta(k, v) + c.Assert(err, check.IsNil) + } + + md, err := store.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 3) + c.Assert(md, check.DeepEquals, expected) + + // Nuke items + count := 3 + for k := range expected { + err = store.DeleteMeta(k) + count-- + c.Assert(err, check.IsNil) + md, err := store.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, count) + } + + md, err = store.GetMeta() + c.Assert(err, check.IsNil) + c.Assert(md, check.HasLen, 0) + + err = store.Close() + c.Assert(err, check.IsNil) +} diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index fb9493e7..06a73abd 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -24,7 +24,9 @@ type FileJSONStore struct { // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. -// The provided metadata cache is not safe for concurrent access. +// The provided metadata cache is not safe for concurrent access, if +// concurrent access safety is requires, wrap local store in a +// ConcurrentLocalStore. func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { return newImpl(baseDir, true) } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 7a6a139c..a6d7cfca 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -77,7 +77,8 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { } for k, v := range expected { - s.SetMeta(k, v) + err := s.SetMeta(k, v) + c.Assert(err, check.IsNil) } md, err := s.GetMeta() From 3a1cce046117f23e34740f0d832bb895981ce681 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 3 Aug 2022 13:43:12 +0200 Subject: [PATCH 26/43] Update client/filejsonstore/filejsonstore_test.go Fixed spelling error found during review. Co-authored-by: Joshua Lock Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index a6d7cfca..778f4808 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -15,7 +15,7 @@ type RawJSONStoreSuite struct{} var _ = check.Suite(&RawJSONStoreSuite{}) -// Hook up gocheck into the "go test" runnder +// Hook up gocheck into the "go test" runner func Test(t *testing.T) { check.TestingT(t) } func (RawJSONStoreSuite) TestNewOk(c *check.C) { From a42ffd546adf8c38a4d949112c3eda3282fd1429 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 4 Aug 2022 08:44:57 +0200 Subject: [PATCH 27/43] Added tests to make sure returned struct implements LocalStore interface. Signed-off-by: Fredrik Skogman --- client/concurrentlocalstore_test.go | 4 +++- client/filejsonstore/filejsonstore_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/client/concurrentlocalstore_test.go b/client/concurrentlocalstore_test.go index 4c883f23..c4824f5d 100644 --- a/client/concurrentlocalstore_test.go +++ b/client/concurrentlocalstore_test.go @@ -15,8 +15,10 @@ var _ = check.Suite(&ConcurrentStoreSuite{}) func ConcurrentTest(t *testing.T) { check.TestingT(t) } func (ConcurrentStoreSuite) TestOperations(c *check.C) { + var store LocalStore + mem := MemoryLocalStore() - store := NewConcurrentLocalStore(mem) + store = NewConcurrentLocalStore(mem) c.Assert(store, check.NotNil) expected := map[string]json.RawMessage{ diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 778f4808..e256f50d 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -9,6 +9,8 @@ import ( "testing" "gopkg.in/check.v1" + + "github.com/theupdateframework/go-tuf/client" ) type RawJSONStoreSuite struct{} @@ -160,3 +162,14 @@ func (RawJSONStoreSuite) TestDelete(c *check.C) { err = s.DeleteMeta("non_existing.json") c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } + +func (RawJSONStoreSuite) TestInterface(c *check.C) { + var localStore client.LocalStore + var err error + + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + localStore, err = NewFileJSONStore(p) + c.Assert(localStore, check.NotNil) + c.Assert(err, check.IsNil) +} From 0e6634c474e43659677db53edeffcdb6b5a6fcff Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Tue, 9 Aug 2022 17:54:30 +0200 Subject: [PATCH 28/43] Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 06a73abd..3272b04e 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -10,8 +10,8 @@ import ( "github.com/theupdateframework/go-tuf/util" ) -// ErrNotJSON is returned when a metadata operation is attempted to be -// performed against a file that does not seem to be a JSON file +// ErrNotJSON is returned when a metadata operation is attempted +// against a file that does not seem to be a JSON file // (e.g. does not end in .json, case sensitive). var ErrNotJSON = errors.New("file is not in JSON format") From 9c5b0241101c067592bf2cf505deb89d822ee262 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Tue, 9 Aug 2022 17:54:55 +0200 Subject: [PATCH 29/43] Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index e256f50d..130767bc 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -88,7 +88,7 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { c.Assert(md, check.HasLen, 3) c.Assert(md, check.DeepEquals, expected) - // Nuke items + // Delete all items count := 3 for k := range expected { err = s.DeleteMeta(k) From 0257d6deb559e3a36415e83bd696ebe6077a6154 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 10 Aug 2022 08:13:14 +0200 Subject: [PATCH 30/43] Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 3272b04e..dd13651f 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -65,7 +65,6 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { // GetMeta returns the currently cached set of metadata files. func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { names, err := os.ReadDir(f.baseDir) - if err != nil { return nil, err } From dafc16122c8fd3ebd6c290e681139229f007e8c0 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 10 Aug 2022 08:13:59 +0200 Subject: [PATCH 31/43] Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 130767bc..8fb4ece6 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -136,7 +136,6 @@ func (RawJSONStoreSuite) TestNoJSON(c *check.C) { err := s.SetMeta(f, []byte{}) c.Assert(err, check.Equals, ErrNotJSON) } - } func (RawJSONStoreSuite) TestClose(c *check.C) { From 96769ffa88a49fd0b95de88f254486ca0befacb5 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 10 Aug 2022 10:55:40 +0200 Subject: [PATCH 32/43] Made FileJSONStore safe for concurrent access. Moved IsMetaFile to new pkg, internal/fsutil Permission bits are validated when access the cache. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 82 +++++++++++++------- client/filejsonstore/filejsonstore_test.go | 88 +++++++++++++++++++--- internal/fsutil/fsutil.go | 39 ++++++++++ local_store.go | 5 +- util/util.go | 14 ---- 5 files changed, 174 insertions(+), 54 deletions(-) create mode 100644 internal/fsutil/fsutil.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index dd13651f..61442800 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "path/filepath" + "sync" + "github.com/theupdateframework/go-tuf/internal/fsutil" "github.com/theupdateframework/go-tuf/util" ) @@ -15,23 +17,33 @@ import ( // (e.g. does not end in .json, case sensitive). var ErrNotJSON = errors.New("file is not in JSON format") +// ErrToPermissive is returned when the metadata directory, or a metadata +// file has too permissive file mode bits set +var ErrTooPermissive = errors.New("permissions are too permissive") + +const ( + // user: rwx + // group: r-x + // other: --- + dirCreateMode = os.FileMode(0750) + // user: rw- + // group: r-- + // other: --- + fileCreateMode = os.FileMode(0640) +) + // FileJSONStore represents a local metadata cache relying on raw JSON files // as retrieved from the remote repository. type FileJSONStore struct { + mtx sync.RWMutex baseDir string } // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. -// The provided metadata cache is not safe for concurrent access, if -// concurrent access safety is requires, wrap local store in a -// ConcurrentLocalStore. +// The provided metadata cache is afe for concurrent access func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { - return newImpl(baseDir, true) -} - -func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { var f = FileJSONStore{ baseDir: baseDir, } @@ -41,22 +53,24 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { fi, err := os.Stat(baseDir) if err != nil { pe, ok := err.(*os.PathError) - if ok && errors.Is(pe.Err, os.ErrNotExist) && recurse { + if ok && errors.Is(pe.Err, os.ErrNotExist) { // Create the directory - // user: rwx - // group: r-x - // other: --- - err = os.MkdirAll(baseDir, 0750) - if err == nil { - return newImpl(baseDir, false) + if err = os.MkdirAll(baseDir, dirCreateMode); err != nil { + return nil, err } + } else { + return nil, err + } + } else { + // Verify that it is a directory + if !fi.IsDir() { + return nil, fmt.Errorf("can not open %s, not a directory", + baseDir) + } + // Verify file mode is not too permissive. + if err = fsutil.EnsurePermission(fi, dirCreateMode); err != nil { + return nil, ErrTooPermissive } - return nil, err - } - - if !fi.IsDir() { - return nil, fmt.Errorf("can not open %s, not a directory", - baseDir) } return &f, nil @@ -64,6 +78,9 @@ func newImpl(baseDir string, recurse bool) (*FileJSONStore, error) { // GetMeta returns the currently cached set of metadata files. func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { + f.mtx.RLock() + defer f.mtx.RUnlock() + names, err := os.ReadDir(f.baseDir) if err != nil { return nil, err @@ -72,11 +89,21 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { meta := map[string]json.RawMessage{} for _, name := range names { p := filepath.FromSlash(filepath.Join(f.baseDir, name.Name())) - if ok, err := util.IsMetaFile(name); !ok { + ok, err := fsutil.IsMetaFile(name) + if err != nil { + return nil, err + } + if !ok { continue - } else if err != nil { + } + // Verify permissions + info, err := name.Info() + if err != nil { return nil, err } + if err = fsutil.EnsurePermission(info, fileCreateMode); err != nil { + return nil, ErrTooPermissive + } b, err := os.ReadFile(p) if err != nil { return nil, err @@ -90,21 +117,24 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { // SetMeta stores a metadata file in the cache. If the metadata file exist, // it will be overwritten. func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { + f.mtx.Lock() + defer f.mtx.Unlock() + if filepath.Ext(name) != ".json" { return ErrNotJSON } p := filepath.FromSlash(filepath.Join(f.baseDir, name)) - // user: rw- - // group: r-- - // other: --- - err := util.AtomicallyWriteFile(p, meta, 0640) + err := util.AtomicallyWriteFile(p, meta, fileCreateMode) return err } // DeleteMeta deletes a metadata file from the cache. // If the file does not exist, an *os.PathError is returned. func (f *FileJSONStore) DeleteMeta(name string) error { + f.mtx.Lock() + defer f.mtx.Unlock() + if filepath.Ext(name) != ".json" { return ErrNotJSON } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 8fb4ece6..dfc54a62 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -13,14 +13,14 @@ import ( "github.com/theupdateframework/go-tuf/client" ) -type RawJSONStoreSuite struct{} +type FileJSONStoreSuite struct{} -var _ = check.Suite(&RawJSONStoreSuite{}) +var _ = check.Suite(&FileJSONStoreSuite{}) // Hook up gocheck into the "go test" runner func Test(t *testing.T) { check.TestingT(t) } -func (RawJSONStoreSuite) TestNewOk(c *check.C) { +func (FileJSONStoreSuite) TestNewOk(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") @@ -41,7 +41,7 @@ func (RawJSONStoreSuite) TestNewOk(c *check.C) { c.Assert(fi.IsDir(), check.Equals, true) } -func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { +func (FileJSONStoreSuite) TestNewFileExists(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") @@ -58,7 +58,37 @@ func (RawJSONStoreSuite) TestNewFileExists(c *check.C) { c.Assert(found, check.Equals, true) } -func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { +func (FileJSONStoreSuite) TestNewDirectoryExists(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + err := os.Mkdir(p, 0750) + c.Assert(err, check.IsNil) + + // Create implementation + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + // Modify the directory permission and try again + err = os.Chmod(p, 0751) + s, err = NewFileJSONStore(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.Equals, ErrTooPermissive) +} + +func (FileJSONStoreSuite) TestNewNoCreate(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Clear the write bit for the user + err := os.Chmod(tmp, 0551) + s, err := NewFileJSONStore(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.NotNil) +} + +func (FileJSONStoreSuite) TestGetMetaEmpty(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -68,7 +98,20 @@ func (RawJSONStoreSuite) TestGetMetaEmpty(c *check.C) { c.Assert(md, check.HasLen, 0) } -func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { +func (FileJSONStoreSuite) TestGetNoDirectory(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + + err = os.Remove(p) + c.Assert(err, check.IsNil) + + md, err := s.GetMeta() + c.Assert(md, check.IsNil) + c.Assert(err, check.NotNil) +} + +func (FileJSONStoreSuite) TestMetadataOperations(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -104,7 +147,7 @@ func (RawJSONStoreSuite) TestMetadataOperations(c *check.C) { c.Assert(md, check.HasLen, 0) } -func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { +func (FileJSONStoreSuite) TestGetNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -113,14 +156,32 @@ func (RawJSONStoreSuite) TestGetNoJSON(c *check.C) { // Create a file which does not end with '.json' fp := filepath.FromSlash(filepath.Join(p, "meta.xml")) - os.WriteFile(fp, []byte{}, 0644) + err = os.WriteFile(fp, []byte{}, 0644) + c.Assert(err, check.IsNil) md, err := s.GetMeta() c.Assert(err, check.IsNil) c.Assert(md, check.HasLen, 0) } -func (RawJSONStoreSuite) TestNoJSON(c *check.C) { +func (FileJSONStoreSuite) TestGetTooPermissive(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + // Create a file which does not end with '.json' + fp := filepath.FromSlash(filepath.Join(p, "meta.json")) + err = os.WriteFile(fp, []byte{}, 0644) + c.Assert(err, check.IsNil) + + md, err := s.GetMeta() + c.Assert(md, check.IsNil) + c.Assert(err, check.Equals, ErrTooPermissive) +} + +func (FileJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -138,7 +199,7 @@ func (RawJSONStoreSuite) TestNoJSON(c *check.C) { } } -func (RawJSONStoreSuite) TestClose(c *check.C) { +func (FileJSONStoreSuite) TestClose(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -149,7 +210,7 @@ func (RawJSONStoreSuite) TestClose(c *check.C) { c.Assert(err, check.IsNil) } -func (RawJSONStoreSuite) TestDelete(c *check.C) { +func (FileJSONStoreSuite) TestDelete(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) @@ -162,7 +223,7 @@ func (RawJSONStoreSuite) TestDelete(c *check.C) { c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } -func (RawJSONStoreSuite) TestInterface(c *check.C) { +func (FileJSONStoreSuite) TestInterface(c *check.C) { var localStore client.LocalStore var err error @@ -172,3 +233,6 @@ func (RawJSONStoreSuite) TestInterface(c *check.C) { c.Assert(localStore, check.NotNil) c.Assert(err, check.IsNil) } + +// test 3 file exist and is too permissive +// test 4 remote base dir after create diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go new file mode 100644 index 00000000..98883952 --- /dev/null +++ b/internal/fsutil/fsutil.go @@ -0,0 +1,39 @@ +// Package fsutil defiens a set of internal utility functions used to +// interact with the file system. +package fsutil + +import ( + "errors" + "io/fs" + "os" + "path/filepath" +) + +var ErrPermission = errors.New("unexpected permission") + +// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. +func IsMetaFile(e os.DirEntry) (bool, error) { + if e.IsDir() || filepath.Ext(e.Name()) != ".json" { + return false, nil + } + + info, err := e.Info() + if err != nil { + return false, err + } + + return info.Mode().IsRegular(), nil +} + +// EnsurePemissions tests the provided file info to make sure the +// permission bits matches the provided. +func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { + // Clear all bits which are note related to the permission. + mode := fi.Mode() & fs.ModePerm + mask := ^perm + if (mode & mask) != 0 { + return ErrPermission + } + + return nil +} diff --git a/local_store.go b/local_store.go index 86aed2c9..1409f13f 100644 --- a/local_store.go +++ b/local_store.go @@ -14,6 +14,7 @@ import ( "github.com/theupdateframework/go-tuf/data" "github.com/theupdateframework/go-tuf/encrypted" + "github.com/theupdateframework/go-tuf/internal/fsutil" "github.com/theupdateframework/go-tuf/internal/sets" "github.com/theupdateframework/go-tuf/pkg/keys" "github.com/theupdateframework/go-tuf/util" @@ -234,7 +235,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range committed { - imf, err := util.IsMetaFile(e) + imf, err := fsutil.IsMetaFile(e) if err != nil { return nil, err } @@ -251,7 +252,7 @@ func (f *fileSystemStore) GetMeta() (map[string]json.RawMessage, error) { } for _, e := range staged { - imf, err := util.IsMetaFile(e) + imf, err := fsutil.IsMetaFile(e) if err != nil { return nil, err } diff --git a/util/util.go b/util/util.go index dfcb67f3..0878c823 100644 --- a/util/util.go +++ b/util/util.go @@ -331,17 +331,3 @@ func AtomicallyWriteFile(filename string, data []byte, perm os.FileMode) error { return nil } - -// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. -func IsMetaFile(e os.DirEntry) (bool, error) { - if e.IsDir() || filepath.Ext(e.Name()) != ".json" { - return false, nil - } - - info, err := e.Info() - if err != nil { - return false, err - } - - return info.Mode().IsRegular(), nil -} From 86d4b3337d33d6cd4101bb066e3731d2276cb996 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Thu, 11 Aug 2022 15:42:46 +0200 Subject: [PATCH 33/43] Spelling error. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 61442800..90cfe24f 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -42,7 +42,7 @@ type FileJSONStore struct { // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. -// The provided metadata cache is afe for concurrent access +// The provided metadata cache is safe for concurrent access. func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { var f = FileJSONStore{ baseDir: baseDir, From c33254ed20f5053280b674a2c476c064aa51648a Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 12 Aug 2022 07:37:51 +0200 Subject: [PATCH 34/43] Updates based on PR comments. Removed a test for satisfying an interface, and replaced with a compile time check. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore.go | 3 +++ client/filejsonstore/filejsonstore_test.go | 16 ---------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 90cfe24f..c80d8ede 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sync" + "github.com/theupdateframework/go-tuf/client" "github.com/theupdateframework/go-tuf/internal/fsutil" "github.com/theupdateframework/go-tuf/util" ) @@ -39,6 +40,8 @@ type FileJSONStore struct { baseDir string } +var _ client.LocalStore = (*FileJSONStore)(nil) + // NewFileJSONStore returns a new metadata cache, implemented using raw JSON // files, stored in a directory provided by the client. // If the provided directory does not exist on disk, it will be created. diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index dfc54a62..1fdc4e8b 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -9,8 +9,6 @@ import ( "testing" "gopkg.in/check.v1" - - "github.com/theupdateframework/go-tuf/client" ) type FileJSONStoreSuite struct{} @@ -222,17 +220,3 @@ func (FileJSONStoreSuite) TestDelete(c *check.C) { err = s.DeleteMeta("non_existing.json") c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } - -func (FileJSONStoreSuite) TestInterface(c *check.C) { - var localStore client.LocalStore - var err error - - tmp := c.MkDir() - p := filepath.Join(tmp, "tuf_raw.db") - localStore, err = NewFileJSONStore(p) - c.Assert(localStore, check.NotNil) - c.Assert(err, check.IsNil) -} - -// test 3 file exist and is too permissive -// test 4 remote base dir after create From f95757cdb451ac3ece4fa23c2f1bd4e00c24951b Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Fri, 19 Aug 2022 08:25:55 +0200 Subject: [PATCH 35/43] Disabled filesystem permission checks for windows. Windows filesystem permission is a bit different from UNIX like systems, and there is no good API in go to manipulate it. Nor is having these checks a requirement by the TUF spec. This commit relies on build tags to inject the correct filesystem permssion check, the Windows version always return "satisfied". Signed-off-by: Fredrik Skogman --- internal/fsutil/fsutil.go | 14 -------------- internal/fsutil/perm.go | 22 ++++++++++++++++++++++ internal/fsutil/perm_windows.go | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 internal/fsutil/perm.go create mode 100644 internal/fsutil/perm_windows.go diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go index 98883952..d07c72a7 100644 --- a/internal/fsutil/fsutil.go +++ b/internal/fsutil/fsutil.go @@ -4,7 +4,6 @@ package fsutil import ( "errors" - "io/fs" "os" "path/filepath" ) @@ -24,16 +23,3 @@ func IsMetaFile(e os.DirEntry) (bool, error) { return info.Mode().IsRegular(), nil } - -// EnsurePemissions tests the provided file info to make sure the -// permission bits matches the provided. -func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { - // Clear all bits which are note related to the permission. - mode := fi.Mode() & fs.ModePerm - mask := ^perm - if (mode & mask) != 0 { - return ErrPermission - } - - return nil -} diff --git a/internal/fsutil/perm.go b/internal/fsutil/perm.go new file mode 100644 index 00000000..9b0cb2ea --- /dev/null +++ b/internal/fsutil/perm.go @@ -0,0 +1,22 @@ +//go:build !windows +// +build !windows + +package fsutil + +import ( + "io/fs" + "os" +) + +// EnsurePemissions tests the provided file info to make sure the +// permission bits matches the provided. +func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { + // Clear all bits which are note related to the permission. + mode := fi.Mode() & fs.ModePerm + mask := ^perm + if (mode & mask) != 0 { + return ErrPermission + } + + return nil +} diff --git a/internal/fsutil/perm_windows.go b/internal/fsutil/perm_windows.go new file mode 100644 index 00000000..2f8ed61c --- /dev/null +++ b/internal/fsutil/perm_windows.go @@ -0,0 +1,14 @@ +package fsutil + +import ( + "os" +) + +// EnsurePemissions tests the provided file info to make sure the +// permission bits matches the provided. +// On Windows system the permission bits are not really compatible with +// UNIX-like permission bits. +// Currently this method will always return nil. +func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { + return nil +} From 7301a8cb964a4dfcf22d9a96d02d7930abe3e0a9 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Mon, 22 Aug 2022 10:54:16 +0200 Subject: [PATCH 36/43] Moved the tests that rely on permission bits to a new test file. The permission bits are not executed during windows builds. Signed-off-by: Fredrik Skogman --- client/filejsonstore/filejsonstore_test.go | 34 -------------- client/filejsonstore/perm_test.go | 52 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 client/filejsonstore/perm_test.go diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 1fdc4e8b..73a7a731 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -67,23 +67,6 @@ func (FileJSONStoreSuite) TestNewDirectoryExists(c *check.C) { s, err := NewFileJSONStore(p) c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) - - // Modify the directory permission and try again - err = os.Chmod(p, 0751) - s, err = NewFileJSONStore(p) - c.Assert(s, check.IsNil) - c.Assert(err, check.Equals, ErrTooPermissive) -} - -func (FileJSONStoreSuite) TestNewNoCreate(c *check.C) { - tmp := c.MkDir() - p := filepath.Join(tmp, "tuf_raw.db") - - // Clear the write bit for the user - err := os.Chmod(tmp, 0551) - s, err := NewFileJSONStore(p) - c.Assert(s, check.IsNil) - c.Assert(err, check.NotNil) } func (FileJSONStoreSuite) TestGetMetaEmpty(c *check.C) { @@ -162,23 +145,6 @@ func (FileJSONStoreSuite) TestGetNoJSON(c *check.C) { c.Assert(md, check.HasLen, 0) } -func (FileJSONStoreSuite) TestGetTooPermissive(c *check.C) { - tmp := c.MkDir() - p := filepath.Join(tmp, "tuf_raw.db") - s, err := NewFileJSONStore(p) - c.Assert(s, check.NotNil) - c.Assert(err, check.IsNil) - - // Create a file which does not end with '.json' - fp := filepath.FromSlash(filepath.Join(p, "meta.json")) - err = os.WriteFile(fp, []byte{}, 0644) - c.Assert(err, check.IsNil) - - md, err := s.GetMeta() - c.Assert(md, check.IsNil) - c.Assert(err, check.Equals, ErrTooPermissive) -} - func (FileJSONStoreSuite) TestNoJSON(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") diff --git a/client/filejsonstore/perm_test.go b/client/filejsonstore/perm_test.go new file mode 100644 index 00000000..70be3b46 --- /dev/null +++ b/client/filejsonstore/perm_test.go @@ -0,0 +1,52 @@ +//go:build !windows +// +build !windows + +package client + +import ( + "os" + "path/filepath" + + "gopkg.in/check.v1" +) + +func (FileJSONStoreSuite) TestNewDirectoryExistsWrongPerm(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + err := os.Mkdir(p, 0750) + c.Assert(err, check.IsNil) + + // Modify the directory permission and try again + err = os.Chmod(p, 0751) + s, err := NewFileJSONStore(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.Equals, ErrTooPermissive) +} + +func (FileJSONStoreSuite) TestNewNoCreate(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + + // Clear the write bit for the user + err := os.Chmod(tmp, 0551) + s, err := NewFileJSONStore(p) + c.Assert(s, check.IsNil) + c.Assert(err, check.NotNil) +} + +func (FileJSONStoreSuite) TestGetTooPermissive(c *check.C) { + tmp := c.MkDir() + p := filepath.Join(tmp, "tuf_raw.db") + s, err := NewFileJSONStore(p) + c.Assert(s, check.NotNil) + c.Assert(err, check.IsNil) + + fp := filepath.FromSlash(filepath.Join(p, "meta.json")) + err = os.WriteFile(fp, []byte{}, 0644) + c.Assert(err, check.IsNil) + + md, err := s.GetMeta() + c.Assert(md, check.IsNil) + c.Assert(err, check.Equals, ErrTooPermissive) +} From 39f4018e9f28e0822df1a4026ff3d0316175d04b Mon Sep 17 00:00:00 2001 From: Ethan Lowman Date: Tue, 23 Aug 2022 17:39:00 -0400 Subject: [PATCH 37/43] Clarify permissions check naming and comment, add tests Signed-off-by: Ethan Lowman --- client/filejsonstore/filejsonstore.go | 4 +- internal/fsutil/fsutil.go | 3 -- internal/fsutil/perm.go | 26 ++++++---- internal/fsutil/perm_test.go | 71 +++++++++++++++++++++++++++ internal/fsutil/perm_windows.go | 4 +- 5 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 internal/fsutil/perm_test.go diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index c80d8ede..b4375437 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -71,7 +71,7 @@ func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { baseDir) } // Verify file mode is not too permissive. - if err = fsutil.EnsurePermission(fi, dirCreateMode); err != nil { + if err = fsutil.EnsureMaxPermissions(fi, dirCreateMode); err != nil { return nil, ErrTooPermissive } } @@ -104,7 +104,7 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { if err != nil { return nil, err } - if err = fsutil.EnsurePermission(info, fileCreateMode); err != nil { + if err = fsutil.EnsureMaxPermissions(info, fileCreateMode); err != nil { return nil, ErrTooPermissive } b, err := os.ReadFile(p) diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go index d07c72a7..774246f8 100644 --- a/internal/fsutil/fsutil.go +++ b/internal/fsutil/fsutil.go @@ -3,13 +3,10 @@ package fsutil import ( - "errors" "os" "path/filepath" ) -var ErrPermission = errors.New("unexpected permission") - // IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. func IsMetaFile(e os.DirEntry) (bool, error) { if e.IsDir() || filepath.Ext(e.Name()) != ".json" { diff --git a/internal/fsutil/perm.go b/internal/fsutil/perm.go index 9b0cb2ea..f94add60 100644 --- a/internal/fsutil/perm.go +++ b/internal/fsutil/perm.go @@ -4,18 +4,26 @@ package fsutil import ( - "io/fs" + "fmt" "os" ) -// EnsurePemissions tests the provided file info to make sure the -// permission bits matches the provided. -func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { - // Clear all bits which are note related to the permission. - mode := fi.Mode() & fs.ModePerm - mask := ^perm - if (mode & mask) != 0 { - return ErrPermission +// EnsureMaxPermissions tests the provided file info, returning an error if the +// file's permission bits contain excess permissions not set in maxPerms. +// +// For example, a file with permissions -rw------- will successfully validate +// with maxPerms -rw-r--r-- or -rw-rw-r--, but will not validate with maxPerms +// -r-------- (due to excess --w------- permission) or --w------- (due to +// excess -r-------- permission). +// +// Only permission bits of the file modes are considered. +func EnsureMaxPermissions(fi os.FileInfo, maxPerms os.FileMode) error { + gotPerm := fi.Mode().Perm() + forbiddenPerms := (^maxPerms).Perm() + excessPerms := gotPerm & forbiddenPerms + + if excessPerms != 0 { + return fmt.Errorf("permission bits for file %v failed validation: want at most %v, got %v with excess perms %v", fi.Name(), maxPerms.Perm(), gotPerm, excessPerms) } return nil diff --git a/internal/fsutil/perm_test.go b/internal/fsutil/perm_test.go new file mode 100644 index 00000000..6061d291 --- /dev/null +++ b/internal/fsutil/perm_test.go @@ -0,0 +1,71 @@ +//go:build !windows +// +build !windows + +package fsutil + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnsureMaxPermissions(t *testing.T) { + tmp := t.TempDir() + p := filepath.Join(tmp, "file.txt") + + // Start with 0644 and change using os.Chmod so umask doesn't interfere. + err := os.WriteFile(p, []byte(`AAA`), 0644) + assert.NoError(t, err) + + // Check matching (1) + err = os.Chmod(p, 0464) + assert.NoError(t, err) + fi, err := os.Stat(p) + assert.NoError(t, err) + err = EnsureMaxPermissions(fi, os.FileMode(0464)) + assert.NoError(t, err) + + // Check matching (2) + err = os.Chmod(p, 0642) + assert.NoError(t, err) + fi, err = os.Stat(p) + assert.NoError(t, err) + err = EnsureMaxPermissions(fi, os.FileMode(0642)) + assert.NoError(t, err) + + // Check matching with file mode bits + err = os.Chmod(p, 0444) + assert.NoError(t, err) + fi, err = os.Stat(p) + assert.NoError(t, err) + err = EnsureMaxPermissions(fi, os.ModeSymlink|os.ModeAppend|os.FileMode(0444)) + assert.NoError(t, err) + + // Check not matching (1) + err = os.Chmod(p, 0444) + assert.NoError(t, err) + fi, err = os.Stat(p) + assert.NoError(t, err) + err = EnsureMaxPermissions(fi, os.FileMode(0400)) + assert.Error(t, err) + + // Check not matching (2) + err = os.Chmod(p, 0444) + assert.NoError(t, err) + fi, err = os.Stat(p) + assert.NoError(t, err) + err = EnsureMaxPermissions(fi, os.FileMode(0222)) + assert.Error(t, err) + fmt.Println(err) + + // Check matching due to more restrictive perms on file + err = os.Chmod(p, 0444) + assert.NoError(t, err) + fi, err = os.Stat(p) + assert.NoError(t, err) + err = EnsureMaxPermissions(fi, os.FileMode(0666)) + assert.NoError(t, err) +} diff --git a/internal/fsutil/perm_windows.go b/internal/fsutil/perm_windows.go index 2f8ed61c..11b0d0c7 100644 --- a/internal/fsutil/perm_windows.go +++ b/internal/fsutil/perm_windows.go @@ -4,11 +4,11 @@ import ( "os" ) -// EnsurePemissions tests the provided file info to make sure the +// EnsureMaxPermissions tests the provided file info to make sure the // permission bits matches the provided. // On Windows system the permission bits are not really compatible with // UNIX-like permission bits. // Currently this method will always return nil. -func EnsurePermission(fi os.FileInfo, perm os.FileMode) error { +func EnsureMaxPermissions(fi os.FileInfo, perm os.FileMode) error { return nil } From 01bbd76469b9719e169b222d825777ebd27dd722 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 24 Aug 2022 07:40:57 +0200 Subject: [PATCH 38/43] Update internal/fsutil/fsutil.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman --- internal/fsutil/fsutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go index 774246f8..a169609d 100644 --- a/internal/fsutil/fsutil.go +++ b/internal/fsutil/fsutil.go @@ -7,7 +7,7 @@ import ( "path/filepath" ) -// IsMetaFile tests wheter a DirEntry appears to be a metaddata file or not. +// IsMetaFile tests wheter a DirEntry appears to be a metadata file or not. func IsMetaFile(e os.DirEntry) (bool, error) { if e.IsDir() || filepath.Ext(e.Name()) != ".json" { return false, nil From 19ef1ace68213f03c278ee98e2e3668f47d22cd2 Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Wed, 24 Aug 2022 08:28:25 +0200 Subject: [PATCH 39/43] Updates based on feeback. Mostly minor things like removing sentinel errors, wrapping errors from std library. Signed-off-by: Fredrik Skogman --- client/concurrentlocalstore.go | 49 ------------------- client/concurrentlocalstore_test.go | 57 ---------------------- client/filejsonstore/filejsonstore.go | 43 +++++++--------- client/filejsonstore/filejsonstore_test.go | 5 +- client/filejsonstore/perm_test.go | 6 +-- internal/fsutil/fsutil.go | 3 +- internal/fsutil/perm_windows.go | 5 +- 7 files changed, 30 insertions(+), 138 deletions(-) delete mode 100644 client/concurrentlocalstore.go delete mode 100644 client/concurrentlocalstore_test.go diff --git a/client/concurrentlocalstore.go b/client/concurrentlocalstore.go deleted file mode 100644 index f37f7aa1..00000000 --- a/client/concurrentlocalstore.go +++ /dev/null @@ -1,49 +0,0 @@ -package client - -import ( - "encoding/json" - "sync" -) - -// ConcurrentLocalStore provides a LocalStore over an existing LocalStore -// implementation that is safe for concurrent access. -type ConcurrentLocalStore struct { - mtx sync.RWMutex - store LocalStore -} - -// NewConcurrentLocalStore returns a wrapped LocalStore that is safe for -// concurrent access. -func NewConcurrentLocalStore(s LocalStore) *ConcurrentLocalStore { - return &ConcurrentLocalStore{ - store: s, - } -} - -// GetMeta returns all known targets. -func (c *ConcurrentLocalStore) GetMeta() (map[string]json.RawMessage, error) { - c.mtx.RLock() - defer c.mtx.RUnlock() - return c.store.GetMeta() -} - -// SetMeta updates the lcoal store with a new target. -func (c *ConcurrentLocalStore) SetMeta(name string, meta json.RawMessage) error { - c.mtx.Lock() - defer c.mtx.Unlock() - return c.store.SetMeta(name, meta) -} - -// DeleteMeta remves a target from the cache. -func (c *ConcurrentLocalStore) DeleteMeta(name string) error { - c.mtx.Lock() - defer c.mtx.Unlock() - return c.store.DeleteMeta(name) -} - -// Close closes the local store. -func (c *ConcurrentLocalStore) Close() error { - c.mtx.Lock() - defer c.mtx.Unlock() - return c.store.Close() -} diff --git a/client/concurrentlocalstore_test.go b/client/concurrentlocalstore_test.go deleted file mode 100644 index c4824f5d..00000000 --- a/client/concurrentlocalstore_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package client - -import ( - "encoding/json" - "testing" - - "gopkg.in/check.v1" -) - -type ConcurrentStoreSuite struct{} - -var _ = check.Suite(&ConcurrentStoreSuite{}) - -// Hook up gocheck into the "go test" runnder -func ConcurrentTest(t *testing.T) { check.TestingT(t) } - -func (ConcurrentStoreSuite) TestOperations(c *check.C) { - var store LocalStore - - mem := MemoryLocalStore() - store = NewConcurrentLocalStore(mem) - - c.Assert(store, check.NotNil) - expected := map[string]json.RawMessage{ - "file1.json": []byte{0xf1, 0xe1, 0xd1}, - "file2.json": []byte{0xf2, 0xe2, 0xd2}, - "file3.json": []byte{0xf3, 0xe3, 0xd3}, - } - - for k, v := range expected { - err := store.SetMeta(k, v) - c.Assert(err, check.IsNil) - } - - md, err := store.GetMeta() - c.Assert(err, check.IsNil) - c.Assert(md, check.HasLen, 3) - c.Assert(md, check.DeepEquals, expected) - - // Nuke items - count := 3 - for k := range expected { - err = store.DeleteMeta(k) - count-- - c.Assert(err, check.IsNil) - md, err := store.GetMeta() - c.Assert(err, check.IsNil) - c.Assert(md, check.HasLen, count) - } - - md, err = store.GetMeta() - c.Assert(err, check.IsNil) - c.Assert(md, check.HasLen, 0) - - err = store.Close() - c.Assert(err, check.IsNil) -} diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index b4375437..fba83571 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -13,15 +13,6 @@ import ( "github.com/theupdateframework/go-tuf/util" ) -// ErrNotJSON is returned when a metadata operation is attempted -// against a file that does not seem to be a JSON file -// (e.g. does not end in .json, case sensitive). -var ErrNotJSON = errors.New("file is not in JSON format") - -// ErrToPermissive is returned when the metadata directory, or a metadata -// file has too permissive file mode bits set -var ErrTooPermissive = errors.New("permissions are too permissive") - const ( // user: rwx // group: r-x @@ -47,22 +38,20 @@ var _ client.LocalStore = (*FileJSONStore)(nil) // If the provided directory does not exist on disk, it will be created. // The provided metadata cache is safe for concurrent access. func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { - var f = FileJSONStore{ + f := &FileJSONStore{ baseDir: baseDir, } - var err error // Does the directory exist? fi, err := os.Stat(baseDir) if err != nil { - pe, ok := err.(*os.PathError) - if ok && errors.Is(pe.Err, os.ErrNotExist) { + if errors.Is(err, os.ErrNotExist) { // Create the directory if err = os.MkdirAll(baseDir, dirCreateMode); err != nil { - return nil, err + return nil, fmt.Errorf("error creating directory for metadata cache: %w", err) } } else { - return nil, err + return nil, fmt.Errorf("error getting FileInfo for %s: %w", baseDir, err) } } else { // Verify that it is a directory @@ -72,11 +61,11 @@ func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { } // Verify file mode is not too permissive. if err = fsutil.EnsureMaxPermissions(fi, dirCreateMode); err != nil { - return nil, ErrTooPermissive + return nil, err } } - return &f, nil + return f, nil } // GetMeta returns the currently cached set of metadata files. @@ -86,12 +75,11 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { names, err := os.ReadDir(f.baseDir) if err != nil { - return nil, err + return nil, fmt.Errorf("error reading directory %s: %w", f.baseDir, err) } meta := map[string]json.RawMessage{} for _, name := range names { - p := filepath.FromSlash(filepath.Join(f.baseDir, name.Name())) ok, err := fsutil.IsMetaFile(name) if err != nil { return nil, err @@ -102,14 +90,15 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { // Verify permissions info, err := name.Info() if err != nil { - return nil, err + return nil, fmt.Errorf("error retrieving FileInfo for %s: %w", name.Name(), err) } if err = fsutil.EnsureMaxPermissions(info, fileCreateMode); err != nil { - return nil, ErrTooPermissive + return nil, err } + p := filepath.Join(f.baseDir, name.Name()) b, err := os.ReadFile(p) if err != nil { - return nil, err + return nil, fmt.Errorf("error reading file %s: %w", name.Name(), err) } meta[name.Name()] = b } @@ -124,7 +113,7 @@ func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { defer f.mtx.Unlock() if filepath.Ext(name) != ".json" { - return ErrNotJSON + return fmt.Errorf("file %s is not a JSON file", name) } p := filepath.FromSlash(filepath.Join(f.baseDir, name)) @@ -139,12 +128,16 @@ func (f *FileJSONStore) DeleteMeta(name string) error { defer f.mtx.Unlock() if filepath.Ext(name) != ".json" { - return ErrNotJSON + return fmt.Errorf("file %s is not a JSON file", name) } p := filepath.FromSlash(filepath.Join(f.baseDir, name)) err := os.Remove(p) - return err + + if err == nil { + return nil + } + return fmt.Errorf("error deleting file %s: %w", name, err) } // Close closes the metadata cache. This is a no-op. diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 73a7a731..d3382718 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -159,7 +159,8 @@ func (FileJSONStoreSuite) TestNoJSON(c *check.C) { } for _, f := range files { err := s.SetMeta(f, []byte{}) - c.Assert(err, check.Equals, ErrNotJSON) + c.Assert(err, check.ErrorMatches, "file.*is not a JSON file") + } } @@ -182,7 +183,7 @@ func (FileJSONStoreSuite) TestDelete(c *check.C) { c.Assert(err, check.IsNil) err = s.DeleteMeta("not_json.yml") - c.Assert(err, check.Equals, ErrNotJSON) + c.Assert(err, check.ErrorMatches, "file not_json\\.yml is not a JSON file") err = s.DeleteMeta("non_existing.json") c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true) } diff --git a/client/filejsonstore/perm_test.go b/client/filejsonstore/perm_test.go index 70be3b46..85efb723 100644 --- a/client/filejsonstore/perm_test.go +++ b/client/filejsonstore/perm_test.go @@ -21,7 +21,7 @@ func (FileJSONStoreSuite) TestNewDirectoryExistsWrongPerm(c *check.C) { err = os.Chmod(p, 0751) s, err := NewFileJSONStore(p) c.Assert(s, check.IsNil) - c.Assert(err, check.Equals, ErrTooPermissive) + c.Assert(err, check.ErrorMatches, "permission bits for file tuf_raw.db failed.*") } func (FileJSONStoreSuite) TestNewNoCreate(c *check.C) { @@ -42,11 +42,11 @@ func (FileJSONStoreSuite) TestGetTooPermissive(c *check.C) { c.Assert(s, check.NotNil) c.Assert(err, check.IsNil) - fp := filepath.FromSlash(filepath.Join(p, "meta.json")) + fp := filepath.Join(p, "meta.json") err = os.WriteFile(fp, []byte{}, 0644) c.Assert(err, check.IsNil) md, err := s.GetMeta() c.Assert(md, check.IsNil) - c.Assert(err, check.Equals, ErrTooPermissive) + c.Assert(err, check.ErrorMatches, "permission bits for file meta.json failed.*") } diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go index a169609d..138f2103 100644 --- a/internal/fsutil/fsutil.go +++ b/internal/fsutil/fsutil.go @@ -3,6 +3,7 @@ package fsutil import ( + "fmt" "os" "path/filepath" ) @@ -15,7 +16,7 @@ func IsMetaFile(e os.DirEntry) (bool, error) { info, err := e.Info() if err != nil { - return false, err + return false, fmt.Errorf("error retrieving FileInfo for %s: %w", e.Name(), err) } return info.Mode().IsRegular(), nil diff --git a/internal/fsutil/perm_windows.go b/internal/fsutil/perm_windows.go index 11b0d0c7..cecfee65 100644 --- a/internal/fsutil/perm_windows.go +++ b/internal/fsutil/perm_windows.go @@ -7,7 +7,10 @@ import ( // EnsureMaxPermissions tests the provided file info to make sure the // permission bits matches the provided. // On Windows system the permission bits are not really compatible with -// UNIX-like permission bits. +// UNIX-like permission bits. By setting the UNIX-like permission bits +// on a Windows system only read/write by all users can be achieved. +// See this issue for tracking and more details: +// https://github.com/theupdateframework/go-tuf/issues/360 // Currently this method will always return nil. func EnsureMaxPermissions(fi os.FileInfo, perm os.FileMode) error { return nil From db6ad66e7938408e4206cb4a596a946338a78f7c Mon Sep 17 00:00:00 2001 From: Ethan Lowman Date: Wed, 24 Aug 2022 12:43:50 -0400 Subject: [PATCH 40/43] Use fs.ErrNotExist instead of os.ErrNotExist as recommended by Go docs Signed-off-by: Ethan Lowman --- client/filejsonstore/filejsonstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index fba83571..6f2c445c 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "os" "path/filepath" "sync" @@ -45,7 +46,7 @@ func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { // Does the directory exist? fi, err := os.Stat(baseDir) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Create the directory if err = os.MkdirAll(baseDir, dirCreateMode); err != nil { return nil, fmt.Errorf("error creating directory for metadata cache: %w", err) @@ -56,8 +57,7 @@ func NewFileJSONStore(baseDir string) (*FileJSONStore, error) { } else { // Verify that it is a directory if !fi.IsDir() { - return nil, fmt.Errorf("can not open %s, not a directory", - baseDir) + return nil, fmt.Errorf("can not open %s, not a directory", baseDir) } // Verify file mode is not too permissive. if err = fsutil.EnsureMaxPermissions(fi, dirCreateMode); err != nil { From ff675c651614b1bb33347a986a7fd06941355a86 Mon Sep 17 00:00:00 2001 From: Ethan Lowman Date: Wed, 24 Aug 2022 12:45:48 -0400 Subject: [PATCH 41/43] Clean up remaining unnecessary usage of filepath.FromSlash Signed-off-by: Ethan Lowman --- client/filejsonstore/filejsonstore.go | 4 ++-- client/filejsonstore/filejsonstore_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 6f2c445c..94950b10 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -116,7 +116,7 @@ func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error { return fmt.Errorf("file %s is not a JSON file", name) } - p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + p := filepath.Join(f.baseDir, name) err := util.AtomicallyWriteFile(p, meta, fileCreateMode) return err } @@ -131,7 +131,7 @@ func (f *FileJSONStore) DeleteMeta(name string) error { return fmt.Errorf("file %s is not a JSON file", name) } - p := filepath.FromSlash(filepath.Join(f.baseDir, name)) + p := filepath.Join(f.baseDir, name) err := os.Remove(p) if err == nil { diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index d3382718..2b78b85c 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -136,7 +136,7 @@ func (FileJSONStoreSuite) TestGetNoJSON(c *check.C) { c.Assert(err, check.IsNil) // Create a file which does not end with '.json' - fp := filepath.FromSlash(filepath.Join(p, "meta.xml")) + fp := filepath.Join(p, "meta.xml") err = os.WriteFile(fp, []byte{}, 0644) c.Assert(err, check.IsNil) From 9ca46f4ffa3022ae032c449e520f58e91dcb48cf Mon Sep 17 00:00:00 2001 From: Ethan Lowman Date: Wed, 24 Aug 2022 13:47:23 -0400 Subject: [PATCH 42/43] Add missing error checks in tests Signed-off-by: Ethan Lowman --- client/filejsonstore/filejsonstore.go | 4 +++- client/filejsonstore/filejsonstore_test.go | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go index 94950b10..41277c88 100644 --- a/client/filejsonstore/filejsonstore.go +++ b/client/filejsonstore/filejsonstore.go @@ -87,6 +87,7 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { if !ok { continue } + // Verify permissions info, err := name.Info() if err != nil { @@ -95,6 +96,7 @@ func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) { if err = fsutil.EnsureMaxPermissions(info, fileCreateMode); err != nil { return nil, err } + p := filepath.Join(f.baseDir, name.Name()) b, err := os.ReadFile(p) if err != nil { @@ -133,10 +135,10 @@ func (f *FileJSONStore) DeleteMeta(name string) error { p := filepath.Join(f.baseDir, name) err := os.Remove(p) - if err == nil { return nil } + return fmt.Errorf("error deleting file %s: %w", name, err) } diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go index 2b78b85c..6d00e046 100644 --- a/client/filejsonstore/filejsonstore_test.go +++ b/client/filejsonstore/filejsonstore_test.go @@ -73,6 +73,7 @@ func (FileJSONStoreSuite) TestGetMetaEmpty(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) + c.Assert(err, check.IsNil) md, err := s.GetMeta() c.Assert(err, check.IsNil) @@ -83,6 +84,7 @@ func (FileJSONStoreSuite) TestGetNoDirectory(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) + c.Assert(err, check.IsNil) err = os.Remove(p) c.Assert(err, check.IsNil) @@ -96,6 +98,8 @@ func (FileJSONStoreSuite) TestMetadataOperations(c *check.C) { tmp := c.MkDir() p := filepath.Join(tmp, "tuf_raw.db") s, err := NewFileJSONStore(p) + c.Assert(err, check.IsNil) + expected := map[string]json.RawMessage{ "file1.json": []byte{0xf1, 0xe1, 0xd1}, "file2.json": []byte{0xf2, 0xe2, 0xd2}, @@ -116,10 +120,12 @@ func (FileJSONStoreSuite) TestMetadataOperations(c *check.C) { count := 3 for k := range expected { err = s.DeleteMeta(k) - count-- c.Assert(err, check.IsNil) + md, err := s.GetMeta() c.Assert(err, check.IsNil) + + count-- c.Assert(md, check.HasLen, count) } @@ -160,7 +166,6 @@ func (FileJSONStoreSuite) TestNoJSON(c *check.C) { for _, f := range files { err := s.SetMeta(f, []byte{}) c.Assert(err, check.ErrorMatches, "file.*is not a JSON file") - } } From 210b417dde860182155655dceb6739bfa2fa7fcb Mon Sep 17 00:00:00 2001 From: Fredrik Skogman Date: Mon, 12 Sep 2022 19:37:44 +0200 Subject: [PATCH 43/43] Make sure to test that returned err is nil in the tests. Signed-off-by: Fredrik Skogman --- client/filejsonstore/perm_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/filejsonstore/perm_test.go b/client/filejsonstore/perm_test.go index 85efb723..cda58c43 100644 --- a/client/filejsonstore/perm_test.go +++ b/client/filejsonstore/perm_test.go @@ -19,6 +19,7 @@ func (FileJSONStoreSuite) TestNewDirectoryExistsWrongPerm(c *check.C) { // Modify the directory permission and try again err = os.Chmod(p, 0751) + c.Assert(err, check.IsNil) s, err := NewFileJSONStore(p) c.Assert(s, check.IsNil) c.Assert(err, check.ErrorMatches, "permission bits for file tuf_raw.db failed.*") @@ -30,6 +31,7 @@ func (FileJSONStoreSuite) TestNewNoCreate(c *check.C) { // Clear the write bit for the user err := os.Chmod(tmp, 0551) + c.Assert(err, check.IsNil) s, err := NewFileJSONStore(p) c.Assert(s, check.IsNil) c.Assert(err, check.NotNil)