feat(redis): redis implementation for MetaDB

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

pkg/api/controller_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/google/go-github/v62/github"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/securecookie"
@@ -154,6 +155,44 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 		So(err, ShouldBeNil)
 		So(driver, ShouldBeNil)
 	})
+	Convey("Test CreateCacheDatabaseDriver redisdb", t, func() {
+		miniRedis := miniredis.RunT(t)
+
+		log := log.NewLogger("debug", "")
+
+		// fail create db, no perm
+		dir := t.TempDir()
+		conf := config.New()
+		conf.Storage.RootDirectory = dir
+		conf.Storage.Dedupe = true
+		conf.Storage.RemoteCache = true
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+			"url":  "redis://" + miniRedis.Addr(),
+		}
+
+		// test initialization for S3 storage
+		conf.Storage.StorageDriver = map[string]interface{}{
+			"name":          "s3",
+			"rootdirectory": "/zot",
+			"url":           "us-east-2",
+		}
+
+		driver, err := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
+		So(err, ShouldBeNil)
+		So(driver, ShouldNotBeNil)
+		So(driver.Name(), ShouldEqual, "redis")
+		So(driver.UsesRelativePaths(), ShouldEqual, false)
+
+		// test initialization for local storage
+		conf.Storage.StorageDriver = nil
+
+		driver, err = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
+		So(err, ShouldBeNil)
+		So(driver, ShouldNotBeNil)
+		So(driver.Name(), ShouldEqual, "redis")
+		So(driver.UsesRelativePaths(), ShouldEqual, true)
+	})
 	tskip.SkipDynamo(t)
 	tskip.SkipS3(t)
 	Convey("Test CreateCacheDatabaseDriver dynamodb", t, func() {
@@ -303,6 +342,58 @@ func TestCreateMetaDBDriver(t *testing.T) {
 		So(testFunc, ShouldNotPanic)
 	})
 
+	Convey("Test create MetaDB redis", t, func() {
+		miniRedis := miniredis.RunT(t)
+
+		log := log.NewLogger("debug", "")
+		dir := t.TempDir()
+		conf := config.New()
+		conf.Storage.RootDirectory = dir
+		conf.Storage.Dedupe = true
+		conf.Storage.RemoteCache = true
+		conf.Storage.StorageDriver = map[string]interface{}{
+			"name":          "s3",
+			"rootdirectory": "/zot",
+			"region":        "us-east-2",
+			"bucket":        "zot-storage",
+			"secure":        true,
+			"skipverify":    false,
+		}
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "dummy",
+		}
+
+		metaDB, err := meta.New(conf.Storage.StorageConfig, log)
+		So(err, ShouldNotBeNil)
+		So(metaDB, ShouldBeNil)
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+		}
+
+		testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
+		So(testFunc, ShouldPanic)
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+			"url":  "url",
+		}
+
+		metaDB, err = meta.New(conf.Storage.StorageConfig, log)
+		So(err, ShouldNotBeNil)
+		So(metaDB, ShouldBeNil)
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+			"url":  "redis://" + miniRedis.Addr(),
+		}
+
+		metaDB, err = meta.New(conf.Storage.StorageConfig, log)
+		So(err, ShouldBeNil)
+		So(metaDB, ShouldNotBeNil)
+	})
+
 	Convey("Test create MetaDB bolt", t, func() {
 		log := log.NewLogger("debug", "")
 		dir := t.TempDir()

pkg/meta/meta_test.go

@@ -11,13 +11,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
 	guuid "github.com/gofrs/uuid"
 	"github.com/notaryproject/notation-core-go/signature/jws"
 	"github.com/notaryproject/notation-go"
 	"github.com/notaryproject/notation-go/signer"
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/redis/go-redis/v9"
 	. "github.com/smartystreets/goconvey/convey"
 
 	zcommon "zotregistry.dev/zot/pkg/common"
@@ -28,6 +30,7 @@ import (
 	"zotregistry.dev/zot/pkg/meta/boltdb"
 	"zotregistry.dev/zot/pkg/meta/common"
 	mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb"
+	"zotregistry.dev/zot/pkg/meta/redisdb"
 	mTypes "zotregistry.dev/zot/pkg/meta/types"
 	reqCtx "zotregistry.dev/zot/pkg/requestcontext"
 	tCommon "zotregistry.dev/zot/pkg/test/common"
@@ -164,6 +167,35 @@ func TestDynamoDBWrapper(t *testing.T) {
 	})
 }
 
+func TestRedisDB(t *testing.T) {
+	miniRedis := miniredis.RunT(t)
+
+	Convey("RedisDB Wrapper", t, func() {
+		rootDir := t.TempDir()
+		log := log.NewLogger("debug", "")
+
+		redisDriver, err := redisdb.GetRedisClient("redis://" + miniRedis.Addr())
+		So(err, ShouldBeNil)
+
+		metaDB, err := redisdb.New(redisDriver, log)
+		So(metaDB, ShouldNotBeNil)
+		So(err, ShouldBeNil)
+
+		imgTrustStore, err := imagetrust.NewLocalImageTrustStore(rootDir)
+		So(err, ShouldBeNil)
+
+		metaDB.SetImageTrustStore(imgTrustStore)
+
+		defer func() {
+			metaDB.ResetDB() //nolint: errcheck
+			os.RemoveAll(path.Join(rootDir, "_cosign"))
+			os.RemoveAll(path.Join(rootDir, "_notation"))
+		}()
+
+		RunMetaDBTests(t, metaDB)
+	})
+}
+
 func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func() error) { //nolint: thelper
 	ctx := context.Background()
 
@@ -1616,9 +1648,20 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
 				So(err, ShouldBeNil)
 				So(len(repoMetaList), ShouldEqual, 2)
 
-				So(repoMetaList[0].Tags[tag1].Digest, ShouldResemble, image1.DigestStr())
-				So(repoMetaList[0].Tags[tag2].Digest, ShouldResemble, image2.DigestStr())
-				So(repoMetaList[1].Tags[tag3].Digest, ShouldResemble, image3.DigestStr())
+				repos := map[string]map[string]string{}
+				for _, repoMeta := range repoMetaList {
+					if _, exists := repos[repoMeta.Name]; !exists {
+						repos[repoMeta.Name] = map[string]string{}
+					}
+
+					for tag, descriptor := range repoMeta.Tags {
+						repos[repoMeta.Name][tag] = descriptor.Digest
+					}
+				}
+
+				So(repos[repo1][tag1], ShouldEqual, image1.DigestStr())
+				So(repos[repo1][tag2], ShouldEqual, image2.DigestStr())
+				So(repos[repo2][tag3], ShouldEqual, image3.DigestStr())
 			})
 
 			Convey("Search a repo by name", func() {
@@ -2577,3 +2620,34 @@ func TestCreateBoltDB(t *testing.T) {
 		So(err, ShouldNotBeNil)
 	})
 }
+
+func TestCreateRedisDB(t *testing.T) {
+	Convey("Create", t, func() {
+		miniRedis := miniredis.RunT(t)
+
+		log := log.NewLogger("debug", "")
+		So(log, ShouldNotBeNil)
+
+		redisDriver, err := redisdb.GetRedisClient("redis://" + miniRedis.Addr())
+		So(err, ShouldBeNil)
+
+		metaDB, err := meta.Create("redis", redisDriver, nil, log)
+		So(metaDB, ShouldNotBeNil)
+		So(err, ShouldBeNil)
+	})
+
+	Convey("fails", t, func() {
+		log := log.NewLogger("debug", "")
+
+		_, err := meta.Create("redis", nil, mdynamodb.DBDriverParameters{}, log)
+		So(err, ShouldNotBeNil)
+
+		// Redis client will not be responding
+		redisURL := "redis://127.0.0.1:" + tCommon.GetFreePort() // must not match miniRedis.Addr()
+		connOpts, _ := redis.ParseURL(redisURL)
+		cacheDB := redis.NewClient(connOpts)
+
+		_, err = meta.Create("redis", cacheDB, nil, log)
+		So(err, ShouldNotBeNil)
+	})
+}

pkg/meta/redisdb/buckets.go

@@ -0,0 +1,12 @@
+package redisdb
+
+// MetadataDB.
+const (
+	ImageMetaBuck       = "zot:ImageMeta"
+	RepoMetaBuck        = "zot:RepoMeta"
+	RepoBlobsBuck       = "zot:RepoBlobsMeta"
+	RepoLastUpdatedBuck = "zot:RepoLastUpdated"
+	UserDataBucket      = "zot:UserData"
+	VersionBucket       = "zot:Version"
+	UserAPIKeysBucket   = "zot:UserAPIKeys" //nolint: gosec // these are not hardcoded credentials
+)