feat(redis): redis implementation for MetaDB

Signed-off-by: Andrei Aaron <[email protected]>

Author: andaaron

pkg/api/controller_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/google/go-github/v62/github"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/securecookie"
@@ -154,6 +155,44 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 		So(err, ShouldBeNil)
 		So(driver, ShouldBeNil)
 	})
+	Convey("Test CreateCacheDatabaseDriver redisdb", t, func() {
+		miniRedis := miniredis.RunT(t)
+
+		log := log.NewLogger("debug", "")
+
+		// fail create db, no perm
+		dir := t.TempDir()
+		conf := config.New()
+		conf.Storage.RootDirectory = dir
+		conf.Storage.Dedupe = true
+		conf.Storage.RemoteCache = true
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+			"url":  "redis://" + miniRedis.Addr(),
+		}
+
+		// test initialization for S3 storage
+		conf.Storage.StorageDriver = map[string]interface{}{
+			"name":          "s3",
+			"rootdirectory": "/zot",
+			"url":           "us-east-2",
+		}
+
+		driver, err := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
+		So(err, ShouldBeNil)
+		So(driver, ShouldNotBeNil)
+		So(driver.Name(), ShouldEqual, "redis")
+		So(driver.UsesRelativePaths(), ShouldEqual, false)
+
+		// test initialization for local storage
+		conf.Storage.StorageDriver = nil
+
+		driver, err = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
+		So(err, ShouldBeNil)
+		So(driver, ShouldNotBeNil)
+		So(driver.Name(), ShouldEqual, "redis")
+		So(driver.UsesRelativePaths(), ShouldEqual, true)
+	})
 	tskip.SkipDynamo(t)
 	tskip.SkipS3(t)
 	Convey("Test CreateCacheDatabaseDriver dynamodb", t, func() {
@@ -303,6 +342,58 @@ func TestCreateMetaDBDriver(t *testing.T) {
 		So(testFunc, ShouldNotPanic)
 	})
 
+	Convey("Test create MetaDB redis", t, func() {
+		miniRedis := miniredis.RunT(t)
+
+		log := log.NewLogger("debug", "")
+		dir := t.TempDir()
+		conf := config.New()
+		conf.Storage.RootDirectory = dir
+		conf.Storage.Dedupe = true
+		conf.Storage.RemoteCache = true
+		conf.Storage.StorageDriver = map[string]interface{}{
+			"name":          "s3",
+			"rootdirectory": "/zot",
+			"region":        "us-east-2",
+			"bucket":        "zot-storage",
+			"secure":        true,
+			"skipverify":    false,
+		}
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "dummy",
+		}
+
+		metaDB, err := meta.New(conf.Storage.StorageConfig, log)
+		So(err, ShouldNotBeNil)
+		So(metaDB, ShouldBeNil)
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+		}
+
+		testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
+		So(testFunc, ShouldPanic)
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+			"url":  "url",
+		}
+
+		metaDB, err = meta.New(conf.Storage.StorageConfig, log)
+		So(err, ShouldNotBeNil)
+		So(metaDB, ShouldBeNil)
+
+		conf.Storage.CacheDriver = map[string]interface{}{
+			"name": "redis",
+			"url":  "redis://" + miniRedis.Addr(),
+		}
+
+		metaDB, err = meta.New(conf.Storage.StorageConfig, log)
+		So(err, ShouldBeNil)
+		So(metaDB, ShouldNotBeNil)
+	})
+
 	Convey("Test create MetaDB bolt", t, func() {
 		log := log.NewLogger("debug", "")
 		dir := t.TempDir()

pkg/extensions/extension_image_trust.go

@@ -19,6 +19,7 @@ import (
 	"zotregistry.dev/zot/pkg/log"
 	mTypes "zotregistry.dev/zot/pkg/meta/types"
 	"zotregistry.dev/zot/pkg/scheduler"
+	sconstants "zotregistry.dev/zot/pkg/storage/constants"
 )
 
 func IsBuiltWithImageTrustExtension() bool {
@@ -172,7 +173,11 @@ func SetupImageTrustExtension(conf *config.Config, metaDB mTypes.MetaDB, log log
 
 	var err error
 
-	if conf.Storage.RemoteCache {
+	if conf.Storage.RemoteCache && conf.Storage.CacheDriver["name"] == sconstants.DynamoDBDriverName {
+		// AWS secrets manager
+		// In case of AWS let's assume if dynamodDB is used, the AWS secrets manager is also used
+		// we use the CacheDriver settings as opposed to the storage settings because we want to
+		// be able to use S3/minio and redis in the same configuration
 		endpoint, _ := conf.Storage.CacheDriver["endpoint"].(string)
 		region, _ := conf.Storage.CacheDriver["region"].(string)
 
@@ -181,6 +186,7 @@ func SetupImageTrustExtension(conf *config.Config, metaDB mTypes.MetaDB, log log
 			return err
 		}
 	} else {
+		// Store secrets on the local disk
 		imgTrustStore, err = imagetrust.NewLocalImageTrustStore(conf.Storage.RootDirectory)
 		if err != nil {
 			return err

pkg/extensions/extension_image_trust_test.go

@@ -17,6 +17,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	guuid "github.com/gofrs/uuid"
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/generate"
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/options"
@@ -123,6 +124,19 @@ func TestSignatureUploadAndVerificationLocal(t *testing.T) {
 	})
 }
 
+func TestSignatureUploadAndVerificationRedis(t *testing.T) {
+	Convey("test with local storage and redis metadb", t, func() {
+		miniRedis := miniredis.RunT(t)
+
+		cacheDriverParams := map[string]interface{}{
+			"name": "redis",
+			"url":  "redis://" + miniRedis.Addr(),
+		}
+
+		RunSignatureUploadAndVerificationTests(t, cacheDriverParams)
+	})
+}
+
 func TestSignatureUploadAndVerificationAWS(t *testing.T) {
 	tskip.SkipDynamo(t)
 

pkg/extensions/imagetrust/image_trust_test.go

@@ -16,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
@@ -653,6 +654,25 @@ func TestLocalTrustStore(t *testing.T) {
 	})
 }
 
+func TestLocalTrustStoreRedis(t *testing.T) {
+	miniRedis := miniredis.RunT(t)
+
+	Convey("test local storage and redis", t, func() {
+		rootDir := t.TempDir()
+
+		imageTrustStore, err := imagetrust.NewLocalImageTrustStore(rootDir)
+		So(err, ShouldBeNil)
+
+		dbDriverParams := map[string]interface{}{
+			"name": "redis",
+			"url":  "redis://" + miniRedis.Addr(),
+		}
+
+		RunUploadTests(t, *imageTrustStore)
+		RunVerificationTests(t, dbDriverParams)
+	})
+}
+
 func TestAWSTrustStore(t *testing.T) {
 	tskip.SkipDynamo(t)
 

pkg/meta/meta.go

@@ -39,7 +39,7 @@ func New(storageConfig config.StorageConfig, log log.Logger) (mTypes.MetaDB, err
 				return nil, err
 			}
 
-			return Create(sconstants.RedisDriverName, client, &redisdb.RedisDB{Client: client}, log) //nolint:contextcheck
+			return Create(sconstants.RedisDriverName, client, nil, log) //nolint:contextcheck
 		}
 
 		// this behavior is also mentioned in the configuration validation logic inside the cli package