fix(ci): omit surrogate keys from the list keys, Caddy cypress E2E tests

pkg/storage/badgerProvider.go

@@ -127,7 +127,10 @@ func (provider *Badger) ListKeys() []string {
 		it := txn.NewIterator(opts)
 		defer it.Close()
 		for it.Rewind(); it.Valid(); it.Next() {
-			keys = append(keys, string(it.Item().Key()))
+			key := string(it.Item().Key())
+			if !strings.Contains(key, surrogatePrefix) {
+				keys = append(keys, key)
+			}
 		}
 		return nil
 	})

pkg/storage/embeddedOlricProvider.go

@@ -126,7 +126,9 @@ func (provider *EmbeddedOlric) ListKeys() []string {
 
 	keys := []string{}
 	for records.Next() {
-		keys = append(keys, records.Key())
+		if !strings.Contains(records.Key(), surrogatePrefix) {
+			keys = append(keys, records.Key())
+		}
 	}
 	records.Close()
 
@@ -144,7 +146,8 @@ func (provider *EmbeddedOlric) MapKeys(prefix string) map[string]string {
 	keys := map[string]string{}
 	for records.Next() {
 		if strings.HasPrefix(records.Key(), prefix) {
-			keys[records.Key()] = string(provider.Get(records.Key()))
+			k, _ := strings.CutPrefix(records.Key(), prefix)
+			keys[k] = string(provider.Get(records.Key()))
 		}
 	}
 	records.Close()

pkg/storage/etcdProvider.go

@@ -85,7 +85,9 @@ func (provider *Etcd) ListKeys() []string {
 		return []string{}
 	}
 	for _, k := range r.Kvs {
-		keys = append(keys, string(k.Key))
+		if !strings.Contains(string(k.Key), surrogatePrefix) {
+			keys = append(keys, string(k.Key))
+		}
 	}
 
 	return keys
@@ -110,7 +112,8 @@ func (provider *Etcd) MapKeys(prefix string) map[string]string {
 	for _, k := range r.Kvs {
 		key := string(k.Key)
 		if strings.HasPrefix(key, prefix) {
-			keys[key] = string(k.Value)
+			nk, _ := strings.CutPrefix(key, prefix)
+			keys[nk] = string(k.Value)
 		}
 	}
 

pkg/storage/nutsProvider.go

@@ -17,6 +17,8 @@ import (
 	"go.uber.org/zap"
 )
 
+var nutsInstanceMap = map[string]*nutsdb.DB{}
+
 // Nuts provider type
 type Nuts struct {
 	*nutsdb.DB
@@ -93,17 +95,29 @@ func NutsConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, er
 		}
 	}
 
+	if instance, ok := nutsInstanceMap[nutsOptions.Dir]; ok && instance != nil {
+		return &Nuts{
+			DB:     instance,
+			stale:  dc.GetStale(),
+			logger: c.GetLogger(),
+		}, nil
+	}
+
 	db, e := nutsdb.Open(nutsOptions)
 
 	if e != nil {
 		c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e)
+		return nil, e
 	}
 
-	return &Nuts{
+	instance := &Nuts{
 		DB:     db,
 		stale:  dc.GetStale(),
 		logger: c.GetLogger(),
-	}, nil
+	}
+	nutsInstanceMap[nutsOptions.Dir] = instance.DB
+
+	return instance, nil
 }
 
 // Name returns the storer name
@@ -118,7 +132,9 @@ func (provider *Nuts) ListKeys() []string {
 	e := provider.DB.View(func(tx *nutsdb.Tx) error {
 		e, _ := tx.GetAll(bucket)
 		for _, k := range e {
-			keys = append(keys, string(k.Key))
+			if !strings.Contains(string(k.Key), surrogatePrefix) {
+				keys = append(keys, string(k.Key))
+			}
 		}
 		return nil
 	})
@@ -138,7 +154,8 @@ func (provider *Nuts) MapKeys(prefix string) map[string]string {
 		e, _ := tx.GetAll(bucket)
 		for _, k := range e {
 			if strings.HasPrefix(string(k.Key), prefix) {
-				keys[string(k.Key)] = string(k.Value)
+				nk, _ := strings.CutPrefix(string(k.Key), prefix)
+				keys[nk] = string(k.Value)
 			}
 		}
 		return nil

pkg/storage/nutsProvider_test.go

@@ -17,7 +17,7 @@ func getNutsClientAndMatchedURL(key string) (types.Storer, configurationtypes.UR
 	return GetCacheProviderClientAndMatchedURL(
 		key,
 		func() configurationtypes.AbstractConfigurationInterface {
-			return tests.MockConfiguration(tests.BaseConfiguration)
+			return tests.MockConfiguration(tests.NutsConfiguration)
 		},
 		func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
 			provider, _ := NutsConnectionFactory(config)
@@ -31,7 +31,6 @@ func getNutsClientAndMatchedURL(key string) (types.Storer, configurationtypes.UR
 func TestNutsConnectionFactory(t *testing.T) {
 	c := tests.MockConfiguration(tests.NutsConfiguration)
 	r, err := NutsConnectionFactory(c)
-	defer r.(*Nuts).DB.Close()
 
 	if nil != err {
 		errors.GenerateError(t, "Shouldn't have panic")
@@ -40,11 +39,14 @@ func TestNutsConnectionFactory(t *testing.T) {
 	if nil == r {
 		errors.GenerateError(t, "Nuts should be instanciated")
 	}
+
+	if nil == r.(*Nuts).DB {
+		errors.GenerateError(t, "Nuts database should be accesible")
+	}
 }
 
 func TestIShouldBeAbleToReadAndWriteDataInNuts(t *testing.T) {
 	client, matchedURL := getNutsClientAndMatchedURL("Test")
-	defer client.(*Nuts).DB.Close()
 
 	_ = client.Set("Test", []byte(BASE_VALUE), matchedURL, time.Duration(20)*time.Second)
 	time.Sleep(1 * time.Second)
@@ -61,7 +63,6 @@ func TestIShouldBeAbleToReadAndWriteDataInNuts(t *testing.T) {
 func TestNuts_GetRequestInCache(t *testing.T) {
 	c := tests.MockConfiguration(tests.BaseConfiguration)
 	client, _ := NutsConnectionFactory(c)
-	defer client.(*Nuts).DB.Close()
 	res := client.Get(NONEXISTENTKEY)
 	if 0 < len(res) {
 		errors.GenerateError(t, fmt.Sprintf("Key %s should not exist", NONEXISTENTKEY))
@@ -70,7 +71,6 @@ func TestNuts_GetRequestInCache(t *testing.T) {
 
 func TestNuts_GetSetRequestInCache_OneByte(t *testing.T) {
 	client, matchedURL := getNutsClientAndMatchedURL(BYTEKEY)
-	defer client.(*Nuts).DB.Close()
 	_ = client.Set(BYTEKEY, []byte("A"), matchedURL, time.Duration(20)*time.Second)
 	time.Sleep(1 * time.Second)
 
@@ -87,14 +87,12 @@ func TestNuts_GetSetRequestInCache_OneByte(t *testing.T) {
 func TestNuts_SetRequestInCache_TTL(t *testing.T) {
 	key := "MyEmptyKey"
 	client, matchedURL := getNutsClientAndMatchedURL(key)
-	defer client.(*Nuts).DB.Close()
 	nv := []byte("Hello world")
 	setValueThenVerify(client, key, nv, matchedURL, time.Duration(20)*time.Second, t)
 }
 
 func TestNuts_DeleteRequestInCache(t *testing.T) {
 	client, _ := NutsConnectionFactory(tests.MockConfiguration(tests.BaseConfiguration))
-	defer client.(*Nuts).DB.Close()
 	client.Delete(BYTEKEY)
 	time.Sleep(1 * time.Second)
 	if 0 < len(client.Get(BYTEKEY)) {
@@ -104,7 +102,6 @@ func TestNuts_DeleteRequestInCache(t *testing.T) {
 
 func TestNuts_Init(t *testing.T) {
 	client, _ := NutsConnectionFactory(tests.MockConfiguration(tests.BaseConfiguration))
-	defer client.(*Nuts).DB.Close()
 	err := client.Init()
 
 	if nil != err {

pkg/storage/olricProvider.go

@@ -72,7 +72,9 @@ func (provider *Olric) ListKeys() []string {
 
 	keys := []string{}
 	for records.Next() {
-		keys = append(keys, records.Key())
+		if !strings.Contains(records.Key(), surrogatePrefix) {
+			keys = append(keys, records.Key())
+		}
 	}
 	records.Close()
 
@@ -100,7 +102,8 @@ func (provider *Olric) MapKeys(prefix string) map[string]string {
 	keys := map[string]string{}
 	for records.Next() {
 		if strings.HasPrefix(records.Key(), prefix) {
-			keys[records.Key()] = string(provider.Get(records.Key()))
+			k, _ := strings.CutPrefix(records.Key(), prefix)
+			keys[k] = string(provider.Get(records.Key()))
 		}
 	}
 	records.Close()

pkg/storage/redisProvider.go

@@ -100,7 +100,8 @@ func (provider *Redis) MapKeys(prefix string) map[string]string {
 	keys, _ := provider.Client.Scan(provider.ctx, 0, "*", 0).Val()
 	for _, key := range keys {
 		if strings.HasPrefix(key, prefix) {
-			m[key] = string(provider.Get(key))
+			k, _ := strings.CutPrefix(key, prefix)
+			m[k] = string(provider.Get(key))
 		}
 	}
 

pkg/storage/storage.go

@@ -15,6 +15,7 @@ const (
 	encodedHeaderSemiColonSeparator = "%3B"
 	encodedHeaderColonSeparator     = "%3A"
 	StalePrefix                     = "STALE_"
+	surrogatePrefix                 = "SURROGATE_"
 )
 
 type StorerInstanciator func(configurationtypes.AbstractConfigurationInterface) (types.Storer, error)

pkg/surrogate/providers/common.go

@@ -33,6 +33,14 @@ const (
 	surrogatePrefix = "SURROGATE_"
 )
 
+var storageToInfiniteTTLMap = map[string]time.Duration{
+	"BADGER": 365 * 24 * time.Hour,
+	"ETCD":   365 * 24 * time.Hour,
+	"NUTS":   0,
+	"OLRIC":  365 * 24 * time.Hour,
+	"REDIS":  0,
+}
+
 func (s *baseStorage) ParseHeaders(value string) []string {
 	return regexp.MustCompile(s.parent.getHeaderSeparator()+" *").Split(value, -1)
 }
@@ -88,13 +96,14 @@ type baseStorage struct {
 	keepStale  bool
 	logger     *zap.Logger
 	mu         *sync.Mutex
+	duration   time.Duration
 }
 
 func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterface) {
 	if configuration, ok := config.GetSurrogateKeys()["_configuration"]; ok {
 		instanciator, err := storage.NewStorageFromName(configuration.SurrogateConfiguration.Storer)
 		if err != nil {
-			instanciator, _ = storage.NewStorageFromName("badger")
+			instanciator, _ = storage.NewStorageFromName("nuts")
 		}
 
 		storer, err := instanciator(config)
@@ -104,7 +113,7 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf
 
 		s.Storage = storer
 	} else {
-		instanciator, _ := storage.NewStorageFromName("badger")
+		instanciator, _ := storage.NewStorageFromName("nuts")
 		storer, err := instanciator(config)
 		if err != nil {
 			panic(fmt.Sprintf("Impossible to instanciate the storer for the surrogate-keys: %v", err))
@@ -140,17 +149,16 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf
 	s.logger = config.GetLogger()
 	s.keysRegexp = keysRegexp
 	s.mu = &sync.Mutex{}
+	s.duration = storageToInfiniteTTLMap[s.Storage.Name()]
 }
 
 func (s *baseStorage) storeTag(tag string, cacheKey string, re *regexp.Regexp) {
 	defer s.mu.Unlock()
 	s.mu.Lock()
 	currentValue := string(s.Storage.Get(surrogatePrefix + tag))
-	if s.dynamic {
-		if !re.MatchString(currentValue) {
-			s.logger.Sugar().Debugf("Store the tag %s", tag)
-			_ = s.Storage.Set(surrogatePrefix+tag, []byte(currentValue+souinStorageSeparator+cacheKey), configurationtypes.URL{}, time.Hour)
-		}
+	if !re.MatchString(currentValue) {
+		s.logger.Sugar().Debugf("Store the tag %s", tag)
+		_ = s.Storage.Set(surrogatePrefix+tag, []byte(currentValue+souinStorageSeparator+cacheKey), configurationtypes.URL{}, s.duration)
 	}
 }
 

pkg/surrogate/providers/common_test.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/errors"
@@ -21,7 +20,7 @@ const (
 )
 
 func mockCommonProvider() (*baseStorage, func() error) {
-	instanciator, _ := storage.NewStorageFromName("badger")
+	instanciator, _ := storage.NewStorageFromName("nuts")
 	config := tests.MockConfiguration(tests.NutsConfiguration)
 	config.DefaultCache.Badger.Configuration = nil
 	storer, _ := instanciator(config)
@@ -38,7 +37,7 @@ func mockCommonProvider() (*baseStorage, func() error) {
 
 	sss.baseStorage.parent = sss
 
-	return sss.baseStorage, storer.(*storage.Badger).Close
+	return sss.baseStorage, storer.(*storage.Nuts).Close
 }
 
 func TestBaseStorage_ParseHeaders(t *testing.T) {
@@ -81,11 +80,11 @@ func TestBaseStorage_Purge(t *testing.T) {
 		errors.GenerateError(t, "The surrogates length should be equal to 0.")
 	}
 
-	_ = bs.Storage.Set("test0", []byte("first,second"), configurationtypes.URL{}, time.Hour)
-	_ = bs.Storage.Set("STALE_test0", []byte("STALE_first,STALE_second"), configurationtypes.URL{}, time.Hour)
-	_ = bs.Storage.Set("test2", []byte("third,fourth"), configurationtypes.URL{}, time.Hour)
-	_ = bs.Storage.Set("test5", []byte("first,second,fifth"), configurationtypes.URL{}, time.Hour)
-	_ = bs.Storage.Set("testInvalid", []byte("invalid"), configurationtypes.URL{}, time.Hour)
+	_ = bs.Storage.Set("test0", []byte("first,second"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("STALE_test0", []byte("STALE_first,STALE_second"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("test2", []byte("third,fourth"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("test5", []byte("first,second,fifth"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("testInvalid", []byte("invalid"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
 	headerMock.Set(surrogateKey, baseHeaderValue)
 	tags, surrogates = bs.Purge(headerMock)
 
@@ -111,11 +110,10 @@ func TestBaseStorage_Store(t *testing.T) {
 		errors.GenerateError(t, "It shouldn't throw an error with a valid key.")
 	}
 
-	bs.Storage.DeleteMany("")
-	_ = bs.Storage.Set("test0", []byte("first,second"), configurationtypes.URL{}, -1)
-	_ = bs.Storage.Set("test2", []byte("third,fourth"), configurationtypes.URL{}, -1)
-	_ = bs.Storage.Set("test5", []byte("first,second,fifth"), configurationtypes.URL{}, -1)
-	_ = bs.Storage.Set("testInvalid", []byte("invalid"), configurationtypes.URL{}, -1)
+	_ = bs.Storage.Set("test0", []byte("first,second"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("test2", []byte("third,fourth"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("test5", []byte("first,second,fifth"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
+	_ = bs.Storage.Set("testInvalid", []byte("invalid"), configurationtypes.URL{}, storageToInfiniteTTLMap[bs.Storage.Name()])
 
 	if e = bs.Store(&res, "stored"); e != nil {
 		errors.GenerateError(t, "It shouldn't throw an error with a valid key.")
@@ -124,7 +122,7 @@ func TestBaseStorage_Store(t *testing.T) {
 	for i := 0; i < 5; i++ {
 		value := bs.Storage.Get(fmt.Sprintf(surrogatePrefix+"test%d", i))
 		if !strings.Contains(string(value), "stored") {
-			errors.GenerateError(t, fmt.Sprintf("The key test%d must include stored, %s given.", i, string(value)))
+			errors.GenerateError(t, fmt.Sprintf("The key %stest%d must include stored, %s given.", surrogatePrefix, i, string(value)))
 		}
 	}
 
@@ -133,7 +131,6 @@ func TestBaseStorage_Store(t *testing.T) {
 		errors.GenerateError(t, "The surrogate storage should not contain stored.")
 	}
 
-	bs.Storage.DeleteMany("")
 	res.Header.Set(surrogateKey, "something")
 	_ = bs.Store(&res, "/something")
 	_ = bs.Store(&res, "/something")
@@ -142,8 +139,8 @@ func TestBaseStorage_Store(t *testing.T) {
 
 	storageSize := len(bs.Storage.ListKeys())
 
-	if storageSize != 4 {
-		errors.GenerateError(t, "The surrogate storage should contain 4 stored elements.")
+	if storageSize != 9 {
+		errors.GenerateError(t, fmt.Sprintf("The surrogate storage should contain 9 stored elements, %v given: %#v.\n", storageSize, bs.Storage.ListKeys()))
 	}
 
 	value = bs.Storage.Get("SURROGATE_STALE_something")

plugins/traefik/override/storage/cacheProvider.go

@@ -46,7 +46,8 @@ func (provider *Cache) ListKeys() []string {
 func (provider *Cache) MapKeys(prefix string) map[string]string {
 	var keys map[string]string
 	provider.Cache.Range(func(key, value interface{}) bool {
-		keys[key.(string)] = value.(string)
+		k, _ := strings.CutPrefix(key.(string), prefix)
+		keys[k] = value.(string)
 		return true
 	})