feat(chore): surrogate-keys distributed-storage

configurationtypes/types.go

@@ -377,8 +377,13 @@ type API struct {
 	Security   SecurityAPI `json:"security" yaml:"security"`
 }
 
+type SurrogateConfiguration struct {
+	Storer string `json:"storer", yaml:"storer"`
+}
+
 // SurrogateKeys structure define the way surrogate keys are stored
 type SurrogateKeys struct {
+	SurrogateConfiguration
 	URL     string            `json:"url" yaml:"url"`
 	Headers map[string]string `json:"headers" yaml:"headers"`
 }

pkg/middleware/middleware.go

@@ -161,16 +161,17 @@ func (s *SouinBaseHandler) Store(
 		return nil
 	}
 
-	if customWriter.Header().Get("Cache-Control") == "" {
+	cacheControlName := s.SurrogateKeyStorer.GetSurrogateControlName()
+	if customWriter.Header().Get(cacheControlName) == "" {
 		// TODO see with @mnot if mandatory to not store the response when no Cache-Control given.
 		// if s.DefaultMatchedUrl.DefaultCacheControl == "" {
 		// 	customWriter.Header().Set("Cache-Status", fmt.Sprintf("%s; fwd=uri-miss; key=%s; detail=EMPTY-RESPONSE-CACHE-CONTROL", rq.Context().Value(context.CacheName), rfc.GetCacheKeyFromCtx(rq.Context())))
 		// 	return nil
 		// }
-		customWriter.Header().Set("Cache-Control", s.DefaultMatchedUrl.DefaultCacheControl)
+		customWriter.Header().Set(cacheControlName, s.DefaultMatchedUrl.DefaultCacheControl)
 	}
 
-	responseCc, _ := cacheobject.ParseResponseCacheControl(customWriter.Header().Get("Cache-Control"))
+	responseCc, _ := cacheobject.ParseResponseCacheControl(customWriter.Header().Get(cacheControlName))
 	s.Configuration.GetLogger().Sugar().Debugf("Response cache-control %+v", responseCc)
 	if responseCc == nil {
 		customWriter.Header().Set("Cache-Status", fmt.Sprintf("%s; fwd=uri-miss; key=%s; detail=INVALID-RESPONSE-CACHE-CONTROL", rq.Context().Value(context.CacheName), rfc.GetCacheKeyFromCtx(rq.Context())))
@@ -346,8 +347,9 @@ func (s *SouinBaseHandler) Upstream(
 			}
 		}
 
-		if customWriter.Header().Get("Cache-Control") == "" {
-			customWriter.Header().Set("Cache-Control", s.DefaultMatchedUrl.DefaultCacheControl)
+		cacheControlName := s.SurrogateKeyStorer.GetSurrogateControlName()
+		if customWriter.Header().Get(cacheControlName) == "" {
+			customWriter.Header().Set(cacheControlName, s.DefaultMatchedUrl.DefaultCacheControl)
 		}
 
 		err := s.Store(customWriter, rq, requestCc, cachedKey)
@@ -527,6 +529,7 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 			}
 		}
 
+		cacheControlName := s.SurrogateKeyStorer.GetSurrogateControlName()
 		if response != nil && (!modeContext.Strict || rfc.ValidateCacheControl(response, requestCc)) {
 			if validator.ResponseETag != "" && validator.Matched {
 				rfc.SetCacheStatusHeader(response)
@@ -554,7 +557,8 @@ func (s *SouinBaseHandler) ServeHTTP(rw http.ResponseWriter, rq *http.Request, n
 
 				return err
 			}
-			if resCc, _ := cacheobject.ParseResponseCacheControl(response.Header.Get("Cache-Control")); resCc.NoCachePresent {
+
+			if resCc, _ := cacheobject.ParseResponseCacheControl(response.Header.Get(cacheControlName)); resCc.NoCachePresent {
 				prometheus.Increment(prometheus.NoCachedResponseCounter)
 				err := s.Revalidate(validator, next, customWriter, req, requestCc, cachedKey)
 				_, _ = customWriter.Send()

pkg/storage/badgerProvider.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"regexp"
+	"strings"
 	"time"
 
 	t "github.com/darkweak/souin/configurationtypes"
@@ -92,6 +93,29 @@ func (provider *Badger) Name() string {
 	return "BADGER"
 }
 
+// MapKeys method returns a map with the key and value
+func (provider *Badger) MapKeys(prefix string) map[string]string {
+	keys := map[string]string{}
+
+	_ = provider.DB.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = false
+		it := txn.NewIterator(opts)
+		defer it.Close()
+		for it.Rewind(); it.ValidForPrefix([]byte(prefix)); it.Next() {
+			it.Item().Value(func(val []byte) error {
+				k, _ := strings.CutPrefix(string(it.Item().Key()), prefix)
+				keys[k] = string(val)
+
+				return nil
+			})
+		}
+		return nil
+	})
+
+	return keys
+}
+
 // ListKeys method returns the list of existing keys
 func (provider *Badger) ListKeys() []string {
 	keys := []string{}

pkg/storage/embeddedOlricProvider.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/buraksezer/olric"
@@ -132,6 +133,25 @@ func (provider *EmbeddedOlric) ListKeys() []string {
 	return keys
 }
 
+// MapKeys method returns a map with the key and value
+func (provider *EmbeddedOlric) MapKeys(prefix string) map[string]string {
+	records, err := provider.dm.Scan(provider.ct)
+	if err != nil {
+		provider.logger.Sugar().Errorf("An error occurred while trying to map keys in Olric: %s\n", err)
+		return map[string]string{}
+	}
+
+	keys := map[string]string{}
+	for records.Next() {
+		if strings.HasPrefix(records.Key(), prefix) {
+			keys[records.Key()] = string(provider.Get(records.Key()))
+		}
+	}
+	records.Close()
+
+	return keys
+}
+
 // Prefix method returns the populated response if exists, empty response then
 func (provider *EmbeddedOlric) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response {
 	records, err := provider.dm.Scan(provider.ct, olric.Match("^"+key+"({|$)"))

pkg/storage/etcdProvider.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 	"time"
 
 	t "github.com/darkweak/souin/configurationtypes"
@@ -90,6 +91,32 @@ func (provider *Etcd) ListKeys() []string {
 	return keys
 }
 
+// MapKeys method returns the map of existing keys
+func (provider *Etcd) MapKeys(prefix string) map[string]string {
+	if provider.reconnecting {
+		provider.logger.Sugar().Error("Impossible to list the etcd keys while reconnecting.")
+		return map[string]string{}
+	}
+
+	keys := map[string]string{}
+	r, e := provider.Client.Get(provider.ctx, "\x00", clientv3.WithFromKey())
+
+	if e != nil {
+		if !provider.reconnecting {
+			go provider.Reconnect()
+		}
+		return map[string]string{}
+	}
+	for _, k := range r.Kvs {
+		key := string(k.Key)
+		if strings.HasPrefix(key, prefix) {
+			keys[key] = string(k.Value)
+		}
+	}
+
+	return keys
+}
+
 // Get method returns the populated response if exists, empty response then
 func (provider *Etcd) Get(key string) (item []byte) {
 	if provider.reconnecting {

pkg/storage/nutsProvider.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	t "github.com/darkweak/souin/configurationtypes"
@@ -129,6 +130,27 @@ func (provider *Nuts) ListKeys() []string {
 	return keys
 }
 
+// MapKeys method returns the map of existing keys
+func (provider *Nuts) MapKeys(prefix string) map[string]string {
+	keys := map[string]string{}
+
+	e := provider.DB.View(func(tx *nutsdb.Tx) error {
+		e, _ := tx.GetAll(bucket)
+		for _, k := range e {
+			if strings.HasPrefix(string(k.Key), prefix) {
+				keys[string(k.Key)] = string(k.Value)
+			}
+		}
+		return nil
+	})
+
+	if e != nil {
+		return map[string]string{}
+	}
+
+	return keys
+}
+
 // Get method returns the populated response if exists, empty response then
 func (provider *Nuts) Get(key string) (item []byte) {
 	_ = provider.DB.View(func(tx *nutsdb.Tx) error {

pkg/storage/olricProvider.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -78,6 +79,35 @@ func (provider *Olric) ListKeys() []string {
 	return keys
 }
 
+// MapKeys method returns the map of existing keys
+func (provider *Olric) MapKeys(prefix string) map[string]string {
+	if provider.reconnecting {
+		provider.logger.Sugar().Error("Impossible to list the olric keys while reconnecting.")
+		return map[string]string{}
+	}
+	dm := provider.dm.Get().(olric.DMap)
+	defer provider.dm.Put(dm)
+
+	records, err := dm.Scan(context.Background())
+	if err != nil {
+		if !provider.reconnecting {
+			go provider.Reconnect()
+		}
+		provider.logger.Sugar().Error("An error occurred while trying to list keys in Olric: %s\n", err)
+		return map[string]string{}
+	}
+
+	keys := map[string]string{}
+	for records.Next() {
+		if strings.HasPrefix(records.Key(), prefix) {
+			keys[records.Key()] = string(provider.Get(records.Key()))
+		}
+	}
+	records.Close()
+
+	return keys
+}
+
 // Prefix method returns the populated response if exists, empty response then
 func (provider *Olric) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response {
 	if provider.reconnecting {

pkg/storage/redisProvider.go

@@ -89,6 +89,24 @@ func (provider *Redis) ListKeys() []string {
 	return keys
 }
 
+// MapKeys method returns the list of existing keys
+func (provider *Redis) MapKeys(prefix string) map[string]string {
+	if provider.reconnecting {
+		provider.logger.Sugar().Error("Impossible to list the redis keys while reconnecting.")
+		return map[string]string{}
+	}
+
+	m := map[string]string{}
+	keys, _ := provider.Client.Scan(provider.ctx, 0, "*", 0).Val()
+	for _, key := range keys {
+		if strings.HasPrefix(key, prefix) {
+			m[key] = string(provider.Get(key))
+		}
+	}
+
+	return m
+}
+
 // Get method returns the populated response if exists, empty response then
 func (provider *Redis) Get(key string) (item []byte) {
 	if provider.reconnecting {

pkg/storage/storage.go

@@ -48,6 +48,14 @@ func getStorageNameFromConfiguration(configuration configurationtypes.AbstractCo
 	return "badger"
 }
 
+func NewStorageFromName(name string) (StorerInstanciator, error) {
+	if newStorage, found := storageMap[name]; found {
+		return newStorage, nil
+	}
+
+	return nil, errors.New("Storer with name" + name + " not found")
+}
+
 func NewStorage(configuration configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
 	storerName := getStorageNameFromConfiguration(configuration)
 	if newStorage, found := storageMap[storerName]; found {

pkg/storage/types/types.go

@@ -9,6 +9,7 @@ import (
 )
 
 type Storer interface {
+	MapKeys(prefix string) map[string]string
 	ListKeys() []string
 	Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response
 	Get(key string) []byte

pkg/surrogate/providers/akamai_test.go

@@ -8,13 +8,17 @@ import (
 
 	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/errors"
+	"github.com/darkweak/souin/pkg/storage"
+	"github.com/darkweak/souin/tests"
 	"go.uber.org/zap"
 )
 
 func mockAkamaiProvider() *AkamaiSurrogateStorage {
+	instanciator, _ := storage.NewStorageFromName("badger")
+	storer, _ := instanciator(tests.MockConfiguration(tests.BadgerConfiguration))
 	ass := &AkamaiSurrogateStorage{
 		baseStorage: &baseStorage{
-			Storage:    &sync.Map{},
+			Storage:    storer,
 			Keys:       make(map[string]configurationtypes.SurrogateKeys),
 			keysRegexp: make(map[string]keysRegexpInner),
 			dynamic:    true,

pkg/surrogate/providers/cloudflare_test.go

@@ -9,13 +9,17 @@ import (
 
 	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/errors"
+	"github.com/darkweak/souin/pkg/storage"
+	"github.com/darkweak/souin/tests"
 	"go.uber.org/zap"
 )
 
 func mockCloudflareProvider() *CloudflareSurrogateStorage {
+	instanciator, _ := storage.NewStorageFromName("badger")
+	storer, _ := instanciator(tests.MockConfiguration(tests.BadgerConfiguration))
 	ass := &CloudflareSurrogateStorage{
 		baseStorage: &baseStorage{
-			Storage:    &sync.Map{},
+			Storage:    storer,
 			Keys:       make(map[string]configurationtypes.SurrogateKeys),
 			keysRegexp: make(map[string]keysRegexpInner),
 			dynamic:    true,