Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(chore): surrogate-keys distributed storage #448

Merged
merged 8 commits into from
Feb 7, 2024
Merged

feat(chore): surrogate-keys distributed storage #448

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a010223
feat(chore): surrogate-keys with storer as storage
darkweak Jan 27, 2024
2c09d3b
feat(chore): surrogate-keys distributed-storage
darkweak Jan 29, 2024
9339825
fix(ci): traefik plugin
darkweak Jan 30, 2024
6699cbf
fix(ci): surrogate unit tests
darkweak Jan 30, 2024
249a8da
fix(ci): omit surrogate keys from the list keys, Caddy cypress E2E tests
darkweak Feb 2, 2024
9d98803
fix(ci): Tyk plugin
darkweak Feb 3, 2024
dcfa029
fix(chore): remove unused files
darkweak Feb 5, 2024
aa85ce4
fix(cache-tests): cc-resp-no-cache fixup
darkweak Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions pkg/api/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/api/debug"
"github.com/darkweak/souin/pkg/api/prometheus"
"github.com/darkweak/souin/pkg/storage"
"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/pkg/surrogate/providers"
)

Expand All @@ -18,7 +18,7 @@ type MapHandler struct {
// GenerateHandlerMap generate the MapHandler
func GenerateHandlerMap(
configuration configurationtypes.AbstractConfigurationInterface,
storers []storage.Storer,
storers []types.Storer,
surrogateStorage providers.SurrogateInterface,
) *MapHandler {
hm := make(map[string]http.HandlerFunc)
Expand All @@ -45,7 +45,7 @@ func GenerateHandlerMap(
}

// Initialize contains all apis that should be enabled
func Initialize(c configurationtypes.AbstractConfigurationInterface, storers []storage.Storer, surrogateStorage providers.SurrogateInterface) []EndpointInterface {
func Initialize(c configurationtypes.AbstractConfigurationInterface, storers []types.Storer, surrogateStorage providers.SurrogateInterface) []EndpointInterface {
return []EndpointInterface{initializeSouin(c, storers,
surrogateStorage), debug.InitializeDebug(c), prometheus.InitializePrometheus(c)}
}
6 changes: 3 additions & 3 deletions pkg/api/souin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ import (
"strings"

"github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/storage"
"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/pkg/surrogate/providers"
)

// SouinAPI object contains informations related to the endpoints
type SouinAPI struct {
basePath string
enabled bool
storers []storage.Storer
storers []types.Storer
surrogateStorage providers.SurrogateInterface
allowedMethods []string
}
Expand All @@ -39,7 +39,7 @@ type invalidation struct {

func initializeSouin(
configuration configurationtypes.AbstractConfigurationInterface,
storers []storage.Storer,
storers []types.Storer,
surrogateStorage providers.SurrogateInterface,
) *SouinAPI {
basePath := configuration.GetAPI().Souin.BasePath
Expand Down
5 changes: 3 additions & 2 deletions pkg/middleware/middleware.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/darkweak/souin/pkg/api/prometheus"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage"
"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/pkg/surrogate"
"github.com/darkweak/souin/pkg/surrogate/providers"
"github.com/pquerna/cachecontrol/cacheobject"
Expand Down Expand Up @@ -104,7 +105,7 @@ func NewHTTPCacheHandler(c configurationtypes.AbstractConfigurationInterface) *S

type SouinBaseHandler struct {
Configuration configurationtypes.AbstractConfigurationInterface
Storers []storage.Storer
Storers []types.Storer
InternalEndpointHandlers *api.MapHandler
ExcludeRegex *regexp.Regexp
RegexpUrls regexp.Regexp
Expand Down Expand Up @@ -259,7 +260,7 @@ func (s *SouinBaseHandler) Store(
default:
for _, storer := range s.Storers {
wg.Add(1)
go func(currentStorer storage.Storer) {
go func(currentStorer types.Storer) {
defer wg.Done()
if currentStorer.Set(cachedKey, response, currentMatchedURL, ma) == nil {
s.Configuration.GetLogger().Sugar().Debugf("Stored the key %s in the %s provider", cachedKey, currentStorer.Name())
Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/badgerProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

t "github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
badger "github.com/dgraph-io/badger/v3"
"github.com/imdario/mergo"
"go.uber.org/zap"
Expand All @@ -36,7 +37,7 @@ func (b *badgerLogger) Warningf(msg string, params ...interface{}) {
}

// BadgerConnectionFactory function create new Badger instance
func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (Storer, error) {
func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) {
dc := c.GetDefaultCache()
badgerConfiguration := dc.GetBadger()
badgerOptions := badger.DefaultOptions(badgerConfiguration.Path)
Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/badgerProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"testing"

"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/tests"

"time"
Expand All @@ -12,13 +13,13 @@ import (
"github.com/darkweak/souin/errors"
)

func getBadgerClientAndMatchedURL(key string) (Storer, configurationtypes.URL) {
func getBadgerClientAndMatchedURL(key string) (types.Storer, configurationtypes.URL) {
return GetCacheProviderClientAndMatchedURL(
key,
func() configurationtypes.AbstractConfigurationInterface {
return tests.MockConfiguration(tests.BaseConfiguration)
},
func(config configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
provider, _ := BadgerConnectionFactory(config)
_ = provider.Init()

Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/embeddedOlricProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/buraksezer/olric/config"
t "github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
"github.com/google/uuid"
"go.uber.org/zap"
yaml "gopkg.in/yaml.v3"
Expand Down Expand Up @@ -60,7 +61,7 @@ func tryToLoadConfiguration(olricInstance *config.Config, olricConfiguration t.C
}

// EmbeddedOlricConnectionFactory function create new EmbeddedOlric instance
func EmbeddedOlricConnectionFactory(configuration t.AbstractConfigurationInterface) (Storer, error) {
func EmbeddedOlricConnectionFactory(configuration t.AbstractConfigurationInterface) (types.Storer, error) {
var olricInstance *config.Config
loaded := false

Expand Down
4 changes: 2 additions & 2 deletions pkg/storage/embeddedOlricProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package storage

/*
func mockEmbeddedConfiguration(c func() string, key string) (Storer, configurationtypes.URL) {
func mockEmbeddedConfiguration(c func() string, key string) (types.Storer, configurationtypes.URL) {
return GetCacheProviderClientAndMatchedURL(
key,
func() configurationtypes.AbstractConfigurationInterface {
return tests.MockConfiguration(c)
},
func(config configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
provider, _ := EmbeddedOlricConnectionFactory(config)

return provider, nil
Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/etcdProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

t "github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"google.golang.org/grpc/connectivity"
Expand All @@ -28,7 +29,7 @@ type Etcd struct {
}

// EtcdConnectionFactory function create new Etcd instance
func EtcdConnectionFactory(c t.AbstractConfigurationInterface) (Storer, error) {
func EtcdConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) {
dc := c.GetDefaultCache()
bc, _ := json.Marshal(dc.GetEtcd().Configuration)
etcdConfiguration := clientv3.Config{
Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/etcdProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"testing"

"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/tests"

"time"
Expand All @@ -12,13 +13,13 @@ import (
"github.com/darkweak/souin/errors"
)

func getEtcdClientAndMatchedURL(key string) (Storer, configurationtypes.URL) {
func getEtcdClientAndMatchedURL(key string) (types.Storer, configurationtypes.URL) {
return GetCacheProviderClientAndMatchedURL(
key,
func() configurationtypes.AbstractConfigurationInterface {
return tests.MockConfiguration(tests.EtcdConfiguration)
},
func(config configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
provider, _ := EtcdConnectionFactory(config)
_ = provider.Init()

Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/nutsProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

t "github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
"github.com/imdario/mergo"
"github.com/nutsdb/nutsdb"
"go.uber.org/zap"
Expand Down Expand Up @@ -67,7 +68,7 @@ func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
}

// NutsConnectionFactory function create new Nuts instance
func NutsConnectionFactory(c t.AbstractConfigurationInterface) (Storer, error) {
func NutsConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) {
dc := c.GetDefaultCache()
nutsConfiguration := dc.GetNuts()
nutsOptions := nutsdb.DefaultOptions
Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/nutsProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"testing"

"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/tests"

"time"
Expand All @@ -12,13 +13,13 @@ import (
"github.com/darkweak/souin/errors"
)

func getNutsClientAndMatchedURL(key string) (Storer, configurationtypes.URL) {
func getNutsClientAndMatchedURL(key string) (types.Storer, configurationtypes.URL) {
return GetCacheProviderClientAndMatchedURL(
key,
func() configurationtypes.AbstractConfigurationInterface {
return tests.MockConfiguration(tests.BaseConfiguration)
},
func(config configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
provider, _ := NutsConnectionFactory(config)
_ = provider.Init()

Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/olricProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/buraksezer/olric/config"
t "github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
"go.uber.org/zap"
)

Expand All @@ -29,7 +30,7 @@ type Olric struct {
}

// OlricConnectionFactory function create new Olric instance
func OlricConnectionFactory(configuration t.AbstractConfigurationInterface) (Storer, error) {
func OlricConnectionFactory(configuration t.AbstractConfigurationInterface) (types.Storer, error) {
c, err := olric.NewClusterClient([]string{configuration.GetDefaultCache().GetOlric().URL})
if err != nil {
configuration.GetLogger().Sugar().Errorf("Impossible to connect to Olric, %v", err)
Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/olricProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,20 @@ import (
"testing"
"time"

"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/tests"

"github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/errors"
)

func getOlricClientAndMatchedURL(key string) (Storer, configurationtypes.URL) {
func getOlricClientAndMatchedURL(key string) (types.Storer, configurationtypes.URL) {
return GetCacheProviderClientAndMatchedURL(
key,
func() configurationtypes.AbstractConfigurationInterface {
return tests.MockConfiguration(tests.OlricConfiguration)
},
func(config configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
provider, _ := OlricConnectionFactory(config)
_ = provider.Init()

Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/redisProvider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

t "github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
redis "github.com/redis/rueidis"
"github.com/redis/rueidis/rueidiscompat"
"go.uber.org/zap"
Expand All @@ -31,7 +32,7 @@ type Redis struct {
}

// RedisConnectionFactory function create new Nuts instance
func RedisConnectionFactory(c t.AbstractConfigurationInterface) (Storer, error) {
func RedisConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) {
dc := c.GetDefaultCache()
bc, _ := json.Marshal(dc.GetRedis().Configuration)

Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/redisProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"testing"

"github.com/darkweak/souin/pkg/storage/types"
"github.com/darkweak/souin/tests"

"time"
Expand All @@ -12,13 +13,13 @@ import (
"github.com/darkweak/souin/errors"
)

func getRedisClientAndMatchedURL(key string) (Storer, configurationtypes.URL) {
func getRedisClientAndMatchedURL(key string) (types.Storer, configurationtypes.URL) {
return GetCacheProviderClientAndMatchedURL(
key,
func() configurationtypes.AbstractConfigurationInterface {
return tests.MockConfiguration(tests.RedisConfiguration)
},
func(config configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
provider, _ := RedisConnectionFactory(config)
_ = provider.Init()

Expand Down
22 changes: 5 additions & 17 deletions pkg/storage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ import (
"net/http"
"net/url"
"strings"
"time"

"github.com/darkweak/souin/configurationtypes"
"github.com/darkweak/souin/pkg/rfc"
"github.com/darkweak/souin/pkg/storage/types"
)

const (
Expand All @@ -17,19 +17,7 @@ const (
StalePrefix = "STALE_"
)

type Storer interface {
ListKeys() []string
Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response
Get(key string) []byte
Set(key string, value []byte, url configurationtypes.URL, duration time.Duration) error
Delete(key string)
DeleteMany(key string)
Init() error
Name() string
Reset() error
}

type StorerInstanciator func(configurationtypes.AbstractConfigurationInterface) (Storer, error)
type StorerInstanciator func(configurationtypes.AbstractConfigurationInterface) (types.Storer, error)

var storageMap = map[string]StorerInstanciator{
"etcd": EtcdConnectionFactory,
Expand Down Expand Up @@ -60,7 +48,7 @@ func getStorageNameFromConfiguration(configuration configurationtypes.AbstractCo
return "badger"
}

func NewStorage(configuration configurationtypes.AbstractConfigurationInterface) (Storer, error) {
func NewStorage(configuration configurationtypes.AbstractConfigurationInterface) (types.Storer, error) {
storerName := getStorageNameFromConfiguration(configuration)
if newStorage, found := storageMap[storerName]; found {
return newStorage(configuration)
Expand All @@ -82,8 +70,8 @@ func uniqueStorers(storers []string) []string {
return s
}

func NewStorages(configuration configurationtypes.AbstractConfigurationInterface) ([]Storer, error) {
storers := []Storer{}
func NewStorages(configuration configurationtypes.AbstractConfigurationInterface) ([]types.Storer, error) {
storers := []types.Storer{}
for _, storerName := range uniqueStorers(configuration.GetDefaultCache().GetStorers()) {
if newStorage, found := storageMap[storerName]; found {
instance, err := newStorage(configuration)
Expand Down
Loading
Loading