Skip to content

Commit

Permalink
feat(chore): retrieve from uuid instead of name to have multiple stor…
Browse files Browse the repository at this point in the history
…age instances concurrently
  • Loading branch information
darkweak committed Jul 7, 2024
1 parent d363dd2 commit 158d10c
Show file tree
Hide file tree
Showing 49 changed files with 272 additions and 116 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/non-regression.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
- name: Run unit static tests
run: go test -v
validate-prod-container-building:
needs: unit-test-golang-with-services
needs: unit-test-golang
name: Validate that the container build for prod
runs-on: ubuntu-latest
steps:
Expand Down
22 changes: 22 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,28 @@ build-caddy: ## Build caddy binary
--with github.com/darkweak/storages/otter/caddy \
--with github.com/darkweak/storages/redis/caddy

build-caddy-dev: ## Build caddy binary
cd plugins/caddy && \
go mod tidy && \
go mod download && \
XCADDY_RACE_DETECTOR=1 XCADDY_DEBUG=1 xcaddy build \
--with github.com/darkweak/souin/plugins/caddy=./ \
--with github.com/darkweak/souin=../.. \
--with github.com/darkweak/storages/badger/caddy=../../../storages/badger/caddy \
--with github.com/darkweak/storages/etcd/caddy=../../../storages/etcd/caddy \
--with github.com/darkweak/storages/nuts/caddy=../../../storages/nuts/caddy \
--with github.com/darkweak/storages/olric/caddy=../../../storages/olric/caddy \
--with github.com/darkweak/storages/otter/caddy=../../../storages/otter/caddy \
--with github.com/darkweak/storages/redis/caddy=../../../storages/redis/caddy \
--with github.com/darkweak/storages/badger=../../../storages/badger \
--with github.com/darkweak/storages/etcd=../../../storages/etcd \
--with github.com/darkweak/storages/nuts=../../../storages/nuts \
--with github.com/darkweak/storages/olric=../../../storages/olric \
--with github.com/darkweak/storages/otter=../../../storages/otter \
--with github.com/darkweak/storages/redis=../../../storages/redis \
--with github.com/darkweak/storages/core=../../../storages/core
cd plugins/caddy && ./caddy run

build-dev: env-dev ## Build containers with dev env vars
$(DC_BUILD) souin
$(MAKE) up
Expand Down
2 changes: 2 additions & 0 deletions configurationtypes/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,8 @@ type URL struct {

// CacheProvider config
type CacheProvider struct {
// Uuid to identify a unique instance.
Uuid string
// Found to determine if we can use that storage.
Found bool `json:"found" yaml:"found"`
// URL to connect to the storage system.
Expand Down
7 changes: 7 additions & 0 deletions docker-compose.yml.test
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ services:
command: redis-server
<<: *networks

redis-insight:
image: redis/redisinsight:latest
volumes:
- ./redis-data:/data
ports:
- 5540:5540

traefik:
image: traefik:latest
command: --providers.docker
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.22.1
require (
github.com/caddyserver/caddy/v2 v2.8.4
github.com/cespare/xxhash/v2 v2.2.0
github.com/darkweak/storages/core v0.0.3
github.com/darkweak/storages/core v0.0.4
github.com/google/uuid v1.6.0
github.com/pierrec/lz4/v4 v4.1.21
github.com/pquerna/cachecontrol v0.2.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns=
github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654=
github.com/darkweak/storages/core v0.0.3 h1:kTO7CZgVjWCA9HM0uiip4qe/NqEIBD36hHxorh+0dPM=
github.com/darkweak/storages/core v0.0.3/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc=
github.com/darkweak/storages/core v0.0.4 h1:rpFTiN+CHwdlXThS28mRpvsSfuMVqWcOe/xtC5A2AQg=
github.com/darkweak/storages/core v0.0.4/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down
23 changes: 15 additions & 8 deletions pkg/middleware/middleware.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,20 +64,27 @@ func NewHTTPCacheHandler(c configurationtypes.AbstractConfigurationInterface) *S
}

storedStorers := core.GetRegisteredStorers()
storers := make([]types.Storer, len(storedStorers))
if len(storedStorers) == 0 {
storers := []types.Storer{}
if len(storedStorers) != 0 {
dc := c.GetDefaultCache()
for _, s := range []string{dc.GetBadger().Uuid, dc.GetEtcd().Uuid, dc.GetNuts().Uuid, dc.GetOlric().Uuid, dc.GetOtter().Uuid, dc.GetRedis().Uuid} {
if s != "" {
if st := core.GetRegisteredStorer(s); st != nil {
storers = append(storers, st.(types.Storer))
}
}
}
}
if len(storers) == 0 {
c.GetLogger().Warn("You're running Souin with the default storage that is not optimized and for development purpose. We recommend to use at least one of the storages from https://github.com/darkweak/storages")
memoryStorer, _ := storage.Factory(c)
core.RegisterStorage(memoryStorer)
storers = append(storers, memoryStorer)
} else {
for id, storer := range storedStorers {
storers[id] = storer.(types.Storer)
}
}
c.GetLogger().Debug("Storer initialized.")

c.GetLogger().Sugar().Debugf("Storer initialized: %#v.", storers)
regexpUrls := helpers.InitializeRegexp(c)
surrogateStorage := surrogate.InitializeSurrogate(c, storers[0].Name())
surrogateStorage := surrogate.InitializeSurrogate(c, fmt.Sprintf("%s-%s", storers[0].Name(), storers[0].Uuid()))
c.GetLogger().Debug("Surrogate storage initialized.")
var excludedRegexp *regexp.Regexp = nil
if c.GetDefaultCache().GetRegex().Exclude != "" {
Expand Down
5 changes: 5 additions & 0 deletions pkg/storage/defaultProvider.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ func (provider *Default) Name() string {
return types.DefaultStorageName
}

// Uuid returns an unique identifier
func (provider *Default) Uuid() string {
return ""
}

// MapKeys method returns a map with the key and value
func (provider *Default) MapKeys(prefix string) map[string]string {
keys := map[string]string{}
Expand Down
1 change: 1 addition & 0 deletions pkg/storage/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type Storer interface {
DeleteMany(key string)
Init() error
Name() string
Uuid() string
Reset() error

// Multi level storer to handle fresh/stale at once
Expand Down
19 changes: 10 additions & 9 deletions pkg/surrogate/providers/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,13 @@ const (
)

var storageToInfiniteTTLMap = map[string]time.Duration{
"BADGER": 365 * 24 * time.Hour,
"ETCD": 365 * 24 * time.Hour,
"NUTS": 0,
"OLRIC": 365 * 24 * time.Hour,
"OTTER": 365 * 24 * time.Hour,
"REDIS": -1,
"BADGER": 365 * 24 * time.Hour,
"ETCD": 365 * 24 * time.Hour,
"NUTS": 0,
"OLRIC": 365 * 24 * time.Hour,
"OTTER": 365 * 24 * time.Hour,
"REDIS": -1,
types.DefaultStorageName: -1,
}

func (s *baseStorage) ParseHeaders(value string) []string {
Expand Down Expand Up @@ -103,15 +104,15 @@ func (s *baseStorage) init(config configurationtypes.AbstractConfigurationInterf
if configuration, ok := config.GetSurrogateKeys()["_configuration"]; ok {
storer := core.GetRegisteredStorer(configuration.SurrogateConfiguration.Storer)
if storer == nil {
storer = core.GetRegisteredStorer(types.DefaultStorageName)
storer = core.GetRegisteredStorer(types.DefaultStorageName + "-")
if storer == nil {
config.GetLogger().Sugar().Errorf("Impossible to retrieve the storers %s, nuts neither for the surrogate-keys", configuration.SurrogateConfiguration.Storer)
config.GetLogger().Sugar().Errorf("Impossible to retrieve the storers %s for the surrogate-keys from it's configuration", configuration.SurrogateConfiguration.Storer)
}
}

s.Storage = storer
} else {
storer := core.GetRegisteredStorer(strings.ToUpper(defaultStorerName))
storer := core.GetRegisteredStorer(defaultStorerName)
if storer == nil {
config.GetLogger().Sugar().Errorf("Impossible to retrieve the storers %s for the surrogate-keys", defaultStorerName)
}
Expand Down
2 changes: 1 addition & 1 deletion plugins/beego/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/darkweak/go-esi v0.0.5 // indirect
github.com/darkweak/storages/badger v0.0.3 // indirect
github.com/darkweak/storages/core v0.0.3 // indirect
github.com/darkweak/storages/core v0.0.4 // indirect
github.com/darkweak/storages/etcd v0.0.3 // indirect
github.com/darkweak/storages/nuts v0.0.3 // indirect
github.com/darkweak/storages/olric v0.0.3 // indirect
Expand Down
4 changes: 2 additions & 2 deletions plugins/beego/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns
github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654=
github.com/darkweak/storages/badger v0.0.3 h1:AHphbQLCdcy+THYLpcjUAk6pKZjmXiQEyusfeCiT+BA=
github.com/darkweak/storages/badger v0.0.3/go.mod h1:eBSVGwo5TucdDg1qPlyKlJ2Wqga/GA2IQ78A1Nbkffo=
github.com/darkweak/storages/core v0.0.3 h1:kTO7CZgVjWCA9HM0uiip4qe/NqEIBD36hHxorh+0dPM=
github.com/darkweak/storages/core v0.0.3/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc=
github.com/darkweak/storages/core v0.0.4 h1:rpFTiN+CHwdlXThS28mRpvsSfuMVqWcOe/xtC5A2AQg=
github.com/darkweak/storages/core v0.0.4/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc=
github.com/darkweak/storages/etcd v0.0.3 h1:/Nfxl2eULXQpFDME9C6wQPBIRRUb1Su2St8kboiJ0Vs=
github.com/darkweak/storages/etcd v0.0.3/go.mod h1:A8ZclVAGUwMMvO+jey8clugULbpMsYmIkLyohVovodE=
github.com/darkweak/storages/nuts v0.0.3 h1:rAupsGwKskQzCOqmgnvX3UjWPz7IR3HXyOiu94/xqlU=
Expand Down
139 changes: 139 additions & 0 deletions plugins/caddy/dispatch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package httpcache

import (
"fmt"
"strconv"

"github.com/caddyserver/caddy/v2"
)

func (s *SouinCaddyMiddleware) parseStorages(ctx caddy.Context) {
if s.Configuration.DefaultCache.Badger.Found {
e := dispatchStorage(ctx, "badger", s.Configuration.DefaultCache.Badger, s.Configuration.DefaultCache.GetStale())
if e != nil {
s.logger.Sugar().Errorf("Error during Badger init, did you include the Badger storage (--with github.com/darkweak/storages/badger/caddy)? %v", e)
} else {
badger := s.Configuration.DefaultCache.Badger
dir := ""
vdir := ""
if c := badger.Configuration; c != nil {
p, ok := c.(map[string]interface{})
if ok {
if d, ok := p["Dir"]; ok {
dir = fmt.Sprint(d)
vdir = fmt.Sprint(d)
}
if d, ok := p["ValueDir"]; ok {
vdir = fmt.Sprint(d)
}
}
}
s.Configuration.DefaultCache.Badger.Uuid = fmt.Sprintf(
"BADGER-%s-%s-%s",
dir,
vdir,
s.Configuration.DefaultCache.GetStale(),
)
}
}
if s.Configuration.DefaultCache.Etcd.Found {
e := dispatchStorage(ctx, "etcd", s.Configuration.DefaultCache.Etcd, s.Configuration.DefaultCache.GetStale())
if e != nil {
s.logger.Sugar().Errorf("Error during Etcd init, did you include the Etcd storage (--with github.com/darkweak/storages/etcd/caddy)? %v", e)
} else {
etcd := s.Configuration.DefaultCache.Etcd
endpoints := etcd.URL
username := ""
password := ""
if c := etcd.Configuration; c != nil {
p, ok := c.(map[string]interface{})
if ok {
if d, ok := p["Endpoints"]; ok {
endpoints = fmt.Sprint(d)
}
if d, ok := p["Username"]; ok {
username = fmt.Sprint(d)
}
if d, ok := p["Password"]; ok {
password = fmt.Sprint(d)
}
}
}
s.Configuration.DefaultCache.Etcd.Uuid = fmt.Sprintf(
"ETCD-%s-%s-%s-%s",
endpoints,
username,
password,
s.Configuration.DefaultCache.GetStale(),
)
}
}
if s.Configuration.DefaultCache.Nuts.Found {
e := dispatchStorage(ctx, "nuts", s.Configuration.DefaultCache.Nuts, s.Configuration.DefaultCache.GetStale())
if e != nil {
s.logger.Sugar().Errorf("Error during Nuts init, did you include the Nuts storage (--with github.com/darkweak/storages/nuts/caddy)? %v", e)
} else {
nuts := s.Configuration.DefaultCache.Nuts
dir := "/tmp/souin-nuts"
if c := nuts.Configuration; c != nil {
p, ok := c.(map[string]interface{})
if ok {
if d, ok := p["Dir"]; ok {
dir = fmt.Sprint(d)
}
}
} else if nuts.Path != "" {
dir = nuts.Path
}
s.Configuration.DefaultCache.Nuts.Uuid = fmt.Sprintf("NUTS-%s-%s", dir, s.Configuration.DefaultCache.GetStale())
}
}
if s.Configuration.DefaultCache.Olric.Found {
e := dispatchStorage(ctx, "olric", s.Configuration.DefaultCache.Olric, s.Configuration.DefaultCache.GetStale())
if e != nil {
s.logger.Sugar().Errorf("Error during Olric init, did you include the Olric storage (--with github.com/darkweak/storages/olric/caddy)? %v", e)
} else {
s.Configuration.DefaultCache.Nuts.Uuid = fmt.Sprintf("OLRIC-%s-%s", s.Configuration.DefaultCache.Olric.URL, s.Configuration.DefaultCache.GetStale())
}
}
if s.Configuration.DefaultCache.Otter.Found {
e := dispatchStorage(ctx, "otter", s.Configuration.DefaultCache.Otter, s.Configuration.DefaultCache.GetStale())
if e != nil {
s.logger.Sugar().Errorf("Error during Otter init, did you include the Otter storage (--with github.com/darkweak/storages/otter/caddy)? %v", e)
} else {
s.Configuration.DefaultCache.Otter.Uuid = fmt.Sprintf("OTTER-%s", s.Configuration.DefaultCache.GetStale())
}
}
if s.Configuration.DefaultCache.Redis.Found {
e := dispatchStorage(ctx, "redis", s.Configuration.DefaultCache.Redis, s.Configuration.DefaultCache.GetStale())
if e != nil {
s.logger.Sugar().Errorf("Error during Redis init, did you include the Redis storage (--with github.com/darkweak/storages/redis/caddy)? %v", e)
} else {
redis := s.Configuration.DefaultCache.Redis
address := redis.URL
dbname := 0
cname := "souin-redis"
if c := redis.Configuration; c != nil {
p, ok := c.(map[string]interface{})
if ok {
if d, ok := p["ClientName"]; ok {
cname = fmt.Sprint(d)
}
if d, ok := p["InitAddress"]; ok {
address = fmt.Sprint(d)
}
if d, ok := p["SelectDB"]; ok {
dbname, _ = strconv.Atoi(fmt.Sprint(d))
}
}
}
s.Configuration.DefaultCache.Redis.Uuid = fmt.Sprintf(
"REDIS-%s-%d-%s-%s",
address,
dbname,
cname,
s.Configuration.DefaultCache.GetStale(),
)
}
}
}
7 changes: 5 additions & 2 deletions plugins/caddy/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.22.1
require (
github.com/caddyserver/caddy/v2 v2.8.4
github.com/darkweak/souin v1.6.49
github.com/darkweak/storages/core v0.0.3
github.com/darkweak/storages/core v0.0.4
go.uber.org/zap v1.27.0
)

Expand Down Expand Up @@ -153,4 +153,7 @@ require (
howett.net/plist v1.0.0 // indirect
)

replace github.com/darkweak/souin v1.6.49 => ../..
replace (
github.com/darkweak/souin v1.6.49 => ../..
github.com/darkweak/storages => ../../../storages
)
4 changes: 2 additions & 2 deletions plugins/caddy/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/darkweak/go-esi v0.0.5 h1:b9LHI8Tz46R+i6p8avKPHAIBRQUCZDebNmKm5w/Zrns=
github.com/darkweak/go-esi v0.0.5/go.mod h1:koCJqwum1u6mslyZuq/Phm6hfG1K3ZK5Y7jrUBTH654=
github.com/darkweak/storages/core v0.0.3 h1:kTO7CZgVjWCA9HM0uiip4qe/NqEIBD36hHxorh+0dPM=
github.com/darkweak/storages/core v0.0.3/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc=
github.com/darkweak/storages/core v0.0.4 h1:rpFTiN+CHwdlXThS28mRpvsSfuMVqWcOe/xtC5A2AQg=
github.com/darkweak/storages/core v0.0.4/go.mod h1:wLp1cOAB4WUd46BBOtV4Lwot4GD+8fZbtIw6QM7fYuc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down
Loading

0 comments on commit 158d10c

Please sign in to comment.