feat(chore): storages externalization refactoring

.github/workflows/non-regression.yml

@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: '1.21'
+  GO_VERSION: '1.22'
 
 jobs:
   lint-validation:
@@ -34,26 +34,9 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Run unit static tests
-        run: go test -v -race $(go list ./... | grep -v pkg/storage)
-  unit-test-golang-with-services:
-    needs: lint-validation
-    name: Unit tests with external services
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Build and run the docker stack
-        run: |
-          docker network create your_network || true
-          docker compose -f docker-compose.yml.test up -d --build --force-recreate --remove-orphans
-      - name: Run pkg storage tests
-        run: docker compose -f docker-compose.yml.test exec -T souin go test -v -race ./pkg/storage
+        run: go test -v
   validate-prod-container-building:
-    needs: unit-test-golang-with-services
+    needs: unit-test-golang
     name: Validate that the container build for prod
     runs-on: ubuntu-latest
     steps:

.github/workflows/plugins-master.yml

@@ -6,7 +6,7 @@ on:
       - master
 
 env:
-  GO_VERSION: '1.21'
+  GO_VERSION: '1.22'
 
 jobs:
   build-caddy-validator:
@@ -30,4 +30,4 @@ jobs:
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
       -
         name: Build current Souin as caddy module with referenced Souin core version when merge on master
-        run: cd plugins/caddy && xcaddy build --with github.com/${{ github.repository }}/plugins/caddy@$(git rev-parse --short "$GITHUB_SHA")
+        run: cd plugins/caddy && xcaddy build --with github.com/${{ github.repository }}/plugins/caddy@$(git rev-parse --short "$GITHUB_SHA") --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy

.github/workflows/plugins.yml

@@ -36,7 +36,7 @@ jobs:
         name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: '1.22'
       -
         name: Checkout code
         uses: actions/checkout@v4
@@ -45,7 +45,7 @@ jobs:
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
       -
         name: Build Souin as caddy module
-        run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../..
+        run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy
       -
         name: Run Caddy tests
         run: cd plugins/caddy && go test -v ./...
@@ -85,7 +85,7 @@ jobs:
         name: Install Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.22'
 
       - name: Install Node.js
         uses: actions/setup-node@v4
@@ -113,7 +113,7 @@ jobs:
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
       -
         name: Build Souin as caddy module for current commit
-        run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../..
+        run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy
       -
         name: Run detached caddy
         run: cd souin/plugins/caddy && ./caddy run --config ../../docs/cache-tests/cache-tests-caddyfile --adapter caddyfile &
@@ -158,74 +158,74 @@ jobs:
     with:
       CAPITALIZED_NAME: Beego
       LOWER_NAME: beego
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-chi-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Chi
       LOWER_NAME: chi
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-dotweb-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Dotweb
       LOWER_NAME: dotweb
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-echo-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Echo
       LOWER_NAME: echo
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-fiber-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Fiber
       LOWER_NAME: fiber
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-gin-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Gin
       LOWER_NAME: gin
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-goa-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Goa
       LOWER_NAME: goa
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-kratos-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Kratos
       LOWER_NAME: kratos
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-souin-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Souin
       LOWER_NAME: souin
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-traefik-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Traefik
       LOWER_NAME: traefik
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'
   build-webgo-validator:
     uses: ./.github/workflows/plugin_template.yml
     secrets: inherit
     with:
       CAPITALIZED_NAME: Webgo
       LOWER_NAME: webgo
-      GO_VERSION: '1.21'
+      GO_VERSION: '1.22'

.github/workflows/release.yml

@@ -5,7 +5,7 @@ on:
     tags: ["v*"]
 
 env:
-  GO_VERSION: '1.21'
+  GO_VERSION: '1.22'
 
 jobs:
   generate-souin-traefik-docker:

.github/workflows/workflow_plugins_generator.sh

@@ -3,7 +3,7 @@
 # Disabled go-zero and hertz temporary
 # plugins=("beego"  "chi"  "dotweb"  "echo"  "fiber"  "gin"  "goa"  "go-zero"  "hertz"  "kratos"  "roadrunner"  "souin"  "traefik"  "tyk"  "webgo")
 plugins=("beego"  "chi"  "dotweb"  "echo"  "fiber"  "gin"  "goa"  "kratos"  "souin"  "traefik"  "webgo")
-go_version=1.21
+go_version=1.22
 
 IFS= read -r -d '' tpl <<EOF
 name: Build and validate Souin as plugins
@@ -53,7 +53,7 @@ jobs:
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
       -
         name: Build Souin as caddy module
-        run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../..
+        run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy
       -
         name: Run Caddy tests
         run: cd plugins/caddy && go test -v ./...
@@ -93,7 +93,7 @@ jobs:
         name: Install Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.22'
 
       - name: Install Node.js
         uses: actions/setup-node@v4
@@ -121,7 +121,7 @@ jobs:
         run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
       -
         name: Build Souin as caddy module for current commit
-        run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../..
+        run: cd souin/plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../.. --with github.com/darkweak/storages/badger/caddy --with github.com/darkweak/storages/etcd/caddy --with github.com/darkweak/storages/nuts/caddy --with github.com/darkweak/storages/olric/caddy --with github.com/darkweak/storages/otter/caddy --with github.com/darkweak/storages/redis/caddy
       -
         name: Run detached caddy
         run: cd souin/plugins/caddy && ./caddy run --config ../../docs/cache-tests/cache-tests-caddyfile --adapter caddyfile &

Dockerfile-prod

@@ -21,8 +21,11 @@ ADD ./plugins /app/src/github.com/darkweak/souin/plugins
 WORKDIR /app/src/github.com/darkweak/souin
 RUN go mod download
 
+WORKDIR /app/src/github.com/darkweak/souin/plugins/souin
+RUN go mod download
+
 RUN CGO_ENABLED=0 go build -a \
-  -tags netgo -ldflags '-w -extldflags "-static"' -o /app/cmd/souin ./plugins/souin
+  -tags netgo -ldflags '-w -extldflags "-static"' -o /app/cmd/souin .
 
 EXPOSE 80
 

Makefile

@@ -62,7 +62,37 @@ build-caddy: ## Build caddy binary
 	cd plugins/caddy && \
 	go mod tidy && \
 	go mod download && \
-	XCADDY_RACE_DETECTOR=1 XCADDY_DEBUG=1 xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin=../..
+	XCADDY_RACE_DETECTOR=1 XCADDY_DEBUG=1 xcaddy build \
+		--with github.com/darkweak/souin/plugins/caddy=./ \
+		--with github.com/darkweak/souin=../.. \
+		--with github.com/darkweak/storages/badger/caddy \
+		--with github.com/darkweak/storages/etcd/caddy \
+		--with github.com/darkweak/storages/nuts/caddy \
+		--with github.com/darkweak/storages/olric/caddy \
+		--with github.com/darkweak/storages/otter/caddy \
+		--with github.com/darkweak/storages/redis/caddy
+
+build-caddy-dev: ## Build caddy binary
+	cd plugins/caddy && \
+	go mod tidy && \
+	go mod download && \
+	XCADDY_RACE_DETECTOR=1 XCADDY_DEBUG=1 xcaddy build \
+		--with github.com/darkweak/souin/plugins/caddy=./ \
+		--with github.com/darkweak/souin=../.. \
+		--with github.com/darkweak/storages/badger/caddy=../../../storages/badger/caddy \
+		--with github.com/darkweak/storages/etcd/caddy=../../../storages/etcd/caddy \
+		--with github.com/darkweak/storages/nuts/caddy=../../../storages/nuts/caddy \
+		--with github.com/darkweak/storages/olric/caddy=../../../storages/olric/caddy \
+		--with github.com/darkweak/storages/otter/caddy=../../../storages/otter/caddy \
+		--with github.com/darkweak/storages/redis/caddy=../../../storages/redis/caddy \
+		--with github.com/darkweak/storages/badger=../../../storages/badger \
+		--with github.com/darkweak/storages/etcd=../../../storages/etcd \
+		--with github.com/darkweak/storages/nuts=../../../storages/nuts \
+		--with github.com/darkweak/storages/olric=../../../storages/olric \
+		--with github.com/darkweak/storages/otter=../../../storages/otter \
+		--with github.com/darkweak/storages/redis=../../../storages/redis \
+		--with github.com/darkweak/storages/core=../../../storages/core
+	cd plugins/caddy && ./caddy run
 
 build-dev: env-dev ## Build containers with dev env vars
 	$(DC_BUILD) souin
@@ -109,9 +139,9 @@ generate-workflow: ## Generate plugin workflow
 	bash .github/workflows/workflow_plugins_generator.sh
 
 golangci-lint: ## Run golangci-lint to ensure the code quality
-	docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.55.2 golangci-lint run -v --timeout 180s ./...
+	docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.57.2 golangci-lint run -v --timeout 180s ./...
 	for plugin in $(PLUGINS_LIST) ; do \
-		echo "Starting lint $$plugin \n" && docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.55.2 golangci-lint run -v --skip-dirs=override --timeout 240s ./plugins/$$plugin; \
+		echo "Starting lint $$plugin \n" && docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.57.2 golangci-lint run -v --skip-dirs=override --timeout 240s ./plugins/$$plugin; \
 	done
 	cd plugins/caddy && go mod tidy && go mod download
 

configurationtypes/types.go

@@ -184,6 +184,10 @@ type URL struct {
 
 // CacheProvider config
 type CacheProvider struct {
+	// Uuid to identify a unique instance.
+	Uuid string
+	// Found to determine if we can use that storage.
+	Found bool `json:"found" yaml:"found"`
 	// URL to connect to the storage system.
 	URL string `json:"url" yaml:"url"`
 	// Path to the configuration file.

context/cache_test.go

@@ -6,14 +6,48 @@ import (
 	"testing"
 
 	"github.com/darkweak/souin/configurationtypes"
-	"github.com/darkweak/souin/plugins/souin/configuration"
 	"go.uber.org/zap"
 )
 
+type testConfiguration struct {
+	defaultCache *configurationtypes.DefaultCache
+	cacheKeys    configurationtypes.CacheKeys
+}
+
+func (*testConfiguration) GetUrls() map[string]configurationtypes.URL {
+	return nil
+}
+func (*testConfiguration) GetPluginName() string {
+	return ""
+}
+func (t *testConfiguration) GetDefaultCache() configurationtypes.DefaultCacheInterface {
+	return t.defaultCache
+}
+func (*testConfiguration) GetAPI() configurationtypes.API {
+	return configurationtypes.API{}
+}
+func (*testConfiguration) GetLogLevel() string {
+	return ""
+}
+func (*testConfiguration) GetLogger() *zap.Logger {
+	return zap.NewNop()
+}
+func (*testConfiguration) SetLogger(*zap.Logger) {
+}
+func (*testConfiguration) GetYkeys() map[string]configurationtypes.SurrogateKeys {
+	return nil
+}
+func (*testConfiguration) GetSurrogateKeys() map[string]configurationtypes.SurrogateKeys {
+	return nil
+}
+func (t *testConfiguration) GetCacheKeys() configurationtypes.CacheKeys {
+	return t.cacheKeys
+}
+
 func Test_CacheContext_SetupContext(t *testing.T) {
 	dc := configurationtypes.DefaultCache{}
-	c := configuration.Configuration{
-		DefaultCache: &dc,
+	c := testConfiguration{
+		defaultCache: &dc,
 	}
 	c.SetLogger(zap.NewNop())
 	ctx := cacheContext{}
@@ -23,7 +57,7 @@ func Test_CacheContext_SetupContext(t *testing.T) {
 		t.Error("The context must be equal to Souin.")
 	}
 
-	c.DefaultCache.CacheName = "Something"
+	c.defaultCache.CacheName = "Something"
 	ctx.SetupContext(&c)
 	if ctx.cacheName != "Something" {
 		t.Error("The context must be equal to Something.")

context/graphql_test.go

@@ -7,24 +7,22 @@ import (
 	"testing"
 
 	"github.com/darkweak/souin/configurationtypes"
-	"github.com/darkweak/souin/plugins/souin/configuration"
 	"go.uber.org/zap"
 )
 
 func Test_GraphQLContext_SetupContext(t *testing.T) {
 	dc := configurationtypes.DefaultCache{}
-	c := configuration.Configuration{
-		DefaultCache: &dc,
+	c := testConfiguration{
+		defaultCache: &dc,
 	}
-	c.SetLogger(zap.NewNop())
 	ctx := graphQLContext{}
 
 	ctx.SetupContext(&c)
 	if ctx.custom {
 		t.Error("The context must not be custom if no allowed HTTP verbs are set in the configuration.")
 	}
 
-	c.DefaultCache.AllowedHTTPVerbs = []string{http.MethodGet}
+	c.defaultCache.AllowedHTTPVerbs = []string{http.MethodGet}
 	ctx.SetupContext(&c)
 	if !ctx.custom {
 		t.Error("The context must be custom if at least one allowed HTTP verb is set in the configuration.")
@@ -33,8 +31,8 @@ func Test_GraphQLContext_SetupContext(t *testing.T) {
 
 func Test_GraphQLContext_SetContext(t *testing.T) {
 	dc := configurationtypes.DefaultCache{}
-	c := configuration.Configuration{
-		DefaultCache: &dc,
+	c := testConfiguration{
+		defaultCache: &dc,
 	}
 	c.SetLogger(zap.NewNop())
 	ctx := graphQLContext{custom: true}