Fix caddy minor bugfixes (#86)

* Fix: Caddy integration using cache instead of souin_cache

* Feat/core/ykeys/introduction (#82)

* Introduce YKeys inside Souin

* Compliant with the existing SetCache method

* Delete from Ykeys on cache deletion

* Fix tests

* Bump version

* Fix tests

* Update Makefile

* Update documentation

.gitignore

@@ -8,3 +8,4 @@ Dockerfile
 traefik.json
 *.iml
 plugins/caddy/caddy
+plugins/caddy/.github/*

Makefile

@@ -1,16 +1,9 @@
-.PHONY: build-app build-dev coverage create-network down env-dev env-prod gatling generate-plantUML help lint log tests up validate
+.PHONY: build-and-run-caddy build-and-run-caddy-json build-app build-caddy build-dev coverage create-network down env-dev env-prod gatling generate-plantUML health-check-prod help lint log tests up validate
 
 DC=docker-compose
 DC_BUILD=$(DC) build
 DC_EXEC=$(DC) exec
 
-build-app: env-prod ## Build containers with prod env vars
-	$(DC_BUILD) souin
-	$(MAKE) up
-
-build-caddy: ## Build caddy binary
-	cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin@latest=../..
-
 build-and-run-caddy: ## Run caddy binary with the Caddyfile configuration
 	$(MAKE) build-caddy
 	cd plugins/caddy && ./caddy run
@@ -19,12 +12,16 @@ build-and-run-caddy-json:  ## Run caddy binary with the json configuration
 	$(MAKE) build-caddy
 	cd plugins/caddy && ./caddy run --config ./configuration.json
 
-build-dev: env-dev ## Build containers with dev env vars
+build-app: env-prod ## Build containers with prod env vars
 	$(DC_BUILD) souin
 	$(MAKE) up
 
-health-check-prod: build-app ## Production container health check
-	$(DC_EXEC) souin ls
+build-caddy: ## Build caddy binary
+	cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin@latest=../..
+
+build-dev: env-dev ## Build containers with dev env vars
+	$(DC_BUILD) souin
+	$(MAKE) up
 
 coverage: ## Show code coverage
 	$(DC_EXEC) souin go test ./... -coverprofile cover.out
@@ -50,6 +47,9 @@ gatling: ## Launch gatling scenarios
 generate-plantUML: ## Generate plantUML diagrams
 	cd ./docs/plantUML && sh generate.sh && cd ../..
 
+health-check-prod: build-app ## Production container health check
+	$(DC_EXEC) souin ls
+
 help:
 	@grep -E '(^[0-9a-zA-Z_-]+:.*?##.*$$)|(^##)' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[32m%-25s\033[0m %s\n", $$1, $$2}' | sed -e 's/\[32m##/[33m/'
 

README.md

@@ -21,6 +21,7 @@
   8.1. [Caddy module](#caddy-module)  
   8.2. [Træfik plugin](#træfik-plugin)  
   8.3. [Prestashop plugin](#prestashop-plugin)  
+  8.3. [Wordpress plugin](#wordpress-plugin)  
 9. [Credits](#credits)
 
 [![Travis CI](https://travis-ci.com/Darkweak/Souin.svg?branch=master)](https://travis-ci.com/Darkweak/Souin)
@@ -30,8 +31,8 @@
 ## Project description
 Souin is a new cache system suitable for every reverse-proxy. It will be placed on top of your current reverse-proxy whether it's Apache, Nginx or Traefik.  
 Since it's written in go, it can be deployed on any server and thanks to the docker integration, it will be easy to install on top of a Swarm, or a kubernetes instance.  
-It's RFC compatible, supporting Vary, request coalescing and other specifications related to the [RFC-7234](https://tools.ietf.org/html/rfc7234)  
-It also supports the [Cache-Status HTTP response header](https://httpwg.org/http-extensions/draft-ietf-httpbis-cache-header.html)
+It's RFC compatible, supporting Vary, request coalescing and other specifications related to the [RFC-7234](https://tools.ietf.org/html/rfc7234).  
+It also supports the [Cache-Status HTTP response header](https://httpwg.org/http-extensions/draft-ietf-httpbis-cache-header.html) and the YKey group such as Varnish.
 
 ## Disclaimer
 If you need redis or other custom cache providers, you have to use the fully-featured version. You can read the documentation, on [the fully-featured branch](https://github.com/Darkweak/Souin/tree/full-version) to understand the specific parts.
@@ -94,22 +95,33 @@ urls:
     headers: # Override default headers
     - Authorization
     - 'Content-Type'
+ykeys:
+  The_First_Test:
+    headers:
+      Content-Type: '.+'
+  The_Second_Test:
+    url: 'the/second/.+'
+  The_Third_Test:
+  The_Fourth_Test:
 ```
 
-|  Key                               |  Description                                                  |  Value example                                                                |
-|:----------------------------------:|:-------------------------------------------------------------:|:-----------------------------------------------------------------------------:|
-| `api.basepath`                     | BasePath for all APIs to avoid conflicts                      | `/your-non-conflicting-route`<br/><br/>`(default: /souin-api)`                |
-| `api.{api}.enable`                 | Enable the new API with related routes                        | `true`<br/><br/>`(default: false)`                                            |
-| `api.security.secret`              | JWT secret key                                                | `Any_charCanW0rk123`                                                          |
-| `api.security.users`               | Array of authorized users with username x password combo      | `- username: admin`<br/><br/>`  password: admin`                              |
-| `api.souin.security`               | Enable JWT validation to access the resource                  | `true`<br/><br/>`(default: false)`                                            |
-| `default_cache.headers`            | List of headers to include to the cache                       | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header`  |
-| `default_cache.regex.exclude`      | The regex used to prevent paths being cached                  | `^[A-z]+.*$`                                                                  |
-| `log_level`                        | The log level                                                 | `One of DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL it's case insensitive` |
-| `ssl_providers`                    | List of your providers handling certificates                  | `- traefik`<br/><br/>`- nginx`<br/><br/>`- apache`                            |
-| `urls.{your url or regex}`         | List of your custom configuration depending each URL or regex | 'https:\/\/yourdomain.com'                                                    |
-| `urls.{your url or regex}.ttl`     | Override the default TTL if defined                           | 99999                                                                         |
-| `urls.{your url or regex}.headers` | Override the default headers if defined                       | `- Authorization`<br/><br/>`- 'Content-Type'`                                 |
+|  Key                                     |  Description                                                                      |  Value example                                                                |
+|:----------------------------------------:|:---------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------:|
+| `api.basepath`                           | BasePath for all APIs to avoid conflicts                                          | `/your-non-conflicting-route`<br/><br/>`(default: /souin-api)`                |
+| `api.{api}.enable`                       | Enable the new API with related routes                                            | `true`<br/><br/>`(default: false)`                                            |
+| `api.security.secret`                    | JWT secret key                                                                    | `Any_charCanW0rk123`                                                          |
+| `api.security.users`                     | Array of authorized users with username x password combo                          | `- username: admin`<br/><br/>`  password: admin`                              |
+| `api.souin.security`                     | Enable JWT validation to access the resource                                      | `true`<br/><br/>`(default: false)`                                            |
+| `default_cache.headers`                  | List of headers to include to the cache                                           | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header`  |
+| `default_cache.regex.exclude`            | The regex used to prevent paths being cached                                      | `^[A-z]+.*$`                                                                  |
+| `log_level`                              | The log level                                                                     | `One of DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL it's case insensitive` |
+| `ssl_providers`                          | List of your providers handling certificates                                      | `- traefik`<br/><br/>`- nginx`<br/><br/>`- apache`                            |
+| `urls.{your url or regex}`               | List of your custom configuration depending each URL or regex                     | 'https:\/\/yourdomain.com'                                                    |
+| `urls.{your url or regex}.ttl`           | Override the default TTL if defined                                               | 99999                                                                         |
+| `urls.{your url or regex}.headers`       | Override the default headers if defined                                           | `- Authorization`<br/><br/>`- 'Content-Type'`                                 |
+| `ykeys.{key name}.headers`               | Headers that should match to be part of the ykey group                            | `Authorization: ey.+`<br/><br/>`Content-Type: json`                           |
+| `ykeys.{key name}.headers.{header name}` | Header name that should be present a match the regex to be part of the ykey group | `Content-Type: json`                                                          |
+| `ykeys.{key name}.url`                   | Url that should match to be part of the ykey group                                | `.+`                                                                          |
 
 ## APIs
 All endpoints are accessible through the `api.basepath` configuration line or by default through `/souin-api` to avoid named route conflicts. Be sure to define an unused route to not break your existing application.
@@ -122,6 +134,7 @@ The base path for the souin API is `/souin`.
 |:-------:|:-----------------:|:-----------------------------------------------------------------------------------------|
 | `GET`   | `/`               | List stored keys cache                                                                   |
 | `PURGE` | `/{id or regexp}` | Purge selected item(s) depending. The parameter can be either a specific key or a regexp |
+| `PURGE` | `?ykey={key}`     | Purge selected item(s) corresponding to the target ykey such as Varnish                  |
 
 ### Security API
 Security API allows users to protect other APIs with JWT authentication.  
@@ -151,7 +164,6 @@ Supported providers
 ### Cache invalidation
 The cache invalidation is built for CRUD requests, if you're doing a GET HTTP request, it will serve the cached response when it exists, otherwise the reverse-proxy response will be served.  
 If you're doing a POST, PUT, PATCH or DELETE HTTP request, the related cache GET request, and the list endpoint will be dropped.  
-It works very well with plain [API Platform](https://api-platform.com) integration (except for custom actions at the moment) and CRUD routes.
 It also supports invalidation via [Souin API](#souin-api) to invalidate the cache programmatically.
 
 ## Examples
@@ -249,7 +261,10 @@ Alternatively, you can go to [the xcaddy builder website](https://xcaddy.tech) t
 Currenly not available because Træfik uses Yaegi to analyse the plugin, which prevents the usage of unsafe libraries unless you're a developper. An example can be found [here](https://github.com/darkweak/souin/tree/master/plugins/traefik) nonetheless.
 
 ### Prestashop plugin
-A repository called [prestashop-souin](https://github.com/lucmichalski/prestashop-souin) has been started by [lucmichalski](https://github.com/lucmichalski). Any help will be appreciated to make it working as soon as possible.
+A repository called [prestashop-souin](https://github.com/lucmichalski/prestashop-souin) has been started by [lucmichalski](https://github.com/lucmichalski). You can manage your Souin instance through the admin panel UI.
+
+### Wordpress plugin
+A repository called [wordpress-souin](https://github.com/Darkweak/wordpress-souin) to be able to manage your Souin instance through the admin panel UI.
 
 
 ## Credits
@@ -260,3 +275,4 @@ Thanks to these users for contributing or helping this project in any way
 * [Sata51](https://github.com/sata51)
 * [Pierre Diancourt](https://github.com/pierrediancourt)
 * [Burak Sezer](https://github.com/buraksezer)
+* [lucmichalski](https://github.com/lucmichalski)

api/main.go

@@ -3,11 +3,12 @@ package api
 import (
 	"github.com/darkweak/souin/api/auth"
 	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/cache/ykeys"
 	"github.com/darkweak/souin/configurationtypes"
 )
 
 // Initialize contains all apis that should be enabled
-func Initialize(provider types.AbstractProviderInterface, c configurationtypes.AbstractConfigurationInterface) []EndpointInterface {
+func Initialize(provider types.AbstractProviderInterface, c configurationtypes.AbstractConfigurationInterface, ykeyStorage *ykeys.YKeyStorage) []EndpointInterface {
 	security := auth.InitializeSecurity(c)
-	return []EndpointInterface{security, initializeSouin(provider, c, security)}
+	return []EndpointInterface{security, initializeSouin(provider, c, security, ykeyStorage)}
 }

api/main_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"github.com/darkweak/souin/cache/providers"
+	"github.com/darkweak/souin/cache/ykeys"
 	"github.com/darkweak/souin/errors"
 	"github.com/darkweak/souin/tests"
 	"testing"
@@ -12,7 +13,7 @@ func TestInitialize(t *testing.T) {
 	config := tests.MockConfiguration(tests.BaseConfiguration)
 	prs := providers.InitializeProvider(config)
 
-	endpoints := Initialize(prs, config)
+	endpoints := Initialize(prs, config, ykeys.InitializeYKeys(config.Ykeys))
 
 	if len(endpoints) != 2 {
 		errors.GenerateError(t, fmt.Sprintf("Endpoints length should be 1, %d received", len(endpoints)))

api/souin.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/darkweak/souin/api/auth"
 	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/cache/ykeys"
 	"github.com/darkweak/souin/configurationtypes"
 	"net/http"
 	"regexp"
@@ -16,9 +17,10 @@ type SouinAPI struct {
 	enabled  bool
 	provider types.AbstractProviderInterface
 	security *auth.SecurityAPI
+	ykeyStorage *ykeys.YKeyStorage
 }
 
-func initializeSouin(provider types.AbstractProviderInterface, configuration configurationtypes.AbstractConfigurationInterface, api *auth.SecurityAPI) *SouinAPI {
+func initializeSouin(provider types.AbstractProviderInterface, configuration configurationtypes.AbstractConfigurationInterface, api *auth.SecurityAPI, ykeyStorage *ykeys.YKeyStorage) *SouinAPI {
 	basePath := configuration.GetAPI().Souin.BasePath
 	enabled := configuration.GetAPI().Souin.Enable
 	var security *auth.SecurityAPI
@@ -33,6 +35,7 @@ func initializeSouin(provider types.AbstractProviderInterface, configuration con
 		enabled,
 		provider,
 		security,
+		ykeyStorage,
 	}
 }
 
@@ -41,6 +44,13 @@ func (s *SouinAPI) BulkDelete(key string) {
 	s.provider.DeleteMany(key)
 }
 
+func (s *SouinAPI) invalidateFromYKey(key string) {
+	urls := s.ykeyStorage.InvalidateTags([]string{key})
+	for _, u := range urls {
+		s.provider.Delete(u)
+	}
+}
+
 // Delete will delete a record into the provider cache system and will update the Souin API if enabled
 func (s *SouinAPI) Delete(key string) {
 	s.provider.Delete(key)
@@ -80,7 +90,10 @@ func (s *SouinAPI) HandleRequest(w http.ResponseWriter, r *http.Request) {
 		}
 		w.Header().Set("Content-Type", "application/json")
 	case "PURGE":
-		if compile {
+		query := r.URL.Query().Get("ykey")
+		if query != "" {
+			s.invalidateFromYKey(query)
+		} else if compile {
 			submatch := regexp.MustCompile(fmt.Sprintf("%s/(.+)", s.GetBasePath())).FindAllStringSubmatch(r.RequestURI, -1)[0][1]
 			s.BulkDelete(submatch)
 		}

api/souin_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"github.com/darkweak/souin/api/auth"
 	"github.com/darkweak/souin/cache/providers"
+	"github.com/darkweak/souin/cache/ykeys"
 	"github.com/darkweak/souin/errors"
 	"github.com/darkweak/souin/tests"
 	"testing"
@@ -18,19 +19,19 @@ func mockSouinAPI() *SouinAPI {
 		true,
 		prs,
 		security,
+		ykeys.InitializeYKeys(config.Ykeys),
 	}
 }
 
 func TestSouinAPI_BulkDelete(t *testing.T) {
 	souinMock := mockSouinAPI()
-	souinMock.provider.Set("key", []byte("value"), tests.GetMatchedURL("key"), 20*time.Second)
-	souinMock.provider.Set("key2", []byte("value"), tests.GetMatchedURL("key"), 20*time.Second)
+	souinMock.provider.Set("firstKey", []byte("value"), tests.GetMatchedURL("firstKey"), 20*time.Second)
+	souinMock.provider.Set("secondKey", []byte("value"), tests.GetMatchedURL("secondKey"), 20*time.Second)
 	time.Sleep(3 * time.Second)
 	if len(souinMock.GetAll()) != 2 {
 		errors.GenerateError(t, "Souin API should have a record")
 	}
 	souinMock.BulkDelete(".+")
-	time.Sleep(2 * time.Second)
 	if len(souinMock.GetAll()) != 0 {
 		errors.GenerateError(t, "Souin API shouldn't have a record")
 	}

cache/coalescing/requestCoalescing_test.go

@@ -3,6 +3,7 @@ package coalescing
 import (
 	"github.com/darkweak/souin/cache/providers"
 	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/cache/ykeys"
 	"github.com/darkweak/souin/helpers"
 	"github.com/darkweak/souin/rfc"
 	"github.com/darkweak/souin/tests"
@@ -20,7 +21,7 @@ func commonInitializer() (*httptest.ResponseRecorder, *http.Request, *types.Retr
 		Provider:      prs,
 		MatchedURL:    tests.GetMatchedURL(tests.PATH),
 		RegexpUrls:    regexpUrls,
-		Transport:     rfc.NewTransport(prs),
+		Transport:     rfc.NewTransport(prs, ykeys.InitializeYKeys(c.Ykeys)),
 	}
 	r := httptest.NewRequest("GET", "http://"+tests.DOMAIN+tests.PATH, nil)
 	w := httptest.NewRecorder()

cache/providers/badgerProvider.go

@@ -14,7 +14,7 @@ type Badger struct {
 }
 
 // BadgerConnectionFactory function create new Badger instance
-func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (*Badger, error) {
+func BadgerConnectionFactory(_ t.AbstractConfigurationInterface) (*Badger, error) {
 	db, _ := badger.Open(badger.DefaultOptions("").WithInMemory(true))
 
 	return &Badger{db}, nil
@@ -87,9 +87,7 @@ func (provider *Badger) Set(key string, value []byte, url t.URL, duration time.D
 
 // Delete method will delete the response in Badger provider if exists corresponding to key param
 func (provider *Badger) Delete(key string) {
-	go func() {
-		_ = provider.DB.DropPrefix([]byte(key))
-	}()
+	_ = provider.DB.DropPrefix([]byte(key))
 }
 
 // DeleteMany method will delete the responses in Badger provider if exists corresponding to the regex key param

cache/types/souin.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"github.com/darkweak/souin/cache/ykeys"
 	"github.com/darkweak/souin/configurationtypes"
 	"net/http"
 	"regexp"
@@ -14,6 +15,7 @@ type TransportInterface interface {
 	UpdateCacheEventually(req *http.Request) (resp *http.Response, err error)
 	GetVaryLayerStorage() *VaryLayerStorage
 	GetCoalescingLayerStorage() *CoalescingLayerStorage
+	GetYkeyStorage() *ykeys.YKeyStorage
 }
 
 // Transport is an implementation of http.RoundTripper that will return values from a cache
@@ -28,6 +30,7 @@ type Transport struct {
 	MarkCachedResponses    bool
 	VaryLayerStorage       *VaryLayerStorage
 	CoalescingLayerStorage *CoalescingLayerStorage
+	YkeyStorage            *ykeys.YKeyStorage
 }
 
 // RetrieverResponsePropertiesInterface interface