fix(providers): Search by prefix and regexp in NutsDB (#224)

* fix(providers): Search by prefix and regexp in NutsDB

* Increase nutsdb limit

* Fix tests

* Replace fmt to logger

* Automatically generate workflow

* Add Surrogate-Key purge/store debug messages

* Support Træfik v2.7+

* Bump version

.github/workflows/workflow_plugins_generator.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+plugins=("beego"  "chi"  "dotweb"  "echo"  "fiber"  "gin"  "go-zero"  "goyave"  "skipper"  "souin"  "traefik"  "tyk"  "webgo")
+durations=("35"   "30"   "30"      "30"    "45"     "30"   "50"       "35"      "50"       "40"     "30"       "30"   "30")
+versions=("16"    "16"   "16"      "16"    "16"     "16"   "16"       "16"      "18"       "16"     "16"       "16"   "16")
+
+IFS= read -r -d '' tpl <<EOF
+name: Build and validate Souin as plugins
+
+on:
+  - pull_request
+
+jobs:
+  build-caddy-validator:
+    name: Check that Souin build as caddy module
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Add domain.com host to /etc/hosts
+        run: |
+          sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts
+      -
+        name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+      -
+        name: Checkout code
+        uses: actions/checkout@v2
+      -
+        name: Install xcaddy
+        run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
+      -
+        name: Build Souin as caddy module
+        run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin@latest=../..
+      -
+        name: Run detached caddy
+        run: cd plugins/caddy && ./caddy run &
+      -
+        name: Run Caddy E2E tests
+        uses: anthonyvscode/newman-action@v1
+        with:
+          collection: "docs/e2e/Souin E2E.postman_collection.json"
+          folder: Caddy
+          reporters: cli
+          delayRequest: 5000
+EOF
+workflow+="$tpl"
+
+for i in ${!plugins[@]}; do
+  lower="${plugins[$i]}"
+  capitalized="$(tr '[:lower:]' '[:upper:]' <<< ${lower:0:1})${lower:1}"
+  IFS= read -d '' tpl <<EOF
+  build-$lower-validator:
+    name: Check that Souin build as $capitalized middleware
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Add domain.com host to /etc/hosts
+        run: |
+          sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts
+      -
+        name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.${versions[$i]}
+      -
+        name: Checkout code
+        uses: actions/checkout@v2
+      -
+        name: Build Souin as $capitalized plugin
+        run: make build-and-run-$lower
+      -
+        name: Wait for Souin is really loaded inside $capitalized as middleware
+        uses: jakejarvis/wait-action@master
+        with:
+          time: ${durations[$i]}s
+      -
+        name: Set $capitalized logs configuration result as environment variable
+        run: cd plugins/$lower && echo "\$(make load-checker)" >> \$GITHUB_ENV
+      -
+        name: Check if the configuration is loaded to define if Souin is loaded too
+        uses: nick-invision/assert-action@v1
+        with:
+          expected: '"Souin configuration is now loaded."'
+          actual: \${{ env.MIDDLEWARE_RESULT }}
+          comparison: contains
+      -
+        name: Run $capitalized E2E tests
+        uses: anthonyvscode/newman-action@v1
+        with:
+          collection: "docs/e2e/Souin E2E.postman_collection.json"
+          folder: $capitalized
+          reporters: cli
+          delayRequest: 5000
+EOF
+  workflow+="$tpl"
+done
+echo "${workflow%$'\n'}" >  "$( dirname -- "$0"; )/plugins.yml"

Makefile

@@ -6,7 +6,7 @@
 DC=docker-compose
 DC_BUILD=$(DC) build
 DC_EXEC=$(DC) exec
-PLUGINS_LIST=beego caddy chi dotweb echo fiber skipper gin go-zero goyave traefik tyk webgo
+PLUGINS_LIST=beego caddy chi dotweb echo fiber skipper gin go-zero goyave traefik tyk webgo souin
 
 base-build-and-run-%:
 	cd plugins/$* && $(MAKE) prepare
@@ -89,6 +89,9 @@ gatling: ## Launch gatling scenarios
 generate-plantUML: ## Generate plantUML diagrams
 	cd ./docs/plantUML && sh generate.sh && cd ../..
 
+generate-workflow: ## Generate plugin workflow
+	bash .github/workflows/workflow_plugins_generator.sh
+
 golangci-lint: ## Run golangci-lint to ensure the code quality
 	docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.46.2 golangci-lint run -v
 

README.md

@@ -226,13 +226,16 @@ The base path for the souin API is `/souin`.
 The Souin API supports the invalidation by surrogate keys such as Fastly which will replace the Varnish system. You can read the doc [about this system](https://github.com/darkweak/souin/blob/master/cache/surrogate/README.md).
 This system is able to invalidate by tags your cloud provider cache. Actually it supports Akamai and Fastly but in a near future some other providers would be implemented like Cloudflare or Varnish.
 
-| Method  | Endpoint          | Description                                                                              |
-|:--------|:------------------|:-----------------------------------------------------------------------------------------|
-| `GET`   | `/`               | List stored keys cache                                                                   |
-| `PURGE` | `/{id or regexp}` | Purge selected item(s) depending. The parameter can be either a specific key or a regexp |
-| `PURGE` | `/?ykey={key}`    | Purge selected item(s) corresponding to the target ykey such as Varnish (deprecated)     |
+| Method  | Endpoint          | Headers                                                    | Description                                                                                                                                                                         |
+|:--------|:------------------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `GET`   | `/`               | -                                                          | List stored keys cache                                                                                                                                                              |
+| `GET`   | `/surrogate_keys` | -                                                          | List stored keys cache                                                                                                                                                              |
+| `PURGE` | `/{id or regexp}` | -                                                          | Purge selected item(s) depending. The parameter can be either a specific key or a regexp                                                                                            |
+| `PURGE` | `/?ykey={key}`    | -                                                          | Purge selected item(s) corresponding to the target ykey such as Varnish (deprecated)                                                                                                |
+| `PURGE` | `/`               | `Surrogate-Key: Surrogate-Key-First, Surrogate-Key-Second` | Purge selected item(s) belong to the target key in the header `Surrogate-Key` (see [Surrogate-Key system](https://github.com/darkweak/souin/blob/master/cache/surrogate/README.md)) |
 
 ### Security API
+**DEPRECATED**  
 Security API allows users to protect other APIs with JWT authentication.  
 The base path for the security API is `/authentication`.
 
@@ -760,7 +763,7 @@ experimental:
   plugins:
     souin:
       moduleName: github.com/darkweak/souin
-      version: v1.6.9
+      version: 
 ```
 After that you can declare either the whole configuration at once in the middleware block or by service. See the examples below.
 ```yaml

cache/providers/badgerProvider.go

@@ -10,12 +10,14 @@ import (
 	t "github.com/darkweak/souin/configurationtypes"
 	badger "github.com/dgraph-io/badger/v3"
 	"github.com/imdario/mergo"
+	"go.uber.org/zap"
 )
 
 // Badger provider type
 type Badger struct {
 	*badger.DB
-	stale time.Duration
+	stale  time.Duration
+	logger *zap.Logger
 }
 
 var enabledBadgerInstances = make(map[string]*Badger)
@@ -29,15 +31,15 @@ func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (*Badger, error
 		var parsedBadger badger.Options
 		if b, e := json.Marshal(badgerConfiguration.Configuration); e == nil {
 			if e = json.Unmarshal(b, &parsedBadger); e != nil {
-				fmt.Println("Impossible to parse the configuration for the default provider (Badger)", e)
+				c.GetLogger().Sugar().Error("Impossible to parse the configuration for the default provider (Badger)", e)
 			}
 		}
 
 		if err := mergo.Merge(&badgerOptions, parsedBadger, mergo.WithOverride); err != nil {
-			fmt.Println("An error occurred during the badgerOptions merge from the default options with your configuration.")
+			c.GetLogger().Sugar().Error("An error occurred during the badgerOptions merge from the default options with your configuration.")
 		}
-	} else {
-		badgerOptions = badgerOptions.WithInMemory(true).WithNumMemtables(1).WithNumLevelZeroTables(1)
+	} else if badgerConfiguration.Path == "" {
+		badgerOptions = badgerOptions.WithInMemory(true)
 	}
 
 	uid := badgerOptions.Dir + badgerOptions.ValueDir
@@ -48,10 +50,10 @@ func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (*Badger, error
 	db, e := badger.Open(badgerOptions)
 
 	if e != nil {
-		fmt.Println("Impossible to open the Badger DB.", e)
+		c.GetLogger().Sugar().Error("Impossible to open the Badger DB.", e)
 	}
 
-	i := &Badger{DB: db, stale: dc.GetStale()}
+	i := &Badger{DB: db, logger: c.GetLogger(), stale: dc.GetStale()}
 	enabledBadgerInstances[uid] = i
 
 	return i, nil
@@ -112,7 +114,6 @@ func (provider *Badger) Prefix(key string, req *http.Request) []byte {
 		defer it.Close()
 		for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
 			if varyVoter(key, req, string(it.Item().Key())) {
-				fmt.Println(string(it.Item().Key()))
 				_ = it.Item().Value(func(val []byte) error {
 					result = val
 					return nil

cache/providers/embeddedOlricProvider.go

@@ -2,7 +2,6 @@ package providers
 
 import (
 	"context"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -19,10 +18,11 @@ import (
 
 // EmbeddedOlric provider type
 type EmbeddedOlric struct {
-	dm    *olric.DMap
-	db    *olric.Olric
-	stale time.Duration
-	ct    context.Context
+	dm     *olric.DMap
+	db     *olric.Olric
+	stale  time.Duration
+	logger *zap.Logger
+	ct     context.Context
 }
 
 func tryToLoadConfiguration(olricInstance *config.Config, olricConfiguration t.CacheProvider, logger *zap.Logger) (*config.Config, bool) {
@@ -70,7 +70,7 @@ func EmbeddedOlricConnectionFactory(configuration t.AbstractConfigurationInterfa
 
 	started, cancel := context.WithCancel(context.Background())
 	olricInstance.Started = func() {
-		fmt.Println("Embedded Olric is ready")
+		configuration.GetLogger().Sugar().Error("Embedded Olric is ready")
 		defer cancel()
 	}
 
@@ -96,13 +96,14 @@ func EmbeddedOlricConnectionFactory(configuration t.AbstractConfigurationInterfa
 	}
 	dm, e := db.NewDMap("souin-map")
 
-	fmt.Println("Embedded Olric is ready for this node.")
+	configuration.GetLogger().Sugar().Info("Embedded Olric is ready for this node.")
 
 	return &EmbeddedOlric{
-		dm:    dm,
-		db:    db,
-		stale: configuration.GetDefaultCache().GetStale(),
-		ct:    context.Background(),
+		dm:     dm,
+		db:     db,
+		stale:  configuration.GetDefaultCache().GetStale(),
+		logger: configuration.GetLogger(),
+		ct:     context.Background(),
 	}, e
 }
 
@@ -122,7 +123,7 @@ func (provider *EmbeddedOlric) ListKeys() []string {
 		defer c.Close()
 	}
 	if err != nil {
-		fmt.Printf("An error occurred while trying to list keys in Olric: %s\n", err)
+		provider.logger.Sugar().Errorf("An error occurred while trying to list keys in Olric: %s\n", err)
 		return []string{}
 	}
 
@@ -139,14 +140,14 @@ func (provider *EmbeddedOlric) ListKeys() []string {
 func (provider *EmbeddedOlric) Prefix(key string, req *http.Request) []byte {
 	c, err := provider.dm.Query(query.M{
 		"$onKey": query.M{
-			"$regexMatch": "^" + key,
+			"$regexMatch": "^" + key + "({|$)",
 		},
 	})
 	if c != nil {
 		defer c.Close()
 	}
 	if err != nil {
-		fmt.Printf("An error occurred while trying to retrieve data in Olric: %s\n", err)
+		provider.logger.Sugar().Errorf("An error occurred while trying to retrieve data in Olric: %s\n", err)
 		return []byte{}
 	}
 
@@ -240,7 +241,7 @@ func (provider *EmbeddedOlric) Reset() error {
 
 // Destruct method will reset or close provider
 func (provider *EmbeddedOlric) Destruct() error {
-	fmt.Println("Destruct current embedded olric...")
+	provider.logger.Sugar().Debug("Destruct current embedded olric...")
 	return provider.Reset()
 }
 

cache/providers/etcdProvider.go

@@ -33,7 +33,7 @@ func EtcdConnectionFactory(c t.AbstractConfigurationInterface) (*Etcd, error) {
 	cli, err := clientv3.New(etcdConfiguration)
 
 	if err != nil {
-		fmt.Println("Impossible to initialize the Etcd DB.", err)
+		c.GetLogger().Sugar().Error("Impossible to initialize the Etcd DB.", err)
 		return nil, err
 	}
 

cache/providers/nutsProvider.go

@@ -19,7 +19,8 @@ type Nuts struct {
 }
 
 const (
-	bucket = "souin-bucket"
+	bucket    = "souin-bucket"
+	nutsLimit = 1 << 16
 )
 
 func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
@@ -35,6 +36,12 @@ func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
 		}
 	}
 
+	for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} {
+		if v := m[i]; v != nil {
+			m[i], _ = v.(int64)
+		}
+	}
+
 	if v := m["EntryIdxMode"]; v != nil {
 		m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode
 		switch v {
@@ -46,11 +53,11 @@ func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
 	}
 
 	if v := m["SyncEnable"]; v != nil {
-		b, ok := v.(bool)
-		if ok {
+		m["SyncEnable"] = true
+		if b, ok := v.(bool); ok {
 			m["SyncEnable"] = b
-		} else {
-			m["SyncEnable"], _ = strconv.ParseBool(v.(string))
+		} else if s, ok := v.(string); ok {
+			m["SyncEnable"], _ = strconv.ParseBool(s)
 		}
 	}
 
@@ -68,12 +75,12 @@ func NutsConnectionFactory(c t.AbstractConfigurationInterface) (*Nuts, error) {
 		nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{}))
 		if b, e := json.Marshal(nutsConfiguration.Configuration); e == nil {
 			if e = json.Unmarshal(b, &parsedNuts); e != nil {
-				fmt.Println("Impossible to parse the configuration for the Nuts provider", e)
+				c.GetLogger().Sugar().Error("Impossible to parse the configuration for the Nuts provider", e)
 			}
 		}
 
 		if err := mergo.Merge(&nutsOptions, parsedNuts, mergo.WithOverride); err != nil {
-			fmt.Println("An error occurred during the nutsOptions merge from the default options with your configuration.")
+			c.GetLogger().Sugar().Error("An error occurred during the nutsOptions merge from the default options with your configuration.")
 		}
 	} else {
 		nutsOptions.RWMode = nutsdb.MMap
@@ -85,12 +92,10 @@ func NutsConnectionFactory(c t.AbstractConfigurationInterface) (*Nuts, error) {
 	db, e := nutsdb.Open(nutsOptions)
 
 	if e != nil {
-		fmt.Println("Impossible to open the Nuts DB.", e)
+		c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e)
 	}
 
-	i := &Nuts{DB: db, stale: dc.GetStale()}
-
-	return i, nil
+	return &Nuts{DB: db, stale: dc.GetStale()}, nil
 }
 
 // ListKeys method returns the list of existing keys
@@ -132,7 +137,7 @@ func (provider *Nuts) Prefix(key string, req *http.Request) []byte {
 	_ = provider.DB.View(func(tx *nutsdb.Tx) error {
 		prefix := []byte(key)
 
-		if entries, _, err := tx.PrefixScan(bucket, prefix, 0, 50); err != nil {
+		if entries, _, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil {
 			return err
 		} else {
 			for _, entry := range entries {
@@ -181,7 +186,7 @@ func (provider *Nuts) Delete(key string) {
 // DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param
 func (provider *Nuts) DeleteMany(key string) {
 	_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
-		if entries, _, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, 100); err != nil {
+		if entries, _, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil {
 			return err
 		} else {
 			for _, entry := range entries {