diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 61289b95a..ea3ccf43b 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -7,6 +7,9 @@ jobs: name: Check that Souin build as caddy module runs-on: ubuntu-latest steps: + - name: Add domain.com host to /etc/hosts + run: | + sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts - name: Install Go uses: actions/setup-go@v2 with: @@ -22,6 +25,15 @@ jobs: sudo apt install xcaddy - name: Build Souin as caddy module run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin@latest=../.. + - name: Run detached caddy + run: cd plugins/caddy && ./caddy run & + - name: Run Caddy E2E tests + uses: anthonyvscode/newman-action@v1 + with: + collection: "docs/e2e/Souin E2E.postman_collection.json" + folder: Caddy + reporters: cli + delayRequest: 5000 build-tyk-validator: name: Check that Souin build as Tyk middleware runs-on: ubuntu-latest @@ -46,10 +58,20 @@ jobs: expected: '[INFO] Olric bindAddr' actual: ${{ env.TYK_DC_RESULT }} comparison: contains + - name: Run Tyk E2E tests + uses: anthonyvscode/newman-action@v1 + with: + collection: "docs/e2e/Souin E2E.postman_collection.json" + folder: Tyk + reporters: cli + delayRequest: 5000 build-traefik-validator: name: Check that Souin build as Træfik plugin runs-on: ubuntu-latest steps: + - name: Add domain.com host to /etc/hosts + run: | + sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts - name: Install Go uses: actions/setup-go@v2 with: @@ -78,3 +100,10 @@ jobs: expected: '"middlewares\":{\"souin\":{\"plugin\":{\"souin-plugin' actual: ${{ env.TRAEFIK_MIDDLEWARE_RESULT }} comparison: contains + - name: Run Træfik E2E tests + uses: anthonyvscode/newman-action@v1 + with: + collection: "docs/e2e/Souin E2E.postman_collection.json" + folder: Traefik + reporters: cli + delayRequest: 5000 diff --git a/Makefile b/Makefile index 8d866d675..56563e8a2 100644 --- a/Makefile +++ b/Makefile @@ -84,3 +84,4 @@ validate: lint tests down health-check-prod ## Run lint, tests and ensure prod c vendor-plugins: ## Generate and prepare vendors for each plugin cd plugins/tyk && $(MAKE) vendor cd plugins/traefik && $(MAKE) vendor + cd plugins/caddy && go mod tidy && go mod download diff --git a/docs/e2e/Souin E2E.postman_collection.json b/docs/e2e/Souin E2E.postman_collection.json new file mode 100644 index 000000000..8e08f7d33 --- /dev/null +++ b/docs/e2e/Souin E2E.postman_collection.json @@ -0,0 +1,486 @@ +{ + "info": { + "_postman_id": "701efdfd-3e8c-4786-8ba3-bfd5871d6293", + "name": "Souin E2E", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "Caddy", + "item": [ + { + "name": "Default", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, `${utils.getVar(pm, 'caddy_url')}/default`)" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Cache-Control", + "value": "", + "type": "text" + } + ], + "url": { + "raw": "{{caddy_url}}/default", + "host": [ + "{{caddy_url}}" + ], + "path": [ + "default" + ] + } + }, + "response": [] + }, + { + "name": "Souin api", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "utils.souinAPI.listKeys(pm, 'GET-localhost:4443-/test1', utils.getVar(pm, 'caddy_url'), '/test1')" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Cache-Control", + "value": "", + "type": "text" + } + ], + "url": { + "raw": "{{caddy_url}}{{souin_base_api}}{{souin_api}}", + "host": [ + "{{caddy_url}}{{souin_base_api}}{{souin_api}}" + ] + } + }, + "response": [] + }, + { + "name": "Default no cache", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, `${utils.getVar(pm, 'caddy_url')}/default`, 'no-cache')" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{caddy_url}}/default", + "host": [ + "{{caddy_url}}" + ], + "path": [ + "default" + ] + } + }, + "response": [] + }, + { + "name": "Default no store", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, `${utils.getVar(pm, 'caddy_url')}/default`, 'no-store')" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Cache-Control", + "value": "no-cache", + "type": "text" + } + ], + "url": { + "raw": "{{caddy_url}}/default", + "host": [ + "{{caddy_url}}" + ], + "path": [ + "default" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + }, + { + "name": "Traefik", + "item": [ + { + "name": "Default", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, utils.getVar(pm, 'traefik_url'))" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Cache-Control", + "value": "", + "type": "text" + } + ], + "url": { + "raw": "{{traefik_url}}", + "host": [ + "{{traefik_url}}" + ] + } + }, + "response": [] + }, + { + "name": "Souin api", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "utils.souinAPI.listKeys(pm, 'GET-domain.com-/', utils.getVar(pm, 'traefik_url'), '/')" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Cache-Control", + "value": "", + "type": "text" + } + ], + "url": { + "raw": "{{traefik_url}}{{souin_base_api}}{{souin_api}}", + "host": [ + "{{traefik_url}}{{souin_base_api}}{{souin_api}}" + ] + } + }, + "response": [] + }, + { + "name": "Default no cache", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, utils.getVar(pm, 'traefik_url'), 'no-cache')" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{traefik_url}}", + "host": [ + "{{traefik_url}}" + ] + } + }, + "response": [] + }, + { + "name": "Default no store", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, utils.getVar(pm, 'traefik_url'), 'no-store')" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{traefik_url}}", + "host": [ + "{{traefik_url}}" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + }, + { + "name": "Tyk", + "item": [ + { + "name": "Default", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "utils.baseEndpoint(pm, `${utils.getVar(pm, 'tyk_url')}/`)" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Cache-Control", + "value": "", + "type": "text" + } + ], + "url": { + "raw": "{{tyk_url}}/", + "host": [ + "{{tyk_url}}" + ], + "path": [ + "" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "utils = {", + " request: (url, cacheControl = '') => ({", + " header: {", + " 'Cache-Control': cacheControl", + " },", + " method: 'GET',", + " url,", + " }),", + " getVar: (pm, v) => pm.collectionVariables.get(v) || '',", + " baseEndpoint: (pm, baseUrl = '', cacheControl = '') => {", + " pm.test(\"Status code is 200 without Cache-Status header\", function () {", + " pm.response.to.have.status(200);", + " pm.response.to.not.have.header(\"Cache-Status\");", + " pm.response.to.not.have.header(\"Age\");", + " });", + " pm.sendRequest(utils.request(baseUrl, cacheControl), function (_, response) {", + " pm.test(`Status code is 200, Cache-Status and Age are ${cacheControl == '' ? '' : 'not '}present`, function () {", + " const expected = pm.expect(response);", + " expected.to.have.status(200);", + "", + " if (cacheControl == '') {", + " expected.to.have.header(\"Age\");", + " expected.to.have.header(\"Cache-Status\");", + " } else {", + " expected.to.not.have.header(\"Age\");", + " expected.to.not.have.header(\"Cache-Status\");", + " }", + " }", + " );", + " });", + " },", + " souinAPI: {", + " listKeys: (pm, baseKey, baseUrl = '', additionalPath = '', cacheControl = '') => {", + " const isCached = cacheControl == ''", + " pm.test(\"Ensure stored keys array is empty\", function () {", + " pm.response.to.have.status(200);", + " let jsonData = pm.response.json();", + " pm.expect(jsonData).to.eql([]);", + " pm.expect(jsonData.length).to.eql(0);", + " });", + " pm.sendRequest(utils.request(baseUrl + additionalPath, cacheControl), function(_, response) {", + " pm.expect(response).to.have.status(200);", + " });", + "", + " pm.sendRequest(utils.request(`${baseUrl}${utils.getVar(pm, 'souin_base_api')}${utils.getVar(pm, 'souin_api')}`, cacheControl), function (_, res) {", + " pm.test(`Check Souin API has ${isCached ? 'one' : 'none'} registered key after the first cache set`, function () {", + " pm.expect(res).to.have.status(200);", + " let jsonData = res.json();", + " pm.expect(jsonData.length).to.eql(isCached ? 1 : 0);", + " pm.expect(jsonData[0]).to.eql(isCached ? baseKey : undefined);", + " }", + " );", + "", + " pm.sendRequest(utils.request(baseUrl + additionalPath + 'testing', cacheControl), function() {", + " pm.sendRequest(utils.request(`${baseUrl}${utils.getVar(pm, 'souin_base_api')}${utils.getVar(pm, 'souin_api')}`, cacheControl), function (_, r) {", + " pm.test(`Check Souin API has ${isCached ? 'two' : 'none'} registered key${isCached ? 's' : ''} after the second cache set`, function () {", + " pm.expect(r).to.have.status(200);", + " pm.expect(r).to.not.have.header(\"Cache-Status\");", + " pm.expect(r).to.not.have.header(\"Age\");", + " let jsonData = r.json();", + " pm.expect(jsonData.length).to.eql(isCached ? 2 : 0);", + " pm.expect(jsonData[0]).to.eql(isCached ? baseKey : undefined);", + " pm.expect(jsonData[1]).to.eql(isCached ? `${baseKey}testing` : undefined);", + " });", + " });", + " });", + " });", + " },", + " },", + "}" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], + "variable": [ + { + "key": "caddy_url", + "value": "http://localhost:4443" + }, + { + "key": "souin_api", + "value": "/souin" + }, + { + "key": "souin_base_api", + "value": "/souin-api" + }, + { + "key": "traefik_url", + "value": "http://domain.com" + }, + { + "key": "tyk_url", + "value": "http://localhost:8080/httpbin" + } + ] +} \ No newline at end of file diff --git a/plugins/caddy/Caddyfile b/plugins/caddy/Caddyfile index 9b191efb9..8b4cdddfc 100644 --- a/plugins/caddy/Caddyfile +++ b/plugins/caddy/Caddyfile @@ -4,21 +4,27 @@ level debug } cache { + api { + souin { + enable true + } + } headers Content-Type Authorization log_level debug ttl 1000s } } -:80 +:4443 +respond "Hello World!" @match path /test1* @match2 path /test2* @matchdefault path /default +@souin-api path /souin-api* cache @match { - ttl 30s - headers Cookie + ttl 5s } cache @match2 { @@ -27,5 +33,7 @@ cache @match2 { } cache @matchdefault { - ttl 25s + ttl 5s } + +cache @souin-api {} diff --git a/plugins/caddy/app.go b/plugins/caddy/app.go index a22cf003d..ae8e5928c 100644 --- a/plugins/caddy/app.go +++ b/plugins/caddy/app.go @@ -2,12 +2,16 @@ package caddy import ( "github.com/caddyserver/caddy/v2" + "github.com/darkweak/souin/cache/types" + "github.com/darkweak/souin/configurationtypes" ) // SouinApp contains the whole Souin necessary items type SouinApp struct { *DefaultCache - LogLevel string `json:"log_level,omitempty"` + Provider types.AbstractProviderInterface + API configurationtypes.API `json:"api,omitempty"` + LogLevel string `json:"log_level,omitempty"` } func init() { diff --git a/plugins/caddy/go.mod b/plugins/caddy/go.mod index 897005d3c..3228dc1bf 100644 --- a/plugins/caddy/go.mod +++ b/plugins/caddy/go.mod @@ -4,8 +4,8 @@ go 1.16 require ( github.com/caddyserver/caddy/v2 v2.4.4 - github.com/darkweak/souin v1.5.4 + github.com/darkweak/souin v1.5.4-beta2 go.uber.org/zap v1.19.0 ) -replace github.com/darkweak/souin v1.5.4 => ../.. +replace github.com/darkweak/souin v1.5.4-beta2 => ../.. diff --git a/plugins/caddy/main.go b/plugins/caddy/main.go index decd79736..393d99940 100644 --- a/plugins/caddy/main.go +++ b/plugins/caddy/main.go @@ -5,7 +5,9 @@ import ( "context" "github.com/darkweak/souin/api" "github.com/darkweak/souin/cache/coalescing" + "github.com/darkweak/souin/cache/types" "net/http" + "strconv" "sync" "time" @@ -132,6 +134,8 @@ func (s *SouinCaddyPlugin) FromApp(app *SouinApp) error { return nil } + s.Configuration.API = app.API + if s.Configuration.DefaultCache == nil { s.Configuration.DefaultCache = &DefaultCache{ Headers: app.Headers, @@ -178,6 +182,12 @@ func (s *SouinCaddyPlugin) Provision(ctx caddy.Context) error { }, } s.Retriever = plugins.DefaultSouinPluginInitializerFromConfiguration(s.Configuration) + if app.(*SouinApp).Provider == nil { + app.(*SouinApp).Provider = s.Retriever.GetProvider() + } else { + s.Retriever.(*types.RetrieverResponseProperties).Provider = app.(*SouinApp).Provider + s.Retriever.GetTransport().(*rfc.VaryTransport).Provider = app.(*SouinApp).Provider + } s.RequestCoalescing = coalescing.Initialize() s.MapHandler = api.GenerateHandlerMap(s.Configuration, s.Retriever.GetTransport().GetProvider(), s.Retriever.GetTransport().GetYkeyStorage()) return nil @@ -215,6 +225,42 @@ func parseCaddyfileGlobalOption(h *caddyfile.Dispenser, _ interface{}) (interfac for nesting := h.Nesting(); h.NextBlock(nesting); { rootOption := h.Val() switch rootOption { + case "api": + apiConfiguration := configurationtypes.API{} + for nesting := h.Nesting(); h.NextBlock(nesting); { + directive := h.Val() + switch directive { + case "basepath": + apiConfiguration.BasePath = h.RemainingArgs()[0] + case "souin": + apiConfiguration.Souin = configurationtypes.APIEndpoint{} + for nesting := h.Nesting(); h.NextBlock(nesting); { + directive := h.Val() + switch directive { + case "basepath": + apiConfiguration.Souin.BasePath = h.RemainingArgs()[0] + case "enable": + apiConfiguration.Souin.Enable, _ = strconv.ParseBool(h.RemainingArgs()[0]) + case "security": + apiConfiguration.Souin.Security, _ = strconv.ParseBool(h.RemainingArgs()[0]) + } + } + case "security": + apiConfiguration.Security = configurationtypes.SecurityAPI{} + for nesting := h.Nesting(); h.NextBlock(nesting); { + directive := h.Val() + switch directive { + case "basepath": + apiConfiguration.Security.BasePath = h.RemainingArgs()[0] + case "enable": + apiConfiguration.Security.Enable, _ = strconv.ParseBool(h.RemainingArgs()[0]) + case "secret": + apiConfiguration.Security.Secret = h.RemainingArgs()[0] + } + } + } + } + cfg.API = apiConfiguration case "headers": args := h.RemainingArgs() cfg.DefaultCache.Headers = append(cfg.DefaultCache.Headers, args...) @@ -264,6 +310,7 @@ func parseCaddyfileGlobalOption(h *caddyfile.Dispenser, _ interface{}) (interfac } souinApp.DefaultCache = cfg.DefaultCache + souinApp.API = cfg.API return httpcaddyfile.App{ Name: moduleName, diff --git a/plugins/traefik/go.mod b/plugins/traefik/go.mod index a5a8b0fbc..06d92ec67 100644 --- a/plugins/traefik/go.mod +++ b/plugins/traefik/go.mod @@ -2,10 +2,10 @@ module github.com/darkweak/souin/plugins/traefik go 1.16 -replace github.com/darkweak/souin v1.5.2 => ../.. +replace github.com/darkweak/souin v1.5.4-beta2 => ../.. require ( - github.com/darkweak/souin v1.5.2 + github.com/darkweak/souin v1.5.4-beta2 github.com/patrickmn/go-cache v2.1.0+incompatible go.uber.org/zap v1.19.0 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect diff --git a/plugins/traefik/souin-configuration.yaml b/plugins/traefik/souin-configuration.yaml index e87266558..52c0d5ef6 100644 --- a/plugins/traefik/souin-configuration.yaml +++ b/plugins/traefik/souin-configuration.yaml @@ -28,11 +28,11 @@ http: - Content-Type regex: exclude: 'ARegexHere' - ttl: 100s + ttl: 5s log_level: debug urls: 'domain.com/testing': - ttl: 2s + ttl: 5s headers: - Authorization 'mysubdomain.domain.com': diff --git a/plugins/traefik/vendor/modules.txt b/plugins/traefik/vendor/modules.txt index 9c94c91d6..107fd1be3 100644 --- a/plugins/traefik/vendor/modules.txt +++ b/plugins/traefik/vendor/modules.txt @@ -25,7 +25,7 @@ github.com/buraksezer/olric/stats github.com/cespare/xxhash # github.com/cespare/xxhash/v2 v2.1.1 github.com/cespare/xxhash/v2 -# github.com/darkweak/souin v1.5.2 => ../.. +# github.com/darkweak/souin v1.5.4-beta2 => ../.. ## explicit github.com/darkweak/souin/api github.com/darkweak/souin/api/auth diff --git a/plugins/tyk/go.mod b/plugins/tyk/go.mod index f484e92e8..efe5a0cd1 100644 --- a/plugins/tyk/go.mod +++ b/plugins/tyk/go.mod @@ -6,7 +6,7 @@ require ( github.com/TykTechnologies/gojsonschema v0.0.0-20170222154038-dcb3e4bb7990 // indirect github.com/TykTechnologies/tyk v2.9.5+incompatible github.com/clbanning/mxj v1.8.4 // indirect - github.com/darkweak/souin v1.5.2 + github.com/darkweak/souin v1.5.4-beta2 github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2 // indirect github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8 // indirect github.com/hashicorp/terraform v1.0.1 // indirect diff --git a/plugins/tyk/samples/apps/httpbin.json b/plugins/tyk/samples/apps/httpbin.json index a61eccb9c..73be48bb3 100644 --- a/plugins/tyk/samples/apps/httpbin.json +++ b/plugins/tyk/samples/apps/httpbin.json @@ -52,7 +52,7 @@ } }, "default_cache": { - "ttl": "100s" + "ttl": "5s" } } }