Skip to content

Commit

Permalink
improve buffer options (#749)
Browse files Browse the repository at this point in the history 
* backend_responses: use of .json_body necessary for JSON parsing response

* backend_request: use of .json_body necessary for JSON parsing request

* also look for backend_requests (plural)

* buffer backend request body only if necessary

* go fmt

* more tests for buffer options; fix for backend_requests.r

* test for effect of buffer options on buffering/json-parsing backend requests/responses

* definie constants directly, without using iota

* changelog entry

* use methods for buffer options check

* early bufferOpts readout

* add test case for non-json-parsed JSON body because there is no reference to request.json_body

---------

Co-authored-by: Marcel Ludwig <1841067+malud@users.noreply.github.com>
Co-authored-by: Marcel Ludwig <marcel.ludwig@milecrew.com>
  • Loading branch information
@johakoch @malud
3 people authored Aug 16, 2023
1 parent cdebb28 commit 8152b0d
Show file tree
Hide file tree
Showing 16 changed files with 446 additions and 93 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Unreleased changes are available as `avenga/couper:edge` container.

* **Changed**
* More specific error log messages for [`oauth2`](https://docs.couper.io/configuration/block/oauth2) and [`beta_token_request`](https://docs.couper.io/configuration/block/token_request) token request errors ([#755](https://github.com/avenga/couper/pull/755))
* In addition to having an appropriate JSON media type in the `Content-Type` header field, (backend) requests or backend responses for an endpoint are only JSON-parsed if indicated by a [`.json_body` reference](https://docs.couper.io/configuration/variables) in the endpoint configuration ([#749](https://github.com/avenga/couper/pull/749))

* **Fixed**
* Erroneously sending an empty [`Server-Timing` header](https://docs.couper.io/configuration/command-line#oberservation-options) ([#700](https://github.com/avenga/couper/pull/700))
Expand Down
47 changes: 35 additions & 12 deletions eval/buffer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,32 @@ import (
type BufferOption uint8

const (
BufferNone BufferOption = iota
BufferRequest
BufferResponse
BufferNone BufferOption = 0
BufferRequest BufferOption = 1
BufferResponse BufferOption = 2
JSONParseRequest BufferOption = 4
JSONParseResponse BufferOption = 8
)

func (i BufferOption) Request() bool {
return i&BufferRequest == BufferRequest
}

func (i BufferOption) JSONRequest() bool {
return i&JSONParseRequest == JSONParseRequest
}

func (i BufferOption) Response() bool {
return i&BufferResponse == BufferResponse
}

func (i BufferOption) JSONResponse() bool {
return i&JSONParseResponse == JSONParseResponse
}

func (i BufferOption) GoString() string {
var result []string
for _, o := range []BufferOption{BufferRequest, BufferResponse} {
for _, o := range []BufferOption{BufferRequest, BufferResponse, JSONParseRequest, JSONParseResponse} {
if (i & o) == o {
result = append(result, o.String())
}
Expand All @@ -33,10 +51,6 @@ func (i BufferOption) GoString() string {
return strings.Join(result, "|")
}

func (i BufferOption) Response() bool {
return i&BufferResponse == BufferResponse
}

// MustBuffer determines if any of the hcl.bodies makes use of 'body', 'form_body' or 'json_body' or
// of known attributes and variables which require a parsed client-request or backend-response body.
func MustBuffer(bodies ...hcl.Body) BufferOption {
Expand Down Expand Up @@ -70,7 +84,7 @@ func MustBuffer(bodies ...hcl.Body) BufferOption {
rootName := traversal.RootName()

if len(traversal) == 1 {
if rootName == ClientRequest || rootName == BackendRequest {
if rootName == ClientRequest || rootName == BackendRequests || rootName == BackendRequest {
result |= BufferRequest
}
if rootName == BackendResponses || rootName == BackendResponse {
Expand All @@ -79,7 +93,7 @@ func MustBuffer(bodies ...hcl.Body) BufferOption {
continue
}

if rootName != ClientRequest && rootName != BackendRequest && rootName != BackendResponses && rootName != BackendResponse {
if rootName != ClientRequest && rootName != BackendRequests && rootName != BackendRequest && rootName != BackendResponses && rootName != BackendResponse {
continue
}

Expand All @@ -92,6 +106,8 @@ func MustBuffer(bodies ...hcl.Body) BufferOption {
case ClientRequest:
fallthrough
case BackendRequest:
fallthrough
case BackendRequests:
result |= BufferRequest

case BackendResponse:
Expand All @@ -100,31 +116,38 @@ func MustBuffer(bodies ...hcl.Body) BufferOption {
result |= BufferResponse
}
case CTX: // e.g. jwt token (value) could be read from any (body) source
if rootName == ClientRequest || rootName == BackendRequest {
if rootName == ClientRequest || rootName == BackendRequests || rootName == BackendRequest {
result |= BufferRequest
}
case FormBody:
if rootName == ClientRequest || rootName == BackendRequest {
if rootName == ClientRequest || rootName == BackendRequests || rootName == BackendRequest {
result |= BufferRequest
}
case JSONBody:
switch rootName {
case ClientRequest:
fallthrough
case BackendRequest:
fallthrough
case BackendRequests:
result |= BufferRequest
result |= JSONParseRequest

case BackendResponse:
fallthrough
case BackendResponses:
result |= BufferResponse
result |= JSONParseResponse
}
default:
// e.g. backend_responses.default
if len(traversal) == 2 {
if rootName == BackendResponse || rootName == BackendResponses {
result |= BufferResponse
}
if rootName == BackendRequest || rootName == BackendRequests {
result |= BufferRequest
}
}
}
}
Expand Down
22 changes: 18 additions & 4 deletions eval/buffer_string.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 14 additions & 5 deletions eval/buffer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,23 @@ func TestMustBuffer(t *testing.T) {
{"buffer request", `endpoint "/" { set_response_headers = { x = request } }`, BufferRequest},
{"buffer request body", `endpoint "/" { set_response_headers = { x = request.body } }`, BufferRequest},
{"buffer request form_body", `endpoint "/" { set_response_headers = { x = request.form_body } }`, BufferRequest},
{"buffer request json_body", `endpoint "/" { set_response_headers = { x = request.json_body } }`, BufferRequest},
{"buffer request json_body", `endpoint "/" { set_response_headers = { x = request.json_body } }`, BufferRequest | JSONParseRequest},
{"buffer backend_requests specific", `endpoint "/" { set_response_headers = { x = backend_requests.r } }`, BufferRequest},
{"buffer backend_requests body", `endpoint "/" { set_response_headers = { x = backend_requests.r.body } }`, BufferRequest},
{"buffer backend_requests form_body", `endpoint "/" { set_response_headers = { x = backend_requests.r.form_body } }`, BufferRequest},
{"buffer backend_requests json_body", `endpoint "/" { set_response_headers = { x = backend_requests.r.json_body } }`, BufferRequest | JSONParseRequest},
{"buffer backend_request body", `backend "b" { set_response_headers = { x = backend_request.body } }`, BufferRequest},
{"buffer backend_request form_body", `backend "b" { set_response_headers = { x = backend_request.form_body } }`, BufferRequest},
{"buffer backend_request json_body", `backend "b" { set_response_headers = { x = backend_request.json_body } }`, BufferRequest | JSONParseRequest},
{"buffer request add_form_params", `endpoint "/" { add_form_params = [] }`, BufferRequest},
{"buffer request set_form_params", `endpoint "/" { set_form_params = [] }`, BufferRequest},
{"buffer request remove_form_params", `endpoint "/" { remove_form_params = [] }`, BufferRequest},
{"buffer responses", `endpoint "/" { set_response_headers = { x = backend_responses } }`, BufferResponse},
{"buffer default response", `endpoint "/" { set_response_headers = { x = backend_responses.default } }`, BufferResponse},
{"buffer response body", `endpoint "/" { set_response_headers = { x = backend_responses.default.body } }`, BufferResponse},
{"buffer response json_body", `endpoint "/" { set_response_headers = { x = backend_responses.default.json_body } }`, BufferResponse},
{"buffer backend_responses", `endpoint "/" { set_response_headers = { x = backend_responses } }`, BufferResponse},
{"buffer backend_responses default", `endpoint "/" { set_response_headers = { x = backend_responses.default } }`, BufferResponse},
{"buffer backend_responses body", `endpoint "/" { set_response_headers = { x = backend_responses.default.body } }`, BufferResponse},
{"buffer backend_responses json_body", `endpoint "/" { set_response_headers = { x = backend_responses.default.json_body } }`, BufferResponse | JSONParseResponse},
{"buffer backend_response body", `backend "b" { set_response_headers = { x = backend_response.body } }`, BufferResponse},
{"buffer backend_response json_body", `backend "b" { set_response_headers = { x = backend_response.json_body } }`, BufferResponse | JSONParseResponse},
{"buffer request/response", `endpoint "/" {
set_response_headers = {
x = request
Expand Down
33 changes: 21 additions & 12 deletions eval/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,12 @@ func (c *Context) WithClientRequest(req *http.Request) *Context {
}
}
port, _ := strconv.ParseInt(p, 10, 64)
body, jsonBody := parseReqBody(req)

var parseJSON bool
if opts, ok := ctx.Value(request.BufferOptions).(BufferOption); ok {
parseJSON = opts.JSONRequest()
}
body, jsonBody := parseReqBody(req, parseJSON)

origin := NewRawOrigin(req.URL)
ctx.eval.Variables[ClientRequest] = cty.ObjectVal(ctxMap.Merge(ContextMap{
Expand Down Expand Up @@ -254,7 +259,13 @@ func newBerespValues(ctx context.Context, readBody bool, beresp *http.Response)
}
port, _ := strconv.ParseInt(p, 10, 64)

body, jsonBody := parseReqBody(bereq)
bufferOption, bOk := bereq.Context().Value(request.BufferOptions).(BufferOption)

var body, jsonBody cty.Value
if bOk && bufferOption.Request() {
body, jsonBody = parseReqBody(bereq, bufferOption.JSONRequest())
}

bereqVal = cty.ObjectVal(ContextMap{
Method: cty.StringVal(bereq.Method),
URL: cty.StringVal(bereq.URL.String()),
Expand All @@ -269,14 +280,12 @@ func newBerespValues(ctx context.Context, readBody bool, beresp *http.Response)
FormBody: seetie.ValuesMapToValue(parseForm(bereq).PostForm),
}.Merge(newVariable(ctx, bereq.Cookies(), bereq.Header)))

bufferOption, bOk := bereq.Context().Value(request.BufferOptions).(BufferOption)

var respBody, respJSONBody cty.Value
if readBody && !IsUpgradeResponse(bereq, beresp) {
if bOk && (bufferOption&BufferResponse) == BufferResponse {
respBody, respJSONBody = parseRespBody(beresp)
if bOk && bufferOption.Response() {
respBody, respJSONBody = parseRespBody(beresp, bufferOption.JSONResponse())
}
} else if bOk && (bufferOption&BufferResponse) != BufferResponse {
} else if bOk && !bufferOption.Response() {
hasBlock, _ := bereq.Context().Value(request.ResponseBlock).(bool)
ws, _ := bereq.Context().Value(request.WebsocketsAllowed).(bool)
if name != "default" || (name == "default" && hasBlock) {
Expand Down Expand Up @@ -467,7 +476,7 @@ func mergeBackendVariables(etx *hcl.EvalContext, key string, cmap ContextMap) {
const defaultMaxMemory = 32 << 20 // 32 MB

// parseForm populates the request PostForm field.
// As Proxy we should not consume the request body.
// As Proxy, we should not consume the request body.
// Rewind body via GetBody method.
func parseForm(r *http.Request) *http.Request {
if r.GetBody == nil || r.Form != nil {
Expand All @@ -487,7 +496,7 @@ func isJSONMediaType(contentType string) bool {
return len(mParts) == 2 && mParts[0] == "application" && (mParts[1] == "json" || strings.HasSuffix(mParts[1], "+json"))
}

func parseReqBody(req *http.Request) (cty.Value, cty.Value) {
func parseReqBody(req *http.Request, parseJSON bool) (cty.Value, cty.Value) {
jsonBody := cty.EmptyObjectVal
if req == nil || req.GetBody == nil {
return cty.NilVal, jsonBody
Expand All @@ -499,21 +508,21 @@ func parseReqBody(req *http.Request) (cty.Value, cty.Value) {
return cty.NilVal, jsonBody
}

if isJSONMediaType(req.Header.Get("Content-Type")) {
if parseJSON && isJSONMediaType(req.Header.Get("Content-Type")) {
jsonBody = parseJSONBytes(b)
}
return cty.StringVal(string(b)), jsonBody
}

func parseRespBody(beresp *http.Response) (cty.Value, cty.Value) {
func parseRespBody(beresp *http.Response, parseJSON bool) (cty.Value, cty.Value) {
jsonBody := cty.EmptyObjectVal

b := parseSetRespBody(beresp)
if b == nil {
return cty.NilVal, jsonBody
}

if isJSONMediaType(beresp.Header.Get("Content-Type")) {
if parseJSON && isJSONMediaType(beresp.Header.Get("Content-Type")) {
jsonBody = parseJSONBytes(b)
}
return cty.StringVal(string(b)), jsonBody
Expand Down
Loading

0 comments on commit 8152b0d

Please sign in to comment.