Backend variables

CHANGELOG.md

@@ -6,6 +6,7 @@ Unreleased changes are available as `avenga/couper:edge` container.
 
 * **Added**
   * `disable_private_caching` attribute for the [JWT Block](./docs/REFERENCE.md#jwt-block) ([#418](https://github.com/avenga/couper/pull/418))
+  * [`backend_request`](./docs/REFERENCE.md#backend_request) and [`backend_response`](./docs/REFERENCE.md#backend_response) variables ([#430](https://github.com/avenga/couper/pull/430))
   * `beta_scope_map` attribute for the [JWT Block](./docs/REFERENCE.md#jwt-block) ([#434](https://github.com/avenga/couper/pull/434))
 
 * **Fixed**
@@ -44,14 +45,10 @@ On top of that the binary installation has been improved for [homebrew](https://
   * The access control for the OIDC redirect endpoint ([`oidc` block](./docs/REFERENCE.md#oidc-block)) now verifies ID token signatures ([#404](https://github.com/avenga/couper/pull/404))
   * `header = "Authorization"` is now the default token source for [JWT](./docs/REFERENCE.md#jwt-block) and may be omitted ([#413](https://github.com/avenga/couper/issues/413))
   * Improved the validation for unique keys in all map-attributes in the config ([#403](https://github.com/avenga/couper/pull/403))
-<<<<<<< HEAD
   * Missing [scope or roles claims](./docs/REFERENCE.md#jwt-block), or scope or roles claim with unsupported values are now ignored instead of causing an error ([#380](https://github.com/avenga/couper/issues/380))
-=======
-  * The access control for the OIDC redirect endpoint ([`oidc` block](./docs/REFERENCE.md#oidc-block)) now verifies ID token signatures ([#404](https://github.com/avenga/couper/pull/404))
   * Unbeta [OIDC block](./docs/REFERENCE.md#oidc-block). The old block name is still usable with Couper 1.7, but will no longer work with Couper 1.8. ([#400](https://github.com/avenga/couper/pull/400))
   * Unbeta the `oauth2_authorization_url()` and `oauth2_verifier()` [function](./docs/REFERENCE.md#functions). The prefix is changed from `beta_oauth_...` to `oauth2_...`. The old function names are still usable with Couper 1.7, but will no longer work with Couper 1.8. ([#400](https://github.com/avenga/couper/pull/400))
   * Automatically add the `private` directive to the response `Cache-Control` HTTP header field value for all resources protected by [JWT](./docs/REFERENCE.md#jwt-block) ([#418](https://github.com/avenga/couper/pull/418))
->>>>>>> 311ec60f... Changelog
 
 * **Fixed**
   * build-date configuration for binary and docker builds ([#396](https://github.com/avenga/couper/pull/396))

docs/REFERENCE.md

@@ -32,7 +32,9 @@
     - [couper](#couper)
     - [env](#env)
     - [request](#request)
+    - [backend_request](#backend_request)
     - [backend_requests](#backend_requests)
+    - [backend_response](#backend_response)
     - [backend_responses](#backend_responses)
   - [Functions](#functions)
   - [Modifiers](#modifiers)
@@ -650,9 +652,14 @@ and for OIDC additionally:
 - `id_token_claims`: a map of claims from the ID token
 - `userinfo`: a map of claims retrieved from the userinfo endpoint
 
+### `backend_request`
+
+`backend_request` holds information about the current backend request. It is only
+available in a [Backend Block](#backend-block), and has the same attributes as a backend request in `backend_requests.<label>` (see [backend_requests](#backend_requests) below).
+
 ### `backend_requests`
 
-`backend_requests.<label>` is a list of all backend requests, and their variables.
+`backend_requests` is an object with all backend requests and their attributes.
 To access a specific request use the related label. [Request](#request-block) and
 [Proxy](#proxy-block) blocks without a label will be available as `default`.
 To access the HTTP method of the `default` request use `backend_requests.default.method` .
@@ -675,9 +682,14 @@ To access the HTTP method of the `default` request use `backend_requests.default
 | `port`                           | integer         | Port of the backend request URL                                                                                                                                                                                                                                                      | `443`                                       |
 | `path`                           | string          | Backend request URL path                                                                                                                                                                                                                                                             | `/path/to`                                  |
 
+### `backend_response`
+
+`backend_response` represents the current backend response.  It is only
+available in a [Backend Block](#backend-block), and has the same attributes as a backend response in `backend_responses.<label>` (see [backend_responses](#backend_responses) below).
+
 ### `backend_responses`
 
-`backend_responses.<label>` is a list of all backend responses, and their variables. Same behaviour as for `backend_requests`.
+`backend_responses` is an object with all backend responses and their attributes.
 Use the related label to access a specific response.
 [Request](#request-block) and [Proxy](#proxy-block) blocks without a label will be available as `default`.
 To access the HTTP status code of the `default` response use `backend_responses.default.status` .

eval/buffer.go

@@ -69,16 +69,16 @@ func MustBuffer(bodies ...hcl.Body) BufferOption {
 			rootName := traversal.RootName()
 
 			if len(traversal) == 1 {
-				if rootName == ClientRequest {
+				if rootName == ClientRequest || rootName == BackendRequest {
 					result |= BufferRequest
 				}
-				if rootName == BackendResponses {
+				if rootName == BackendResponses || rootName == BackendResponse {
 					result |= BufferResponse
 				}
 				continue
 			}
 
-			if rootName != ClientRequest && rootName != BackendResponses {
+			if rootName != ClientRequest && rootName != BackendRequest && rootName != BackendResponses && rootName != BackendResponse {
 				continue
 			}
 
@@ -89,34 +89,47 @@ func MustBuffer(bodies ...hcl.Body) BufferOption {
 				case Body:
 					switch rootName {
 					case ClientRequest:
+						fallthrough
+					case BackendRequest:
 						result |= BufferRequest
+
+					case BackendResponse:
+						fallthrough
 					case BackendResponses:
 						result |= BufferResponse
 					}
 				case CTX: // e.g. jwt token (value) could be read from any (body) source
-					if rootName == ClientRequest {
+					if rootName == ClientRequest || rootName == BackendRequest {
 						result |= BufferRequest
 					}
 				case FormBody:
-					if rootName == ClientRequest {
+					if rootName == ClientRequest || rootName == BackendRequest {
 						result |= BufferRequest
 					}
 				case JsonBody:
 					switch rootName {
 					case ClientRequest:
+						fallthrough
+					case BackendRequest:
 						result |= BufferRequest
+
+					case BackendResponse:
+						fallthrough
 					case BackendResponses:
 						result |= BufferResponse
 					}
 				default:
 					// e.g. backend_responses.default
-					if rootName == BackendResponses && len(traversal) == 2 {
-						result |= BufferResponse
+					if len(traversal) == 2 {
+						if rootName == BackendResponse || rootName == BackendResponses {
+							result |= BufferResponse
+						}
 					}
 				}
 			}
 		}
 	}
+
 	return result
 }
 

eval/context.go

@@ -166,7 +166,7 @@ func (c *Context) WithClientRequest(req *http.Request) *Context {
 	return ctx
 }
 
-func (c *Context) WithBeresps(beresps ...*http.Response) *Context {
+func (c *Context) WithBeresp(beresp *http.Response) *Context {
 	ctx := &Context{
 		eval:              c.cloneEvalContext(),
 		inner:             c.inner,
@@ -180,14 +180,14 @@ func (c *Context) WithBeresps(beresps ...*http.Response) *Context {
 
 	resps := make(ContextMap)
 	bereqs := make(ContextMap)
-	for _, beresp := range beresps {
-		if beresp == nil {
-			continue
-		}
 
+	if beresp != nil {
 		name, bereqVal, berespVal := newBerespValues(ctx, false, beresp)
 		bereqs[name] = bereqVal
 		resps[name] = berespVal
+
+		ctx.eval.Variables[BackendRequest] = bereqVal
+		ctx.eval.Variables[BackendResponse] = berespVal
 	}
 
 	// Prevent overriding existing variables with successive calls to this method.

eval/context_test.go

@@ -99,7 +99,7 @@ func TestNewHTTPContext(t *testing.T) {
 
 			helper.Must(eval.SetGetBody(req, eval.BufferRequest, 512))
 
-			ctx := baseCtx.WithClientRequest(req).WithBeresps(beresp).HCLContext()
+			ctx := baseCtx.WithClientRequest(req).WithBeresp(beresp).HCLContext()
 			ctx.Functions = nil // we are not interested in a functions test
 
 			var resultMap map[string]cty.Value

eval/lib/jwt_test.go

@@ -633,7 +633,7 @@ func TestJwtSignDynamic(t *testing.T) {
 
 			evalCtx := cf.Context.Value(request.ContextType).(*eval.Context).
 				WithClientRequest(req).
-				WithBeresps(beresp)
+				WithBeresp(beresp)
 
 			now := time.Now().Unix()
 			token, err := evalCtx.HCLContext().Functions[lib.FnJWTSign].Call([]cty.Value{cty.StringVal(tt.jspLabel), claims})

eval/lib/oauth2_test.go

@@ -135,7 +135,7 @@ func TestNewOAuthAuthorizationUrlFunction(t *testing.T) {
 			ctx := eval.NewContext(nil, &config.Defaults{}).
 				WithOidcConfig(oidc.Configs{conf.Name: conf}).
 				WithClientRequest(req).
-				WithBeresps(res)
+				WithBeresp(res)
 
 			hclCtx := ctx.HCLContext()
 			val, err := hclCtx.Functions[lib.FnOAuthAuthorizationUrl].Call([]cty.Value{cty.StringVal("auth-ref")})

eval/sync.go

@@ -1,9 +1,10 @@
 package eval
 
 import (
-	"github.com/zclconf/go-cty/cty"
 	"net/http"
 	"sync"
+
+	"github.com/zclconf/go-cty/cty"
 )
 
 type SyncedVariables struct {

eval/variables.go

@@ -1,7 +1,9 @@
 package eval
 
 const (
+	BackendRequest   = "backend_request"
 	BackendRequests  = "backend_requests"
+	BackendResponse  = "backend_response"
 	BackendResponses = "backend_responses"
 	BackendDefault   = "default"
 	Body             = "body"

handler/transport/backend.go

@@ -160,7 +160,7 @@ func (b *Backend) RoundTrip(req *http.Request) (*http.Response, error) {
 	// to the current beresp obj. Downstream response context evals reading their beresp variable values
 	// from this result.
 	evalCtx := eval.ContextFromRequest(req)
-	evalCtx = evalCtx.WithBeresps(beresp)
+	evalCtx = evalCtx.WithBeresp(beresp)
 	err = eval.ApplyResponseContext(evalCtx.HCLContext(), b.context, beresp)
 
 	if varSync, ok := req.Context().Value(request.ContextVariablesSynced).(*eval.SyncedVariables); ok {

logging/hooks/custom_logs.go

@@ -68,7 +68,29 @@ func fire(entry *logrus.Entry, bodyKey request.ContextKey) {
 		return
 	}
 
-	if fields := eval.ApplyCustomLogs(evalCtx.HCLContextSync(), hclBodies, entry); len(fields) > 0 {
+	ctx := evalCtx.HCLContextSync()
+
+	if request.LogCustomUpstream == bodyKey {
+		if _, ok := ctx.Variables[eval.BackendRequests]; ok {
+			for k, v := range ctx.Variables[eval.BackendRequests].AsValueMap() {
+				if k == entry.Context.Value(request.RoundTripName) {
+					ctx.Variables[eval.BackendRequest] = v
+					break
+				}
+			}
+		}
+
+		if _, ok := ctx.Variables[eval.BackendResponses]; ok {
+			for k, v := range ctx.Variables[eval.BackendResponses].AsValueMap() {
+				if k == entry.Context.Value(request.RoundTripName) {
+					ctx.Variables[eval.BackendResponse] = v
+					break
+				}
+			}
+		}
+	}
+
+	if fields := eval.ApplyCustomLogs(ctx, hclBodies, entry); len(fields) > 0 {
 		entry.Data[customLogField] = fields
 	}
 }

server/http_endpoints_test.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/hcl/v2"
+	"github.com/sirupsen/logrus"
 
 	"github.com/avenga/couper/config/configload"
 	"github.com/avenga/couper/internal/test"
@@ -27,6 +28,130 @@ import (
 
 const testdataPath = "testdata/endpoints"
 
+func TestBackend_BackendVariable_RequestResponse(t *testing.T) {
+	client := newClient()
+	helper := test.New(t)
+
+	shutdown, hook := newCouper("testdata/integration/backend/01_couper.hcl", helper)
+	defer shutdown()
+
+	req, err := http.NewRequest(http.MethodGet, "http://example.com:8080/request", nil)
+	helper.Must(err)
+
+	hook.Reset()
+	res, err := client.Do(req)
+	helper.Must(err)
+
+	if res.Header.Get("X-From-Request-Header") != "bar" ||
+		res.Header.Get("X-From-Request-Json-Body") != "1" ||
+		res.Header.Get("X-From-Requests-Header") != "bar" ||
+		res.Header.Get("X-From-Requests-Json-Body") != "1" ||
+		// res.Header.Get("X-From-Response-Json-Body") != "/anything" || // not yet
+		// res.Header.Get("X-From-Responses-Json-Body") != "/anything" || // not yet
+		res.Header.Get("X-From-Response-Header") != "application/json" ||
+		res.Header.Get("X-From-Responses-Header") != "application/json" {
+		t.Errorf("Unexpected header given: %#v", res.Header)
+	}
+
+	for _, entry := range hook.AllEntries() {
+		if entry.Data["type"] != "couper_backend" {
+			continue
+		}
+
+		responseHeaders := entry.Data["response"].(logging.Fields)["headers"].(map[string]string)
+		data := entry.Data["custom"].(logrus.Fields)
+
+		url := entry.Data["url"]
+		if url == "http://localhost:8081/token" {
+			if data["x-from-request-body"] != "grant_type=client_credentials" ||
+				// data["x-from-request-form-body"] != "client_credentials" || // not yet
+				data["x-from-request-header"] != "Basic cXBlYjpiZW4=" ||
+				data["x-from-response-header"] != "60s" ||
+				data["x-from-response-body"] != `{"access_token":"the_access_token","expires_in":60}` ||
+				data["x-from-response-json-body"] != "the_access_token" ||
+				responseHeaders["location"] != "Basic cXBlYjpiZW4=||60s|" {
+				// responseHeaders["location"] != "Basic cXBlYjpiZW4=|client_credentials|60s|the_access_token" { // not yet
+				t.Errorf("Unexpected logs given: %#v", data)
+			}
+		} else {
+			if data["x-from-request-json-body"] != float64(1) ||
+				data["x-from-request-header"] != "bar" ||
+				data["x-from-requests-json-body"] != float64(1) ||
+				data["x-from-requests-header"] != "bar" ||
+				data["x-from-response-header"] != "application/json" ||
+				data["x-from-response-json-body"] != "/anything" ||
+				data["x-from-responses-header"] != "application/json" ||
+				data["x-from-responses-json-body"] != "/anything" {
+				t.Errorf("Unexpected logs given: %#v", data)
+			}
+		}
+	}
+}
+
+func TestBackend_BackendVariable(t *testing.T) {
+	client := newClient()
+	helper := test.New(t)
+
+	shutdown, hook := newCouper("testdata/integration/backend/01_couper.hcl", helper)
+	defer shutdown()
+
+	req, err := http.NewRequest(http.MethodGet, "http://example.com:8080/", nil)
+	helper.Must(err)
+
+	req.Header.Set("Cookie", "Cookie")
+	req.Header.Set("User-Agent", "Couper")
+
+	hook.Reset()
+	res, err := client.Do(req)
+	helper.Must(err)
+
+	var check int
+
+	for _, entry := range hook.AllEntries() {
+		if entry.Data["type"] != "couper_backend" {
+			continue
+		}
+
+		name := entry.Data["request"].(logging.Fields)["name"]
+		data := entry.Data["custom"].(logrus.Fields)
+		// The Cookie request header is not proxied, so *-req is not set in log.
+
+		if name == "default" {
+			check++
+
+			if len(data) != 2 || data["default-res"] != "application/json" || data["default-ua"] != "Couper" {
+				t.Errorf("unexpected data given: %#v", data)
+			}
+		} else if name == "request" {
+			check++
+
+			if len(data) != 2 || data["request-res"] != "text/plain; charset=utf-8" || data["request-ua"] != "" {
+				t.Errorf("unexpected data given: %#v", data)
+			}
+		} else if name == "r1" {
+			check++
+
+			if len(data) != 2 || data["definitions-res"] != "text/plain; charset=utf-8" || data["definitions-ua"] != "" {
+				t.Errorf("unexpected data given: %#v", data)
+			}
+		} else if name == "r2" {
+			check++
+
+			if len(data) != 2 || data["definitions-res"] != "application/json" || data["definitions-ua"] != "" {
+				t.Errorf("unexpected data given: %#v", data)
+			}
+		}
+	}
+
+	if check != 4 {
+		t.Error("missing 4 backend logs")
+	}
+
+	if got := res.Header.Get("Test-Header"); got != "application/json" {
+		t.Errorf("Unexpected header given: %#v", got)
+	}
+}
+
 func TestEndpoints_Protected404(t *testing.T) {
 	client := newClient()