Add additional variables validation on startup #59

With fallback to raw string expression

config/gateway.go

@@ -3,6 +3,7 @@ package config
 import "github.com/hashicorp/hcl/v2"
 
 type Gateway struct {
+	Bytes       []byte           `hcl:"-"`
 	Context     *hcl.EvalContext `hcl:"-"`
 	Definitions *Definitions     `hcl:"definitions,block"`
 	Server      []*Server        `hcl:"server,block"`

config/load.go

@@ -26,6 +26,7 @@ func LoadFile(filePath string) (*Gateway, error) {
 
 func LoadBytes(src []byte, filePath string) (*Gateway, error) {
 	config := &Gateway{
+		Bytes:    src[:],
 		Context:  eval.NewENVContext(src),
 		Settings: &Settings{DefaultPort: DefaultListenPort},
 	}

config/runtime/server.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"strconv"
@@ -19,6 +20,7 @@ import (
 	"github.com/avenga/couper/config"
 	"github.com/avenga/couper/config/runtime/server"
 	"github.com/avenga/couper/errors"
+	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/handler"
 	"github.com/avenga/couper/internal/seetie"
 	"github.com/avenga/couper/utils"
@@ -75,17 +77,23 @@ func NewServerConfiguration(conf *config.Gateway, httpConf *HTTPConfig, log *log
 		defaultPort = httpConf.ListenPort
 	}
 
+	// confCtx is created to evaluate request / response related configuration errors on start.
+	noopReq := httptest.NewRequest(http.MethodGet, "https://couper.io", nil)
+	noopResp := httptest.NewRecorder().Result()
+	noopResp.Request = noopReq
+	confCtx := eval.NewHTTPContext(conf.Context, 0, noopReq, noopReq, noopResp)
+
 	validPortMap, hostsMap, err := validatePortHosts(conf, defaultPort)
 	if err != nil {
 		return nil, err
 	}
 
-	backends, err := newBackendsFromDefinitions(conf, log)
+	backends, err := newBackendsFromDefinitions(conf, confCtx, log)
 	if err != nil {
 		return nil, err
 	}
 
-	accessControls, err := configureAccessControls(conf)
+	accessControls, err := configureAccessControls(conf, confCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +160,7 @@ func NewServerConfiguration(conf *config.Gateway, httpConf *HTTPConfig, log *log
 				}
 				endpoints[cleanPattern] = true
 
-				if err := validateInlineScheme(conf.Context, endpoint.InlineDefinition, endpoint); err != nil {
+				if err := validateInlineScheme(confCtx, endpoint.InlineDefinition, endpoint); err != nil {
 					return nil, err
 				}
 
@@ -174,7 +182,7 @@ func NewServerConfiguration(conf *config.Gateway, httpConf *HTTPConfig, log *log
 					// set server context for defined backends
 					be := backends[endpoint.Backend]
 					_, remain := be.conf.Merge(&config.Backend{Options: endpoint.InlineDefinition})
-					refBackend := newProxy(conf.Context, be.conf, srvConf.API.CORS, remain, log, serverOptions)
+					refBackend := newProxy(confCtx, be.conf, srvConf.API.CORS, remain, log, serverOptions)
 
 					setACHandlerFn(refBackend)
 					err = setRoutesFromHosts(serverConfiguration, defaultPort, srvConf.Hosts, pattern, api[endpoint], KindAPI)
@@ -185,7 +193,7 @@ func NewServerConfiguration(conf *config.Gateway, httpConf *HTTPConfig, log *log
 				}
 
 				// otherwise try to parse an inline block and fallback for api reference or inline block
-				inlineBackend, inlineConf, err := newInlineBackend(conf.Context, backends, endpoint.InlineDefinition, srvConf.API.CORS, log, serverOptions)
+				inlineBackend, inlineConf, err := newInlineBackend(confCtx, backends, endpoint.InlineDefinition, srvConf.API.CORS, log, serverOptions)
 				if err == errorMissingBackend {
 					if srvConf.API.Backend != "" {
 						if _, ok := backends[srvConf.API.Backend]; !ok {
@@ -198,20 +206,20 @@ func NewServerConfiguration(conf *config.Gateway, httpConf *HTTPConfig, log *log
 						}
 						continue
 					}
-					inlineBackend, inlineConf, err = newInlineBackend(conf.Context, backends, srvConf.API.InlineDefinition, srvConf.API.CORS, log, serverOptions)
+					inlineBackend, inlineConf, err = newInlineBackend(confCtx, backends, srvConf.API.InlineDefinition, srvConf.API.CORS, log, serverOptions)
 					if err != nil {
 						return nil, err
 					}
 
-					if inlineConf.Name == "" && getAttribute(conf.Context, "origin", inlineConf.Options) == "" {
+					if inlineConf.Name == "" && getAttribute(confCtx, "origin", inlineConf.Options, conf.Bytes) == "" {
 						return nil, fmt.Errorf("api inline backend requires an origin attribute: %q", pattern)
 					}
 				} else if err != nil { // TODO hcl.diagnostics error
 					return nil, fmt.Errorf("range: %s: %v", endpoint.InlineDefinition.MissingItemRange().String(), err)
 				}
 
 				if e := validateOrigin(
-					getAttribute(conf.Context, "origin", inlineConf.Options),
+					getAttribute(confCtx, "origin", inlineConf.Options, conf.Bytes),
 					inlineConf.Options.MissingItemRange()); e != nil {
 					return nil, e
 				}
@@ -245,7 +253,7 @@ func newProxy(ctx *hcl.EvalContext, beConf *config.Backend, corsOpts *config.COR
 	return proxy
 }
 
-func newBackendsFromDefinitions(conf *config.Gateway, log *logrus.Entry) (map[string]backendDefinition, error) {
+func newBackendsFromDefinitions(conf *config.Gateway, confCtx *hcl.EvalContext, log *logrus.Entry) (map[string]backendDefinition, error) {
 	backends := make(map[string]backendDefinition)
 
 	if conf.Definitions == nil {
@@ -257,7 +265,7 @@ func newBackendsFromDefinitions(conf *config.Gateway, log *logrus.Entry) (map[st
 			return nil, fmt.Errorf("backend name must be unique: %q", beConf.Name)
 		}
 
-		origin := getAttribute(conf.Context, "origin", beConf.Options)
+		origin := getAttribute(confCtx, "origin", beConf.Options, conf.Bytes)
 		if e := validateOrigin(origin, beConf.Options.MissingItemRange()); e != nil {
 			return nil, e
 		}
@@ -267,21 +275,27 @@ func newBackendsFromDefinitions(conf *config.Gateway, log *logrus.Entry) (map[st
 		srvOpts, _ := server.NewServerOptions(&config.Server{})
 		backends[beConf.Name] = backendDefinition{
 			conf:    beConf,
-			handler: newProxy(conf.Context, beConf, nil, []hcl.Body{beConf.Options}, log, srvOpts),
+			handler: newProxy(confCtx, beConf, nil, []hcl.Body{beConf.Options}, log, srvOpts),
 		}
 	}
 	return backends, nil
 }
 
 // hasAttribute checks for a configured string value and ignores unrelated errors.
-func getAttribute(ctx *hcl.EvalContext, name string, body hcl.Body) string {
+func getAttribute(ctx *hcl.EvalContext, name string, body hcl.Body, configBytes []byte) string {
 	attr, _ := body.JustAttributes()
 
 	if _, ok := attr[name]; !ok {
 		return ""
 	}
 
-	val, _ := attr[name].Expr.Value(ctx)
+	val, diags := attr[name].Expr.Value(ctx)
+	if diags.HasErrors() && attr[name].Expr.Range().CanSliceBytes(configBytes) { // fallback to origin string
+		rawString := attr[name].Expr.Range().SliceBytes(configBytes)
+		if len(rawString) > 2 { // more then quotes
+			return string(attr[name].Expr.Range().SliceBytes(configBytes)[1 : len(rawString)-1]) //unquote
+		}
+	}
 	return seetie.ValueToString(val)
 }
 
@@ -310,7 +324,7 @@ func splitWildcardHostPort(host string, configuredPort int) (string, Port, error
 	return ho, Port(po), nil
 }
 
-func configureAccessControls(conf *config.Gateway) (ac.Map, error) {
+func configureAccessControls(conf *config.Gateway, confCtx *hcl.EvalContext) (ac.Map, error) {
 	accessControls := make(ac.Map)
 
 	if conf.Definitions != nil {
@@ -360,7 +374,7 @@ func configureAccessControls(conf *config.Gateway) (ac.Map, error) {
 
 			var claims ac.Claims
 			if jwt.Claims != nil {
-				c, diags := seetie.ExpToMap(conf.Context, jwt.Claims)
+				c, diags := seetie.ExpToMap(confCtx, jwt.Claims)
 				if diags.HasErrors() {
 					return nil, diags
 				}

config/runtime/server_validation.go

@@ -83,10 +83,15 @@ func validateOrigin(origin string, ctxRange hcl.Range) error {
 		Subject: &ctxRange,
 		Summary: "invalid backend.origin value",
 	}
+
 	if origin == "" {
 		diagErr.Detail = "origin attribute is required"
 		return hcl.Diagnostics{diagErr}
 	}
+
+	// if origin contains fallback content with variables
+	origin = strings.ReplaceAll(strings.ReplaceAll(origin, "}", ""), "${", "")
+
 	u, err := url.Parse(origin)
 	if err != nil {
 		diagErr.Detail = fmt.Sprintf("url parse error: %v", err)

handler/proxy_test.go

@@ -11,8 +11,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/avenga/couper/internal/seetie"
-
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsimple"
 	"github.com/sirupsen/logrus"
@@ -22,6 +20,7 @@ import (
 	"github.com/avenga/couper/errors"
 	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/handler"
+	"github.com/avenga/couper/internal/seetie"
 	"github.com/avenga/couper/internal/test"
 )
 

internal/test/test_backend.go

@@ -43,6 +43,8 @@ func anything(rw http.ResponseWriter, req *http.Request) {
 	type anything struct {
 		Args                               url.Values
 		Headers                            http.Header
+		Host                               string
+		Path                               string
 		Method, RemoteAddr, Url, UserAgent string
 	}
 
@@ -51,7 +53,9 @@ func anything(rw http.ResponseWriter, req *http.Request) {
 	resp := &anything{
 		Args:       req.Form,
 		Headers:    req.Header.Clone(),
+		Host:       req.Host,
 		Method:     req.Method,
+		Path:       req.URL.Path,
 		RemoteAddr: req.RemoteAddr,
 		Url:        req.URL.String(),
 		UserAgent:  req.UserAgent(),

server/http_integration_test.go

@@ -3,6 +3,7 @@ package server_test
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -109,6 +110,7 @@ func newClient() *http.Client {
 	dialer := &net.Dialer{}
 	return &http.Client{
 		Transport: &http.Transport{
+			DisableCompression: true,
 			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 				_, port, _ := net.SplitHostPort(addr)
 				if port != "" {
@@ -455,3 +457,61 @@ func TestHTTPServer_XFHHeader(t *testing.T) {
 
 	cleanup(shutdown, t)
 }
+
+func TestHTTPServer_Endpoint_Evaluation(t *testing.T) {
+	client := newClient()
+
+	confPath := path.Join("testdata/integration/endpoint_eval/01_couper.hcl")
+	shutdown, _ := newCouper(confPath, test.New(t))
+
+	type expectation struct {
+		Host, Origin, Path string
+	}
+
+	type testCase struct {
+		reqPath string
+		exp     expectation
+	}
+
+	for _, tc := range []testCase{
+		{"/my-waffik/my.host.de/" + testBackend.Addr()[7:], expectation{
+			Host:   "my.host.de",
+			Origin: testBackend.Addr()[7:],
+			Path:   "/anything",
+		}},
+		{"/my-respo/my.host.com/" + testBackend.Addr()[7:], expectation{
+			Host:   "my.host.com",
+			Origin: testBackend.Addr()[7:],
+			Path:   "/anything",
+		}},
+	} {
+		t.Run("_"+tc.reqPath, func(subT *testing.T) {
+			helper := test.New(subT)
+
+			req, err := http.NewRequest(http.MethodGet, "http://example.com:8080"+tc.reqPath, nil)
+			helper.Must(err)
+
+			res, err := client.Do(req)
+			helper.Must(err)
+
+			resBytes, err := ioutil.ReadAll(res.Body)
+			helper.Must(err)
+
+			_ = res.Body.Close()
+
+			var jsonResult expectation
+			err = json.Unmarshal(resBytes, &jsonResult)
+			if err != nil {
+				t.Errorf("unmarshal json: %v: got:\n%s", err, string(resBytes))
+			}
+
+			jsonResult.Origin = res.Header.Get("X-Origin")
+
+			if !reflect.DeepEqual(jsonResult, tc.exp) {
+				t.Errorf("want: %#v, got: %#v, payload:\n%s", tc.exp, jsonResult, string(resBytes))
+			}
+		})
+	}
+
+	cleanup(shutdown, t)
+}

server/http_test.go

@@ -152,7 +152,7 @@ func TestHTTPServer_ServeHTTP_Files2(t *testing.T) {
 	httpConf := runtime.NewHTTPConfig(nil)
 	httpConf.ListenPort = 0 // random
 
-	conf, err := config.LoadBytes(confBytes.Bytes(), "conf_fileserving")
+	conf, err := config.LoadBytes(confBytes.Bytes(), "conf_fileserving.hcl")
 	helper.Must(err)
 
 	error404Content := []byte("<html><body><h1>3001: Files route not found: My custom error template</h1></body></html>")

server/testdata/integration/endpoint_eval/01_couper.hcl

@@ -0,0 +1,27 @@
+server "api" {
+  error_file = "./../server_error.html"
+
+  api {
+    error_file = "./../api_error.json"
+
+    endpoint "/{path}/{hostname}/{origin}" {
+      path = "/unset/by/backend"
+      backend {
+        path = "/anything"
+        origin = "http://${req.path_param.origin}"
+        hostname = req.path_param.hostname
+        response_headers = {
+          x-origin = req.path_param.origin
+        }
+      }
+    }
+  }
+}
+
+definitions {
+  # backend origin within a definition block gets replaced with the integration test "anything" server.
+  backend "anything" {
+    path = "/anything"
+    origin = "http://anyserver/"
+  }
+}