Allow evaluation for the defaults block (#630)

* merge without value eval

* evaluate twice and simplify code

* fixup test; method signature

also missing bodiesToConfig call for LoadBytes

* fixup test; use loadBytes

* fix context test, pass env

* handle different key expression types

* restrict ScopeTraversalExpr to identifier

* Documented key restrictions

* fixed  TestHTTPServer_ServeHTTP_Files2

* Fixup explicit expectation content

* test cases for key/value type errors

Co-authored-by: Johannes Koch <johannes.koch@avenga.com>

CHANGELOG.md

@@ -14,6 +14,7 @@ Unreleased changes are available as `avenga/couper:edge` container.
 * **Changed**
   * Replaced the JWT library because the former library was no longer maintained ([#612](https://github.com/avenga/couper/pull/612))
   * Routing and [OpenAPI validation](https://docs.couper.io/configuration/block/openapi) now use gorilla/mux ([#614](https://github.com/avenga/couper/pull/614))
+  * Usage of `env` variables and functions is now possible for the `defaults` block ([#630](https://github.com/avenga/couper/pull/630))
 
 * **Fixed**
   * Aligned the evaluation of [`beta_oauth2`](https://docs.couper.io/configuration/block/oauth2_ac)/[`oidc`](https://docs.couper.io/configuration/block/oidc) `redirect_uri` to `saml` `sp_acs_url` ([#589](https://github.com/avenga/couper/pull/589))

config/configload/helper.go

@@ -12,7 +12,6 @@ import (
 	hclbody "github.com/avenga/couper/config/body"
 	"github.com/avenga/couper/config/sequence"
 	"github.com/avenga/couper/errors"
-	"github.com/avenga/couper/eval"
 )
 
 type helper struct {
@@ -23,16 +22,11 @@ type helper struct {
 }
 
 // newHelper creates a container with some methods to keep things simple here and there.
-func newHelper(body hcl.Body, src [][]byte, environment string) (*helper, error) {
-	defaultsBlock := &config.DefaultsBlock{}
-	if diags := gohcl.DecodeBody(body, nil, defaultsBlock); diags.HasErrors() {
-		return nil, diags
-	}
-
+func newHelper(body hcl.Body) (*helper, error) {
 	couperConfig := &config.Couper{
-		Context:     eval.NewContext(src, defaultsBlock.Defaults, environment),
+		Context:     evalContext,
 		Definitions: &config.Definitions{},
-		Defaults:    defaultsBlock.Defaults,
+		Defaults:    defaultsConfig,
 		Settings:    config.NewDefaultSettings(),
 	}
 
@@ -45,7 +39,7 @@ func newHelper(body hcl.Body, src [][]byte, environment string) (*helper, error)
 	return &helper{
 		config:       couperConfig,
 		content:      content,
-		context:      couperConfig.Context.(*eval.Context).HCLContext(),
+		context:      evalContext.HCLContext(),
 		defsBackends: make(map[string]*hclsyntax.Body),
 	}, nil
 }

config/configload/load.go

@@ -75,14 +75,16 @@ func init() {
 
 func updateContext(body hcl.Body, srcBytes [][]byte, environment string) hcl.Diagnostics {
 	defaultsBlock := &config.DefaultsBlock{}
-	if diags := gohcl.DecodeBody(body, nil, defaultsBlock); diags.HasErrors() {
+	// defaultsCtx is a temporary one to allow env variables and functions for defaults {}
+	defaultsCtx := eval.NewContext(srcBytes, nil, environment).HCLContext()
+	if diags := gohcl.DecodeBody(body, defaultsCtx, defaultsBlock); diags.HasErrors() {
 		return diags
 	}
+	defaultsConfig = defaultsBlock.Defaults // global assign
 
 	// We need the "envContext" to be able to resolve absolute paths in the config.
-	defaultsConfig = defaultsBlock.Defaults
 	evalContext = eval.NewContext(srcBytes, defaultsConfig, environment)
-	envContext = evalContext.HCLContext()
+	envContext = evalContext.HCLContext() // global assign
 
 	return nil
 }
@@ -179,7 +181,7 @@ func bodiesToConfig(parsedBodies []*hclsyntax.Body, srcBytes [][]byte, env strin
 		return nil, err
 	}
 
-	conf, err := LoadConfig(configBody, srcBytes, env)
+	conf, err := LoadConfig(configBody)
 	if err != nil {
 		return nil, err
 	}
@@ -206,12 +208,12 @@ func LoadFiles(filesList []string, env string) (*config.Couper, error) {
 
 	if env == "" {
 		settingsBlock := mergeSettings(parsedBodies)
-		settings := &config.Settings{}
-		if diags := gohcl.DecodeBody(settingsBlock.Body, nil, settings); diags.HasErrors() {
+		confSettings := &config.Settings{}
+		if diags := gohcl.DecodeBody(settingsBlock.Body, nil, confSettings); diags.HasErrors() {
 			return nil, diags
 		}
-		if settings.Environment != "" {
-			return LoadFiles(filesList, settings.Environment)
+		if confSettings.Environment != "" {
+			return LoadFiles(filesList, confSettings.Environment)
 		}
 	}
 
@@ -257,6 +259,10 @@ func loadTestContents(tcs []testContent) (*config.Couper, error) {
 }
 
 func LoadBytes(src []byte, filename string) (*config.Couper, error) {
+	return LoadBytesEnv(src, filename, "")
+}
+
+func LoadBytesEnv(src []byte, filename, env string) (*config.Couper, error) {
 	hclBody, err := parser.Load(src, filename)
 	if err != nil {
 		return nil, err
@@ -266,17 +272,17 @@ func LoadBytes(src []byte, filename string) (*config.Couper, error) {
 		return nil, err
 	}
 
-	return LoadConfig(hclBody, [][]byte{src}, "")
+	return bodiesToConfig([]*hclsyntax.Body{hclBody}, [][]byte{src}, env)
 }
 
-func LoadConfig(body *hclsyntax.Body, src [][]byte, environment string) (*config.Couper, error) {
+func LoadConfig(body *hclsyntax.Body) (*config.Couper, error) {
 	var err error
 
 	if diags := ValidateConfigSchema(body, &config.Couper{}); diags.HasErrors() {
 		return nil, diags
 	}
 
-	helper, err := newHelper(body, src, environment)
+	helper, err := newHelper(body)
 	if err != nil {
 		return nil, err
 	}

config/configload/load_test.go

@@ -616,6 +616,44 @@ func TestConfigErrors(t *testing.T) {
 			}`,
 			`configuration error: referenced backend "as" is not defined`,
 		},
+		{
+			"wrong environment_variables type",
+			`server {}
+			defaults {
+			  environment_variables = "val"
+			}`,
+			"couper.hcl:3,30-35: environment_variables must be object type; ",
+		},
+		{
+			"unsupported key scope traversal expression",
+			`server {}
+			defaults {
+			  environment_variables = {
+			    env.FOO = "val"
+			  }
+			}`,
+			"couper.hcl:4,8-15: unsupported key scope traversal expression; ",
+		},
+		{
+			"unsupported key template expression",
+			`server {}
+			defaults {
+			  environment_variables = {
+			    "key${1 + 0}" = "val"
+			  }
+			}`,
+			"couper.hcl:4,8-21: unsupported key template expression; ",
+		},
+		{
+			"unsupported key expression",
+			`server {}
+			defaults {
+			  environment_variables = {
+			    to_upper("key") = "val"
+			  }
+			}`,
+			"couper.hcl:4,8-23: unsupported key expression; ",
+		},
 	}
 
 	for _, tt := range tests {

config/configload/merge.go

@@ -563,48 +563,82 @@ func mergeDefinitions(bodies []*hclsyntax.Body) (*hclsyntax.Block, map[string]*h
 
 func mergeDefaults(bodies []*hclsyntax.Body) (*hclsyntax.Block, error) {
 	attrs := make(hclsyntax.Attributes)
-	envVars := make(map[string]cty.Value)
+	envVars := make(map[string]hclsyntax.Expression)
 
 	for _, body := range bodies {
 		for _, block := range body.Blocks {
-			if block.Type == defaults {
-				for name, attr := range block.Body.Attributes {
-					if name == environmentVars {
-						v, err := eval.Value(nil, attr.Expr)
-						if err != nil {
-							return nil, err
-						}
+			if block.Type != defaults {
+				continue
+			}
 
-						for k, value := range v.AsValueMap() {
-							if value.Type() != cty.String {
-								return nil, fmt.Errorf("value in 'environment_variables' is not a string")
-							}
+			for name, attr := range block.Body.Attributes {
+				if name != environmentVars {
+					attrs[name] = attr // Currently not used
+					continue
+				}
+
+				expObj, ok := attr.Expr.(*hclsyntax.ObjectConsExpr)
+				if !ok {
+					r := attr.Expr.Range()
+					return nil, newDiagErr(&r, fmt.Sprintf("%s must be object type", environmentVars))
+				}
 
-							envVars[k] = value
+				for _, item := range expObj.Items {
+					k := item.KeyExpr.(*hclsyntax.ObjectConsKeyExpr)
+					r := item.KeyExpr.Range()
+					var keyName string
+					switch exp := k.Wrapped.(type) {
+					case *hclsyntax.ScopeTraversalExpr:
+						if len(exp.Traversal) > 1 {
+							return nil, newDiagErr(&r, "unsupported key scope traversal expression")
 						}
-					} else {
-						attrs[name] = attr // Currently not used
+						keyName = exp.Traversal.RootName()
+					case *hclsyntax.TemplateExpr:
+						if !exp.IsStringLiteral() {
+							return nil, newDiagErr(&r, "unsupported key template expression")
+						}
+						v, _ := exp.Value(nil)
+						keyName = v.AsString()
+					default:
+						r := item.KeyExpr.Range()
+						return nil, newDiagErr(&r, "unsupported key expression")
 					}
+
+					envVars[keyName] = item.ValueExpr
 				}
 			}
 		}
 	}
 
 	if len(envVars) > 0 {
+		items := make([]hclsyntax.ObjectConsItem, 0)
+		for k, v := range envVars {
+			items = append(items, hclsyntax.ObjectConsItem{
+				KeyExpr: &hclsyntax.ObjectConsKeyExpr{
+					Wrapped: &hclsyntax.ScopeTraversalExpr{
+						Traversal: hcl.Traversal{hcl.TraverseRoot{Name: k}},
+					},
+					ForceNonLiteral: false,
+				},
+				ValueExpr: v,
+			})
+		}
+
 		attrs[environmentVars] = &hclsyntax.Attribute{
 			Name: environmentVars,
-			Expr: &hclsyntax.LiteralValueExpr{
-				Val: cty.MapVal(envVars),
+			Expr: &hclsyntax.ObjectConsExpr{
+				Items: items,
 			},
 		}
 	}
 
-	return &hclsyntax.Block{
+	defaultsBlock := &hclsyntax.Block{
 		Type: defaults,
 		Body: &hclsyntax.Body{
 			Attributes: attrs,
 		},
-	}, nil
+	}
+	return defaultsBlock, nil
 }
 
 func mergeSettings(bodies []*hclsyntax.Body) *hclsyntax.Block {

config/defaults.go

@@ -5,7 +5,7 @@ import "github.com/hashicorp/hcl/v2"
 type DefaultEnvVars map[string]string
 
 type Defaults struct {
-	EnvironmentVariables DefaultEnvVars `hcl:"environment_variables,optional" docs:"One or more environment variable assignments"`
+	EnvironmentVariables DefaultEnvVars `hcl:"environment_variables,optional" docs:"One or more environment variable assignments. Keys must be either identifiers or simple string expressions."`
 }
 
 type DefaultsBlock struct {

docs/website/content/2.configuration/4.block/defaults.md

@@ -12,7 +12,7 @@ The `defaults` block lets you define default values.
 values: [
   {
     "default": "",
-    "description": "One or more environment variable assignments",
+    "description": "One or more environment variable assignments. Keys must be either identifiers or simple string expressions.",
     "name": "environment_variables",
     "type": "object"
   }

eval/context_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/avenga/couper/config/configload"
-	"github.com/avenga/couper/config/parser"
 	"github.com/avenga/couper/config/request"
 	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/internal/seetie"
@@ -191,7 +190,8 @@ func TestDefaultEnvVariables(t *testing.T) {
 		t.Run(tt.name, func(subT *testing.T) {
 			cf, err := configload.LoadBytes([]byte(tt.hcl), "couper.hcl")
 			if err != nil {
-				subT.Fatal(err)
+				subT.Error(err)
+				return
 			}
 
 			hclContext := cf.Context.(*eval.Context).HCLContext()
@@ -236,14 +236,7 @@ func TestCouperVariables(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(subT *testing.T) {
-			bytes := []byte(tt.hcl)
-			hclBody, err := parser.Load(bytes, "couper.hcl")
-			if err != nil {
-				subT.Error(err)
-				return
-			}
-
-			cf, err := configload.LoadConfig(hclBody, [][]byte{bytes}, tt.env)
+			cf, err := configload.LoadBytesEnv([]byte(tt.hcl), "couper.hcl", tt.env)
 			if err != nil {
 				subT.Error(err)
 				return

server/http_integration_test.go

@@ -518,6 +518,41 @@ func TestHTTPServer_EnvVars(t *testing.T) {
 	}
 }
 
+func TestHTTPServer_DefaultEnvVars(t *testing.T) {
+	helper := test.New(t)
+	client := newClient()
+
+	env.SetTestOsEnviron(func() []string {
+		return []string{"VALUE_4=value4"}
+	})
+	defer env.SetTestOsEnviron(os.Environ)
+
+	shutdown, hook := newCouper("testdata/integration/env/02_couper.hcl", test.New(t))
+	defer shutdown()
+
+	hook.Reset()
+
+	req, err := http.NewRequest(http.MethodGet, "http://example.com:8080", nil)
+	helper.Must(err)
+
+	res, err := client.Do(req)
+	helper.Must(err)
+
+	if res.StatusCode != http.StatusOK {
+		t.Errorf("expected 200, got %d", res.StatusCode)
+	}
+
+	b, err := io.ReadAll(res.Body)
+	helper.Must(err)
+
+	var result []string
+	helper.Must(json.Unmarshal(b, &result))
+
+	if diff := cmp.Diff(result, []string{"value1", "", "default_value_3", "value4", "value5"}); diff != "" {
+		t.Error(diff)
+	}
+}
+
 func TestHTTPServer_XFHHeader(t *testing.T) {
 	client := newClient()
 

server/http_test.go

@@ -176,8 +176,7 @@ func TestHTTPServer_ServeHTTP_Files2(t *testing.T) {
 	helper.Must(err)
 
 	error404Content := []byte("<html><body><h1>route not found error: My custom error template</h1></body></html>")
-	spaContent, err := os.ReadFile(conf.Servers[0].SPAs[0].BootstrapFile)
-	helper.Must(err)
+	spaContent := []byte("<html><body><h1>vue.js</h1></body></html>")
 
 	tmpStoreCh := make(chan struct{})
 	defer close(tmpStoreCh)

server/testdata/integration/env/02_couper.hcl

@@ -0,0 +1,17 @@
+server {
+  endpoint "/" {
+    response {
+      json_body = [env.KEY1, env.KEY2, env.KEY3, env.KEY4, env.KEY5]
+    }
+  }
+}
+
+defaults {
+  environment_variables = {
+    KEY1 = "value1"
+    KEY2 = env.VALUE_2
+    KEY3 = default(env.VALUE_3, "default_value_3")
+    KEY4 = env.VALUE_4
+    "KEY5" = "value5"
+  }
+}