diff --git a/go.mod b/go.mod index 617f58c4c..f28005d54 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/coreos/go-oidc/v3 v3.11.0 github.com/cyphar/filepath-securejoin v0.3.4 github.com/evanphx/json-patch/v5 v5.9.0 + github.com/expr-lang/expr v1.16.9 github.com/fatih/structtag v1.2.0 github.com/fluxcd/pkg/kustomize v1.13.0 github.com/go-git/go-git/v5 v5.12.0 @@ -39,6 +40,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b + github.com/valyala/fasttemplate v1.2.2 github.com/xeipuuv/gojsonschema v1.2.0 go.uber.org/ratelimit v0.3.1 golang.org/x/crypto v0.28.0 @@ -118,6 +120,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.7.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect diff --git a/go.sum b/go.sum index f41a36fff..bd11ca81f 100644 --- a/go.sum +++ b/go.sum @@ -164,6 +164,8 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= +github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= @@ -503,6 +505,10 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b h1:fo0GUa0B+vxSZ8bgnL3fpCPHReM/QPlALdak9T/Zw5Y= github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/expressions/json_templates.go b/internal/expressions/json_templates.go new file mode 100644 index 000000000..125f4122e --- /dev/null +++ b/internal/expressions/json_templates.go @@ -0,0 +1,168 @@ +package expressions + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "maps" + "strconv" + "strings" + + "github.com/expr-lang/expr" + "github.com/valyala/fasttemplate" +) + +// EvaluateJSONTemplate evaluates a JSON byte slice, which is presumed to be a +// template containing expr-lang expressions offset by ${{ and }}, using the +// provided environment as context. The evaluated JSON is returned as a new byte +// slice, ready for unmarshaling. +// +// Only expressions contained within values are evaluated. i.e. Any expressions +// within keys are NOT evaluated. +// +// Since the template itself must be valid JSON, all expressions MUST be +// enclosed in quotes. +// +// If, after evaluating all expressions in a single value (multiples are +// permitted), the result can be parsed as a bool, float64, or other valid +// non-string JSON, it will be treated as such. This ensures the possibility of +// expressions being used to construct any valid JSON value, despite the fact +// that expressions must, themselves, be contained within a string value. This +// does mean that for expressions which may evaluate as something resembling a +// valid non-string JSON value, the user must take care to ensure that the +// expression evaluates to a string enclosed in quotes. e.g. ${{ true }} will +// evaluated as a bool, but ${{ quote(true) }} will be evaluated as a string. +// This behavior should be intuitive to anyone familiar with YAML. +func EvaluateJSONTemplate(jsonBytes []byte, env map[string]any) ([]byte, error) { + if _, ok := env["quote"]; ok { + return nil, fmt.Errorf( + `"quote" is a forbidden key in the environment map; it is reserved for internal use`, + ) + } + env = maps.Clone(env) // We don't want to add the quote function to the user's map. + env["quote"] = func(a any) string { return fmt.Sprintf(`"%v"`, a) } + var parsed map[string]any + if err := json.Unmarshal(jsonBytes, &parsed); err != nil { + return nil, + fmt.Errorf("input is not valid JSON; are all expressions enclosed in quotes? %w", err) + } + if err := evaluateExpressions(parsed, env); err != nil { + return nil, err + } + return json.Marshal(parsed) +} + +// evaluateExpressions recursively evaluates all expressions contained within +// elements of a map[string]any or []any, updating those elements in place. +// Passing any other type to this function will have no effect. Expressions are +// evaluated using the provided environment map as context. +func evaluateExpressions(collection any, env map[string]any) error { + switch col := collection.(type) { + case map[string]any: + for key, val := range col { + switch v := val.(type) { + case map[string]any: + if err := evaluateExpressions(v, env); err != nil { + return err + } + case []any: + if err := evaluateExpressions(v, env); err != nil { + return err + } + case string: + var err error + if col[key], err = evaluateTemplate(v, env); err != nil { + return err + } + } + } + case []any: + for i, val := range col { + switch v := val.(type) { + case map[string]any: + if err := evaluateExpressions(v, env); err != nil { + return err + } + case []any: + if err := evaluateExpressions(v, env); err != nil { + return err + } + case string: + var err error + if col[i], err = evaluateTemplate(v, env); err != nil { + return err + } + } + } + } + return nil +} + +// evaluateTemplate evaluates a single template string with the provided +// environment. Note that a single template string can contain multiple +// expressions. +func evaluateTemplate(template string, env map[string]any) (any, error) { + t := fasttemplate.New(template, "${{", "}}") + out := &bytes.Buffer{} + if _, err := t.ExecuteFunc(out, getExpressionEvaluator(env)); err != nil { + return nil, err + } + result := out.String() + // If the result is enclosed in quotes, this is probably the result of an + // expression that deliberately enclosed the result in quotes to prevent it + // from being mistaken for a number, bool, etc. e.g. ${{ quote(true) }} + // instead of ${{ true }}. Strip the quotes and make no attempt to parse the + // result as any other type. + // + // Note: There's an edge case where this is NOT the reason for the leading and + // trailing quotes, but the likelihood of this occurring in the context in + // which we are using this function is so low that it's not worth sacrificing + // the convenience of this behavior. + if len(result) > 1 && strings.HasPrefix(result, `"`) && strings.HasSuffix(result, `"`) { + return result[1 : len(result)-1], nil + } + // If the result is parseable as a bool return that. + if resBool, err := strconv.ParseBool(result); err == nil { + return resBool, nil + } + // If the result is parseable as a float64, return that. float64 is used + // because it can represent all JSON numbers. + if resNum, err := strconv.ParseFloat(result, 64); err == nil { + return resNum, nil + } + // If the result is valid JSON, return its unmarshaled value. + var resMap any + if err := json.Unmarshal([]byte(result), &resMap); err == nil { + return resMap, nil + } + // If we get to here, just return the string. + return result, nil +} + +// getExpressionEvaluator returns a fasttemplate.TagFunc that evaluates input +// as a single expr-lang expression with the provided map as the environment. +func getExpressionEvaluator(env map[string]any) fasttemplate.TagFunc { + return func(out io.Writer, expression string) (int, error) { + program, err := expr.Compile(expression, expr.Env(env)) + if err != nil { + return 0, err + } + result, err := expr.Run(program, env) + if err != nil { + return 0, err + } + if resStr, ok := result.(string); ok { + // A string result can be written directly to the output as is. + return out.Write([]byte(resStr)) + } + // For non-string results, which could include nils, bools, numbers of any + // type, structs, collections, etc. the result must be marshaled to JSON + // before being written to the output. + resJSON, err := json.Marshal(result) + if err != nil { + return 0, err + } + return out.Write(resJSON) + } +} diff --git a/internal/expressions/json_templates_test.go b/internal/expressions/json_templates_test.go new file mode 100644 index 000000000..1e0e89e36 --- /dev/null +++ b/internal/expressions/json_templates_test.go @@ -0,0 +1,204 @@ +package expressions + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEvaluateJSONTemplate(t *testing.T) { + // Context used for all test cases: + type testStruct struct { + AString string + AnInt int + AFloat float64 + ABool bool + AStringMap map[string]string + AnIntMap map[string]int + AStringArr []string + AnIntArr []int + AStruct *testStruct + } + + testStringMap := map[string]string{"aString": "hello", "anotherString": "world"} + testIntMap := map[string]int{"anInt": 42, "anotherInt": 43} + testStringArr := []string{"one", "two", "three"} + testIntArr := []int{1, 2, 3} + testEnv := map[string]any{ + "aString": "hello", + "anInt": 42, + "aFloat": 3.14, + "aBool": true, + "aStringMap": testStringMap, + "anIntMap": testIntMap, + "aStringArr": testStringArr, + "anIntArr": testIntArr, + "aStruct": testStruct{ + AString: "hello", + AnInt: 42, + }, + } + + testCases := []struct { + name string + jsonTemplate string + assertions func(t *testing.T, jsonOutput []byte, err error) + }{ + { + name: "template is not valid JSON", + // This is invalid because the expression itself is not enclosed in + // quotes. This would never be able to move over the wire. + jsonTemplate: `{ "key": ${{ true }} }`, + assertions: func(t *testing.T, _ []byte, err error) { + require.Error(t, err) + require.Contains(t, err.Error(), "input is not valid JSON") + }, + }, + { + name: "scalar values", + jsonTemplate: `{ + "AString": "${{ aString }}", + "AnInt": "${{ anInt }}", + "AFloat": "${{ aFloat }}", + "ABool": "${{ aBool }}" + }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal(t, testEnv["aString"], parsed.AString) + require.Equal(t, testEnv["anInt"], parsed.AnInt) + require.Equal(t, testEnv["aFloat"], parsed.AFloat) + require.Equal(t, testEnv["aBool"], parsed.ABool) + }, + }, + { + name: "mixing an expression with string literals", + jsonTemplate: `{ "AString": "${{ aString }}, world!" }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal( + t, + testEnv["aString"].(string)+", world!", // nolint: forcetypeassert + parsed.AString, + ) + }, + }, + { + name: "multiple expressions in one value", + jsonTemplate: `{ "AString": "${{ aString }}, ${{ anInt }}!" }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal( + t, + fmt.Sprintf("%s, %d!", testEnv["aString"], testEnv["anInt"]), // nolint: forcetypeassert + parsed.AString, + ) + }, + }, + { + name: "maps", + jsonTemplate: `{ + "AStringMap": "${{ aStringMap }}", + "AnIntMap": "${{ anIntMap }}" + }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal(t, testStringMap, parsed.AStringMap) + require.Equal(t, testIntMap, parsed.AnIntMap) + }, + }, + { + name: "arrays", + jsonTemplate: `{ + "AStringArr": "${{ aStringArr }}", + "AnIntArr": "${{ anIntArr }}" + }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal(t, testStringArr, parsed.AStringArr) + require.Equal(t, testIntArr, parsed.AnIntArr) + }, + }, + { + name: "structs", + jsonTemplate: `{ "AStruct": "${{ aStruct }}" }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.NotNil(t, parsed.AStruct) + require.Equal(t, testEnv["aStruct"], *parsed.AStruct) + }, + }, + { + name: "null", + jsonTemplate: `{ "AStruct": "${{ \"null\" }}" }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{ + AStruct: &testStruct{}, // This should get nilled out + } + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Nil(t, parsed.AStruct) + }, + }, + { + name: "test recursion", + jsonTemplate: `{ + "AStringMap": { "key": "${{ aString }}" }, + "AnIntMap": { "key": "${{ anInt }}" }, + "AStringArr": [ "${{ aString }}"], + "AnIntArr": [ "${{ anInt }}"] + }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal( + t, + // nolint: forcetypeassert + testStruct{ + AStringMap: map[string]string{"key": testEnv["aString"].(string)}, + AnIntMap: map[string]int{"key": testEnv["anInt"].(int)}, + AStringArr: []string{testEnv["aString"].(string)}, + AnIntArr: []int{testEnv["anInt"].(int)}, + }, + parsed, + ) + }, + }, + { + name: "quote function forces string result", + jsonTemplate: `{ "AString": "${{ quote(anInt) }}" }`, + assertions: func(t *testing.T, jsonOutput []byte, err error) { + require.NoError(t, err) + parsed := testStruct{} + require.NoError(t, json.Unmarshal(jsonOutput, &parsed)) + require.Equal(t, fmt.Sprintf("%d", testEnv["anInt"]), parsed.AString) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + jsonOutput, err := EvaluateJSONTemplate([]byte(testCase.jsonTemplate), testEnv) + testCase.assertions(t, jsonOutput, err) + }) + } + + t.Run("quote function is forbidden", func(t *testing.T) { + _, err := EvaluateJSONTemplate([]byte(`{}`), map[string]any{"quote": nil}) + require.ErrorContains(t, err, `"quote" is a forbidden key`) + }) +}