Add object manipulation built-ins

This commit adds in the following built-ins:

`object.remove`
`object.union`
`object.filter`

All of which are helpers for object manipulation in policies.

Reference: open-policy-agent#1617
Signed-off-by: Patrick East <east.patrick@gmail.com>

ast/builtins.go

@@ -125,12 +125,15 @@ var DefaultBuiltins = [...]*Builtin{
 	YAMLMarshal,
 	YAMLUnmarshal,
 
+	// Object Manipulation
+	ObjectUnion,
+	ObjectRemove,
+	ObjectFilter,
+	ObjectGet,
+
 	// JSON Object Manipulation
 	JSONFilter,
 
-	// Other object functions
-	ObjectGet,
-
 	// Tokens
 	JWTDecode,
 	JWTVerifyRS256,
@@ -966,6 +969,62 @@ var JSONFilter = &Builtin{
 	),
 }
 
+// ObjectUnion creates a new object that is the asymmetric union of two objects
+var ObjectUnion = &Builtin{
+	Name: "object.union",
+	Decl: types.NewFunction(
+		types.Args(
+			types.NewObject(
+				nil,
+				types.NewDynamicProperty(types.A, types.A),
+			),
+			types.NewObject(
+				nil,
+				types.NewDynamicProperty(types.A, types.A),
+			),
+		),
+		types.A,
+	),
+}
+
+// ObjectRemove Removes specified keys from an object
+var ObjectRemove = &Builtin{
+	Name: "object.remove",
+	Decl: types.NewFunction(
+		types.Args(
+			types.NewObject(
+				nil,
+				types.NewDynamicProperty(types.A, types.A),
+			),
+			types.NewAny(
+				types.NewArray(nil, types.A),
+				types.NewSet(types.A),
+				types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
+			),
+		),
+		types.A,
+	),
+}
+
+// ObjectFilter filters the object by keeping only specified keys
+var ObjectFilter = &Builtin{
+	Name: "object.filter",
+	Decl: types.NewFunction(
+		types.Args(
+			types.NewObject(
+				nil,
+				types.NewDynamicProperty(types.A, types.A),
+			),
+			types.NewAny(
+				types.NewArray(nil, types.A),
+				types.NewSet(types.A),
+				types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)),
+			),
+		),
+		types.A,
+	),
+}
+
 // Base64Encode serializes the input string into base64 encoding.
 var Base64Encode = &Builtin{
 	Name: "base64.encode",

ast/term.go

@@ -1571,6 +1571,7 @@ type Object interface {
 	Diff(other Object) Object
 	Intersect(other Object) [][3]*Term
 	Merge(other Object) (Object, bool)
+	MergeWith(other Object, conflictResolver func(v1, v2 *Term) (*Term, bool)) (Object, bool)
 	Filter(filter Object) (Object, error)
 	Keys() []*Term
 }
@@ -1808,28 +1809,48 @@ func (obj *object) MarshalJSON() ([]byte, error) {
 // overlapping keys between obj and other, the values of associated with the keys are merged. Only
 // objects can be merged with other objects. If the values cannot be merged, the second turn value
 // will be false.
-func (obj object) Merge(other Object) (result Object, ok bool) {
-	result = NewObject()
+func (obj object) Merge(other Object) (Object, bool) {
+	return obj.MergeWith(other, func(v1, v2 *Term) (*Term, bool) {
+		obj1, ok1 := v1.Value.(Object)
+		obj2, ok2 := v2.Value.(Object)
+		if !ok1 || !ok2 {
+			return nil, true
+		}
+		obj3, ok := obj1.Merge(obj2)
+		if !ok {
+			return nil, true
+		}
+		return NewTerm(obj3), false
+	})
+}
+
+// MergeWith returns a new Object containing the merged keys of obj and other.
+// If there are overlapping keys between obj and other, the conflictResolver
+// is called. The conflictResolver can return a merged value and a boolean
+// indicating if the merge has failed and should stop.
+func (obj object) MergeWith(other Object, conflictResolver func(v1, v2 *Term) (*Term, bool)) (Object, bool) {
+	result := NewObject()
 	stop := obj.Until(func(k, v *Term) bool {
-		if v2 := other.Get(k); v2 == nil {
+		v2 := other.Get(k)
+		// The key didn't exist in other, keep the original value
+		if v2 == nil {
 			result.Insert(k, v)
-		} else {
-			obj1, ok1 := v.Value.(Object)
-			obj2, ok2 := v2.Value.(Object)
-			if !ok1 || !ok2 {
-				return true
-			}
-			obj3, ok := obj1.Merge(obj2)
-			if !ok {
-				return true
-			}
-			result.Insert(k, NewTerm(obj3))
+			return false
 		}
-		return false
+
+		// The key exists in both, resolve the conflict if possible
+		merged, stop := conflictResolver(v, v2)
+		if !stop {
+			result.Insert(k, merged)
+		}
+		return stop
 	})
+
 	if stop {
 		return nil, false
 	}
+
+	// Copy in any values from other for keys that don't exist in obj
 	other.Foreach(func(k, v *Term) {
 		if v2 := obj.Get(k); v2 == nil {
 			result.Insert(k, v)

docs/content/policy-cheatsheet.md

@@ -259,6 +259,15 @@ merge_objects(a, b) = c {
 }
 ```
 
+> Note: use the `object.union` builtin function unless custom behavior is required!
+
+```live:rules/merge_builtin:query:read_only
+x := {"a": true, "b": false}
+y := {"b": "foo", "c": 4}
+z := {"a": true, "b": "foo", "c": 4}
+object.union(y, x) == z
+```
+
 ## Tests
 
 ```live:tests:module:read_only

docs/content/policy-reference.md

@@ -69,8 +69,15 @@ complex types.
 ### Objects
 | Built-in | Description |
 | -------- | ----------- |
-| <span class="opa-keep-it-together">`filtered := json.filter(object, paths)`</span> | `filtered` is the remaining data from `object` with only keys specified in `paths` which is an array or set of key paths. Each path may be a JSON string path or an array of path segments. For example: `json.filter({"a": {"b": "x", "c": "y"}}, ["a/b"]` will result in `{"a": {"b": "x"}}`). The `json` string `paths` may reference into array values by using index numbers. For example with the object `{"a": ["x", "y", "z"]}` the path `a/1` references `y` |
 | <span class="opa-keep-it-together">`value := object.get(object, key, default)`</span> | `value` is the value stored by the `object` at `key`. If no value is found, `default` is returned. |
+| <span class="opa-keep-it-together">`output := object.remove(object, keys)`</span> | `output` is a new object which is the result of removing the specified `keys` from `object`. `keys` must be either an array, object, or set of keys. |
+| <span class="opa-keep-it-together">`output := object.union(objectA, objectB)`</span> | `output` is a new object which is the result of an asymmetric recursive union of two objects where conflicts are resolved by choosing the key from the right-hand object (`objectB`). For example: `object.union({"a": 1, "b": 2, "c": {"d": 3}}, {"a": 7, "c": {"d": 4, "e": 5}})` will result in `{"a": 7, "b": 2, "c": {"d": 4, "e": 5}}`  |
+| <span class="opa-keep-it-together">`filtered := object.filter(object, keys)`</span> | `filtered` is a new object with the remaining data from `object` with only keys specified in `keys` which is an array, object, or set of keys. For example: `object.filter({"a": {"b": "x", "c": "y"}, "d": "z"}, ["a"])` will result in `{"a": {"b": "x", "c": "y"}}`). |
+| <span class="opa-keep-it-together">`filtered := json.filter(object, paths)`</span> | `filtered` is the remaining data from `object` with only keys specified in `paths` which is an array or set of JSON string paths. For example: `json.filter({"a": {"b": "x", "c": "y"}}, ["a/b"])` will result in `{"a": {"b": "x"}}`). |
+
+> The `json` string `paths` may reference into array values by using index numbers. For example with the object `{"a": ["x", "y", "z"]}` the path `a[1]` references `y`
+
+> When `keys` are provided as an object only the top level keys on the object will be used, values are ignored. For example: `object.remove({"a": {"b": {"c": 2}}, "x": 123}, {"a": 1}) == {"x": 123}` regardless of the value for key `a` in the keys object, the following `keys` object gives the same result `object.remove({"a": {"b": {"c": 2}}, "x": 123}, {"a": {"b": {"foo": "bar"}}}) == {"x": 123}` 
 
 ### Strings
 

topdown/array_test.go

@@ -5,7 +5,6 @@
 package topdown
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -17,8 +16,8 @@ func TestTopDownArray(t *testing.T) {
 		expected interface{}
 	}{
 		{"concat", []string{`p = x { x = array.concat([1,2], [3,4]) }`}, "[1,2,3,4]"},
-		{"concat: err", []string{`p = x { x = array.concat(data.b, [3,4]) }`}, fmt.Errorf("operand 1 must be array")},
-		{"concat: err rhs", []string{`p = x { x = array.concat([1,2], data.b) }`}, fmt.Errorf("operand 2 must be array")},
+		{"concat: err", []string{`p = x { x = array.concat(data.b, [3,4]) }`}, &Error{Code: TypeErr, Message: "array.concat: operand 1 must be array but got object"}},
+		{"concat: err rhs", []string{`p = x { x = array.concat([1,2], data.b) }`}, &Error{Code: TypeErr, Message: "array.concat: operand 2 must be array but got object"}},
 
 		{"slice", []string{`p = x { x = array.slice([1,2,3,4,5], 1, 3) }`}, "[2,3]"},
 		{"slice: empty slice", []string{`p = x { x = array.slice([1,2,3], 0, 0) }`}, "[]"},

topdown/casts_test.go

@@ -5,7 +5,6 @@
 package topdown
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/open-policy-agent/opa/ast"
@@ -20,7 +19,7 @@ func TestToArray(t *testing.T) {
 		panic(err)
 	}
 
-	typeErr := fmt.Errorf("type")
+	typeErr := &Error{Code: TypeErr, Message: "operand 1 must be one of {array, set}"}
 
 	tests := []struct {
 		note     string
@@ -41,7 +40,7 @@ func TestToArray(t *testing.T) {
 
 func TestToSet(t *testing.T) {
 
-	typeErr := fmt.Errorf("type")
+	typeErr := &Error{Code: TypeErr, Message: "operand 1 must be one of {array, set}"}
 
 	tests := []struct {
 		note     string
@@ -61,7 +60,7 @@ func TestToSet(t *testing.T) {
 }
 
 func TestCasts(t *testing.T) {
-	typeErr := fmt.Errorf("type")
+	typeErr := &Error{Code: TypeErr}
 
 	tests := []struct {
 		note     string

topdown/cidr_test.go

@@ -2,7 +2,7 @@ package topdown
 
 import (
 	"context"
-	"errors"
+	"net"
 	"testing"
 	"time"
 
@@ -120,7 +120,7 @@ func TestNetCIDRExpand(t *testing.T) {
 			rules: []string{
 				`p = x { net.cidr_expand("192.168.1.1/33", x) }`,
 			},
-			expected: errors.New("invalid CIDR address"),
+			expected: &net.ParseError{Type: "CIDR address", Text: "192.168.1.1/33"},
 		},
 	}
 

topdown/crypto_test.go

@@ -40,7 +40,7 @@ func TestCryptoX509ParseCertificates(t *testing.T) {
 			note:     "bad",
 			certs:    `YmFkc3RyaW5n`,
 			rule:     rule,
-			expected: fmt.Errorf("asn1: structure error"),
+			expected: &Error{Code: BuiltinErr, Message: "asn1: structure error"},
 		},
 	}
 

topdown/http_test.go

@@ -9,7 +9,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -183,8 +182,8 @@ func TestHTTPCustomHeaders(t *testing.T) {
 	}
 }
 
-// TestHTTPostRequest adds a new person
-func TestHTTPostRequest(t *testing.T) {
+// TestHTTPPostRequest adds a new person
+func TestHTTPPostRequest(t *testing.T) {
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 
@@ -258,7 +257,7 @@ func TestHTTPostRequest(t *testing.T) {
 				"headers": {"Content-Type": "application/x-www-form-encoded"},
 				"raw_body": {"bar": "bar"}
 			}`,
-			expected: errors.New("raw_body must be a string"),
+			expected: &Error{Code: BuiltinErr, Message: "raw_body must be a string"},
 		},
 	}
 
@@ -365,8 +364,8 @@ func TestInvalidKeyError(t *testing.T) {
 		rules    []string
 		expected interface{}
 	}{
-		{"invalid keys", []string{`p = x { http.send({"method": "get", "url": "http://127.0.0.1:51113", "bad_key": "bad_value"}, x) }`}, fmt.Errorf(`invalid request parameters(s): {"bad_key"}`)},
-		{"missing keys", []string{`p = x { http.send({"method": "get"}, x) }`}, fmt.Errorf(`missing required request parameters(s): {"url"}`)},
+		{"invalid keys", []string{`p = x { http.send({"method": "get", "url": "http://127.0.0.1:51113", "bad_key": "bad_value"}, x) }`}, &Error{Code: TypeErr, Message: `invalid request parameters(s): {"bad_key"}`}},
+		{"missing keys", []string{`p = x { http.send({"method": "get"}, x) }`}, &Error{Code: TypeErr, Message: `missing required request parameters(s): {"url"}`}},
 	}
 
 	data := loadSmallTestData()
@@ -644,7 +643,7 @@ func TestHTTPSClient(t *testing.T) {
 
 	t.Run("Negative Test: No Root Ca", func(t *testing.T) {
 
-		expectedResult := Error{Code: BuiltinErr, Message: "x509: certificate signed by unknown authority", Location: nil}
+		expectedResult := &Error{Code: BuiltinErr, Message: "x509: certificate signed by unknown authority", Location: nil}
 		data := loadSmallTestData()
 		rule := []string{fmt.Sprintf(
 			`p = x { http.send({"method": "get", "url": "%s", "tls_client_cert_file": "%s", "tls_client_key_file": "%s"}, x) }`, s.URL, localClientCertFile, localClientKeyFile)}
@@ -655,7 +654,7 @@ func TestHTTPSClient(t *testing.T) {
 
 	t.Run("Negative Test: Wrong Cert/Key Pair", func(t *testing.T) {
 
-		expectedResult := Error{Code: BuiltinErr, Message: "tls: private key does not match public key", Location: nil}
+		expectedResult := &Error{Code: BuiltinErr, Message: "tls: private key does not match public key", Location: nil}
 		data := loadSmallTestData()
 		rule := []string{fmt.Sprintf(
 			`p = x { http.send({"method": "get", "url": "%s", "tls_ca_cert_file": "%s", "tls_client_cert_file": "%s", "tls_client_key_file": "%s"}, x) }`, s.URL, localCaFile, localClientCert2File, localClientKeyFile)}
@@ -666,7 +665,7 @@ func TestHTTPSClient(t *testing.T) {
 
 	t.Run("Negative Test: System Certs do not include local rootCA", func(t *testing.T) {
 
-		expectedResult := Error{Code: BuiltinErr, Message: "x509: certificate signed by unknown authority", Location: nil}
+		expectedResult := &Error{Code: BuiltinErr, Message: "x509: certificate signed by unknown authority", Location: nil}
 		data := loadSmallTestData()
 		rule := []string{fmt.Sprintf(
 			`p = x { http.send({"method": "get", "url": "%s", "tls_client_cert_file": "%s", "tls_client_key_file": "%s", "tls_use_system_certs": true}, x) }`, s.URL, localClientCertFile, localClientKeyFile)}