Add support for non-json reponse result to http.send builtin.

CHANGELOG.md

@@ -5,6 +5,12 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+* `http.send` function takes an optional `force_json_decode` input. By default,
+  it is set to `false`. This backwards incompatible change will need users to
+  specify the `Content-type` as `application/json` in the HTTP server
+  response or set `force_json_decode` to `true` in order to json decode
+  the HTTP response body.
+
 ## 0.10.5
 
 * These release contians a small but backwards incompatible change to

docs/book/language-reference.md

@@ -209,9 +209,10 @@ evaluation query will always return the same value.
 | <span class="opa-keep-it-together">``walk(x, [path, value])``</span> | ``walk`` is a relation that produces ``path`` and ``value`` pairs for documents under ``x``. ``path`` is ``array`` representing a pointer to ``value`` in ``x``.  Queries can use ``walk`` to traverse documents nested under ``x`` (recursively). |
 
 ### HTTP
+
 | Built-in | Description |
 | ------- |-------------|
-| <span class="opa-keep-it-together">``output := http.send(request)``</span> | ``http.send`` executes a HTTP request and returns the response.``request`` is an object containing keys ``method``, ``url`` and  optionally ``body``, ``enable_redirect``, ``headers``, ``tls_use_system_certs``, ``tls_ca_cert_file``, ``tls_ca_cert_env_variable``, ``tls_client_cert_env_variable``, ``tls_client_key_env_variable`` or ``tls_client_cert_file``, ``tls_client_key_file`` . For example, ``http.send({"method": "get", "url": "http://www.openpolicyagent.org/", "headers": {"X-Foo":"bar", "X-Opa": "rules"}}, output)``. ``output`` is an object containing keys ``status``, ``status_code`` and ``body`` which represent the HTTP status, status code and response body respectively. Sample output, ``{"status": "200 OK", "status_code": 200, "body": null``}. By default, http redirects are not enabled. To enable, set ``enable_redirect`` to ``true``.|
+| <span class="opa-keep-it-together">``http.send(request, output)``</span> | ``http.send`` executes a HTTP request and returns the response.``request`` is an object containing keys ``method``, ``url`` and  optionally ``body``, ``enable_redirect``, ``force_json_decode``, ``headers``, ``tls_use_system_certs``, ``tls_ca_cert_file``, ``tls_ca_cert_env_variable``, ``tls_client_cert_env_variable``, ``tls_client_key_env_variable`` or ``tls_client_cert_file``, ``tls_client_key_file`` . For example, ``http.send({"method": "get", "url": "http://www.openpolicyagent.org/", "headers": {"X-Foo":"bar", "X-Opa": "rules"}}, output)``. ``output`` is an object containing keys ``status``, ``status_code``, ``body`` and ``raw_body`` which represent the HTTP status, status code, JSON value from the response body and response body as string respectively. Sample output, ``{"status": "200 OK", "status_code": 200, "body": {"hello": "world"}, "raw_body": "{\"hello\": \"world\"}"``}. By default, HTTP redirects are not enabled. To enable, set ``enable_redirect`` to ``true``. Also ``force_json_decode`` is set to ``false`` by default. This means if the HTTP server response does not specify the ``Content-type`` as ``application/json``, the response body will not be JSON decoded ie. output's ``body`` field will be ``null``. To change this behaviour, set ``force_json_decode`` to ``true``.|
 
 #### HTTPs Usage
 

topdown/http.go

@@ -9,6 +9,8 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strconv"
 
 	"net/http"
@@ -27,6 +29,7 @@ var allowedKeyNames = [...]string{
 	"url",
 	"body",
 	"enable_redirect",
+	"force_json_decode",
 	"headers",
 	"tls_use_system_certs",
 	"tls_ca_cert_file",
@@ -127,6 +130,7 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
 	var tlsClientKeyFile string
 	var body *bytes.Buffer
 	var enableRedirect bool
+	var forceJSONDecode bool
 	var tlsUseSystemCerts bool
 	var tlsConfig tls.Config
 	var clientCerts []tls.Certificate
@@ -150,6 +154,11 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
 			if err != nil {
 				return nil, err
 			}
+		case "force_json_decode":
+			forceJSONDecode, err = strconv.ParseBool(obj.Get(val).String())
+			if err != nil {
+				return nil, err
+			}
 		case "body":
 			bodyVal := obj.Get(val).Value
 			bodyValInterface, err := ast.JSON(bodyVal)
@@ -272,12 +281,27 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
 
 	// format the http result
 	var resultBody interface{}
-	json.NewDecoder(resp.Body).Decode(&resultBody)
+	var resultRawBody []byte
+
+	var buf bytes.Buffer
+	tee := io.TeeReader(resp.Body, &buf)
+	resultRawBody, err = ioutil.ReadAll(tee)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the response body cannot be JSON decoded,
+	// an error will not be returned. Instead the "body" field
+	// in the result will be null.
+	if isContentTypeJSON(resp.Header) || forceJSONDecode {
+		json.NewDecoder(&buf).Decode(&resultBody)
+	}
 
 	result := make(map[string]interface{})
 	result["status"] = resp.Status
 	result["status_code"] = resp.StatusCode
 	result["body"] = resultBody
+	result["raw_body"] = string(resultRawBody)
 
 	resultObj, err := ast.InterfaceToValue(result)
 	if err != nil {
@@ -291,6 +315,10 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
 	return resultObj, nil
 }
 
+func isContentTypeJSON(header http.Header) bool {
+	return strings.Contains(header.Get("Content-Type"), "application/json")
+}
+
 // getCtxKey returns the cache key.
 // Key format: <METHOD>_<url>
 func getCtxKey(method string, url string) string {

topdown/http_test.go

@@ -51,6 +51,7 @@ func TestHTTPGetRequest(t *testing.T) {
 	bodyMap := map[string]string{"id": "1", "firstname": "John"}
 	body = append(body, bodyMap)
 	expectedResult["body"] = body
+	expectedResult["raw_body"] = "[{\"id\":\"1\",\"firstname\":\"John\"}]\n"
 
 	resultObj, err := ast.InterfaceToValue(expectedResult)
 	if err != nil {
@@ -64,7 +65,46 @@ func TestHTTPGetRequest(t *testing.T) {
 		expected interface{}
 	}{
 		{"http.send", []string{fmt.Sprintf(
-			`p = x { http.send({"method": "get", "url": "%s"}, x) }`, ts.URL)}, resultObj.String()},
+			`p = x { http.send({"method": "get", "url": "%s", "force_json_decode": true}, x) }`, ts.URL)}, resultObj.String()},
+	}
+
+	data := loadSmallTestData()
+
+	for _, tc := range tests {
+		runTopDownTestCase(t, data, tc.note, tc.rules, tc.expected)
+	}
+}
+
+func TestHTTPEnableJSONDecode(t *testing.T) {
+
+	// test server
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body := "*Hello World®"
+		fmt.Fprint(w, body)
+	}))
+
+	defer ts.Close()
+
+	// expected result
+	expectedResult := make(map[string]interface{})
+	expectedResult["status"] = "200 OK"
+	expectedResult["status_code"] = http.StatusOK
+	expectedResult["body"] = nil
+	expectedResult["raw_body"] = "*Hello World®"
+
+	resultObj, err := ast.InterfaceToValue(expectedResult)
+	if err != nil {
+		panic(err)
+	}
+
+	// run the test
+	tests := []struct {
+		note     string
+		rules    []string
+		expected interface{}
+	}{
+		{"http.send", []string{fmt.Sprintf(
+			`p = x { http.send({"method": "get", "url": "%s", "force_json_decode": true}, x) }`, ts.URL)}, resultObj.String()},
 	}
 
 	data := loadSmallTestData()
@@ -101,6 +141,7 @@ func TestHTTPCustomHeaders(t *testing.T) {
 
 	bodyMap := map[string][]string{"X-Foo": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, "X-Opa": {"server"}}
 	expectedResult["body"] = bodyMap
+	expectedResult["raw_body"] = "{\"X-Foo\":[\"ISO-8859-1,utf-8;q=0.7,*;q=0.7\"],\"X-Opa\":[\"server\"]}\n"
 
 	jsonString, err := json.Marshal(expectedResult)
 	if err != nil {
@@ -142,6 +183,7 @@ func TestHTTPostRequest(t *testing.T) {
 			return
 		}
 
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(http.StatusOK)
 		json.NewEncoder(w).Encode(person)
 	}))
@@ -153,6 +195,7 @@ func TestHTTPostRequest(t *testing.T) {
 		"status":      "200 OK",
 		"status_code": http.StatusOK,
 		"body":        map[string]string{"id": "2", "firstname": "Joe"},
+		"raw_body":    "{\"id\":\"2\",\"firstname\":\"Joe\"}\n",
 	}
 
 	resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -213,6 +256,7 @@ func TestHTTDeleteRequest(t *testing.T) {
 			}
 		}
 
+		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(http.StatusOK)
 		json.NewEncoder(w).Encode(people)
 	}))
@@ -228,6 +272,7 @@ func TestHTTDeleteRequest(t *testing.T) {
 	bodyMap := map[string]string{"id": "1", "firstname": "John"}
 	body = append(body, bodyMap)
 	expectedResult["body"] = body
+	expectedResult["raw_body"] = "[{\"id\":\"1\",\"firstname\":\"John\"}]\n"
 
 	resultObj, err := ast.InterfaceToValue(expectedResult)
 	if err != nil {
@@ -286,9 +331,10 @@ func TestHTTPRedirectDisable(t *testing.T) {
 
 	// expected result
 	expectedResult := make(map[string]interface{})
+	expectedResult["body"] = nil
+	expectedResult["raw_body"] = "<a href=\"/test\">Moved Permanently</a>.\n\n"
 	expectedResult["status"] = "301 Moved Permanently"
 	expectedResult["status_code"] = http.StatusMovedPermanently
-	expectedResult["body"] = nil
 
 	resultObj, err := ast.InterfaceToValue(expectedResult)
 	if err != nil {
@@ -316,6 +362,7 @@ func TestHTTPRedirectEnable(t *testing.T) {
 	expectedResult["status"] = "200 OK"
 	expectedResult["status_code"] = http.StatusOK
 	expectedResult["body"] = nil
+	expectedResult["raw_body"] = ""
 
 	resultObj, err := ast.InterfaceToValue(expectedResult)
 	if err != nil {
@@ -436,6 +483,7 @@ func TestHTTPSClient(t *testing.T) {
 		expectedResult := map[string]interface{}{
 			"status":      "200 OK",
 			"status_code": http.StatusOK,
+			"raw_body":    "{\"CommonName\":\"my-ca\"}",
 		}
 		expectedResult["body"] = bodyMap
 
@@ -458,6 +506,7 @@ func TestHTTPSClient(t *testing.T) {
 			"status":      "200 OK",
 			"status_code": http.StatusOK,
 			"body":        nil,
+			"raw_body":    "",
 		}
 
 		resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -479,6 +528,7 @@ func TestHTTPSClient(t *testing.T) {
 			"status":      "200 OK",
 			"status_code": http.StatusOK,
 			"body":        nil,
+			"raw_body":    "",
 		}
 
 		resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -500,6 +550,7 @@ func TestHTTPSClient(t *testing.T) {
 			"status":      "200 OK",
 			"status_code": http.StatusOK,
 			"body":        nil,
+			"raw_body":    "",
 		}
 
 		resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -521,6 +572,7 @@ func TestHTTPSClient(t *testing.T) {
 			"status":      "200 OK",
 			"status_code": http.StatusOK,
 			"body":        nil,
+			"raw_body":    "",
 		}
 
 		resultObj, err := ast.InterfaceToValue(expectedResult)