diff --git a/CHANGELOG.md b/CHANGELOG.md
index 14c698f815..aecbaa8391 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ project adheres to [Semantic Versioning](http://semver.org/).
## Unreleased
+* `http.send` function takes an optional `force_json_decode` input. By default,
+ it is set to `false`. This backwards incompatible change will need users to
+ specify the `Content-type` as `application/json` in the HTTP server
+ response or set `force_json_decode` to `true` in order to json decode
+ the HTTP response body.
+
## 0.10.5
* These release contians a small but backwards incompatible change to
diff --git a/docs/book/language-reference.md b/docs/book/language-reference.md
index 019fd8d16c..3405eb98d7 100644
--- a/docs/book/language-reference.md
+++ b/docs/book/language-reference.md
@@ -209,9 +209,10 @@ evaluation query will always return the same value.
| ``walk(x, [path, value])`` | ``walk`` is a relation that produces ``path`` and ``value`` pairs for documents under ``x``. ``path`` is ``array`` representing a pointer to ``value`` in ``x``. Queries can use ``walk`` to traverse documents nested under ``x`` (recursively). |
### HTTP
+
| Built-in | Description |
| ------- |-------------|
-| ``output := http.send(request)`` | ``http.send`` executes a HTTP request and returns the response.``request`` is an object containing keys ``method``, ``url`` and optionally ``body``, ``enable_redirect``, ``headers``, ``tls_use_system_certs``, ``tls_ca_cert_file``, ``tls_ca_cert_env_variable``, ``tls_client_cert_env_variable``, ``tls_client_key_env_variable`` or ``tls_client_cert_file``, ``tls_client_key_file`` . For example, ``http.send({"method": "get", "url": "http://www.openpolicyagent.org/", "headers": {"X-Foo":"bar", "X-Opa": "rules"}}, output)``. ``output`` is an object containing keys ``status``, ``status_code`` and ``body`` which represent the HTTP status, status code and response body respectively. Sample output, ``{"status": "200 OK", "status_code": 200, "body": null``}. By default, http redirects are not enabled. To enable, set ``enable_redirect`` to ``true``.|
+| ``http.send(request, output)`` | ``http.send`` executes a HTTP request and returns the response.``request`` is an object containing keys ``method``, ``url`` and optionally ``body``, ``enable_redirect``, ``force_json_decode``, ``headers``, ``tls_use_system_certs``, ``tls_ca_cert_file``, ``tls_ca_cert_env_variable``, ``tls_client_cert_env_variable``, ``tls_client_key_env_variable`` or ``tls_client_cert_file``, ``tls_client_key_file`` . For example, ``http.send({"method": "get", "url": "http://www.openpolicyagent.org/", "headers": {"X-Foo":"bar", "X-Opa": "rules"}}, output)``. ``output`` is an object containing keys ``status``, ``status_code``, ``body`` and ``raw_body`` which represent the HTTP status, status code, JSON value from the response body and response body as string respectively. Sample output, ``{"status": "200 OK", "status_code": 200, "body": {"hello": "world"}, "raw_body": "{\"hello\": \"world\"}"``}. By default, HTTP redirects are not enabled. To enable, set ``enable_redirect`` to ``true``. Also ``force_json_decode`` is set to ``false`` by default. This means if the HTTP server response does not specify the ``Content-type`` as ``application/json``, the response body will not be JSON decoded ie. output's ``body`` field will be ``null``. To change this behaviour, set ``force_json_decode`` to ``true``.|
#### HTTPs Usage
diff --git a/topdown/http.go b/topdown/http.go
index e185211640..e2d290ccf7 100644
--- a/topdown/http.go
+++ b/topdown/http.go
@@ -9,6 +9,8 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
+ "io"
+ "io/ioutil"
"strconv"
"net/http"
@@ -27,6 +29,7 @@ var allowedKeyNames = [...]string{
"url",
"body",
"enable_redirect",
+ "force_json_decode",
"headers",
"tls_use_system_certs",
"tls_ca_cert_file",
@@ -127,6 +130,7 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
var tlsClientKeyFile string
var body *bytes.Buffer
var enableRedirect bool
+ var forceJSONDecode bool
var tlsUseSystemCerts bool
var tlsConfig tls.Config
var clientCerts []tls.Certificate
@@ -150,6 +154,11 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
if err != nil {
return nil, err
}
+ case "force_json_decode":
+ forceJSONDecode, err = strconv.ParseBool(obj.Get(val).String())
+ if err != nil {
+ return nil, err
+ }
case "body":
bodyVal := obj.Get(val).Value
bodyValInterface, err := ast.JSON(bodyVal)
@@ -272,12 +281,27 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
// format the http result
var resultBody interface{}
- json.NewDecoder(resp.Body).Decode(&resultBody)
+ var resultRawBody []byte
+
+ var buf bytes.Buffer
+ tee := io.TeeReader(resp.Body, &buf)
+ resultRawBody, err = ioutil.ReadAll(tee)
+ if err != nil {
+ return nil, err
+ }
+
+ // If the response body cannot be JSON decoded,
+ // an error will not be returned. Instead the "body" field
+ // in the result will be null.
+ if isContentTypeJSON(resp.Header) || forceJSONDecode {
+ json.NewDecoder(&buf).Decode(&resultBody)
+ }
result := make(map[string]interface{})
result["status"] = resp.Status
result["status_code"] = resp.StatusCode
result["body"] = resultBody
+ result["raw_body"] = string(resultRawBody)
resultObj, err := ast.InterfaceToValue(result)
if err != nil {
@@ -291,6 +315,10 @@ func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error)
return resultObj, nil
}
+func isContentTypeJSON(header http.Header) bool {
+ return strings.Contains(header.Get("Content-Type"), "application/json")
+}
+
// getCtxKey returns the cache key.
// Key format: _
func getCtxKey(method string, url string) string {
diff --git a/topdown/http_test.go b/topdown/http_test.go
index 9a24877b60..185f10cb61 100644
--- a/topdown/http_test.go
+++ b/topdown/http_test.go
@@ -51,6 +51,7 @@ func TestHTTPGetRequest(t *testing.T) {
bodyMap := map[string]string{"id": "1", "firstname": "John"}
body = append(body, bodyMap)
expectedResult["body"] = body
+ expectedResult["raw_body"] = "[{\"id\":\"1\",\"firstname\":\"John\"}]\n"
resultObj, err := ast.InterfaceToValue(expectedResult)
if err != nil {
@@ -64,7 +65,46 @@ func TestHTTPGetRequest(t *testing.T) {
expected interface{}
}{
{"http.send", []string{fmt.Sprintf(
- `p = x { http.send({"method": "get", "url": "%s"}, x) }`, ts.URL)}, resultObj.String()},
+ `p = x { http.send({"method": "get", "url": "%s", "force_json_decode": true}, x) }`, ts.URL)}, resultObj.String()},
+ }
+
+ data := loadSmallTestData()
+
+ for _, tc := range tests {
+ runTopDownTestCase(t, data, tc.note, tc.rules, tc.expected)
+ }
+}
+
+func TestHTTPEnableJSONDecode(t *testing.T) {
+
+ // test server
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body := "*Hello World®"
+ fmt.Fprint(w, body)
+ }))
+
+ defer ts.Close()
+
+ // expected result
+ expectedResult := make(map[string]interface{})
+ expectedResult["status"] = "200 OK"
+ expectedResult["status_code"] = http.StatusOK
+ expectedResult["body"] = nil
+ expectedResult["raw_body"] = "*Hello World®"
+
+ resultObj, err := ast.InterfaceToValue(expectedResult)
+ if err != nil {
+ panic(err)
+ }
+
+ // run the test
+ tests := []struct {
+ note string
+ rules []string
+ expected interface{}
+ }{
+ {"http.send", []string{fmt.Sprintf(
+ `p = x { http.send({"method": "get", "url": "%s", "force_json_decode": true}, x) }`, ts.URL)}, resultObj.String()},
}
data := loadSmallTestData()
@@ -101,6 +141,7 @@ func TestHTTPCustomHeaders(t *testing.T) {
bodyMap := map[string][]string{"X-Foo": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, "X-Opa": {"server"}}
expectedResult["body"] = bodyMap
+ expectedResult["raw_body"] = "{\"X-Foo\":[\"ISO-8859-1,utf-8;q=0.7,*;q=0.7\"],\"X-Opa\":[\"server\"]}\n"
jsonString, err := json.Marshal(expectedResult)
if err != nil {
@@ -142,6 +183,7 @@ func TestHTTPostRequest(t *testing.T) {
return
}
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(person)
}))
@@ -153,6 +195,7 @@ func TestHTTPostRequest(t *testing.T) {
"status": "200 OK",
"status_code": http.StatusOK,
"body": map[string]string{"id": "2", "firstname": "Joe"},
+ "raw_body": "{\"id\":\"2\",\"firstname\":\"Joe\"}\n",
}
resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -213,6 +256,7 @@ func TestHTTDeleteRequest(t *testing.T) {
}
}
+ w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(people)
}))
@@ -228,6 +272,7 @@ func TestHTTDeleteRequest(t *testing.T) {
bodyMap := map[string]string{"id": "1", "firstname": "John"}
body = append(body, bodyMap)
expectedResult["body"] = body
+ expectedResult["raw_body"] = "[{\"id\":\"1\",\"firstname\":\"John\"}]\n"
resultObj, err := ast.InterfaceToValue(expectedResult)
if err != nil {
@@ -286,9 +331,10 @@ func TestHTTPRedirectDisable(t *testing.T) {
// expected result
expectedResult := make(map[string]interface{})
+ expectedResult["body"] = nil
+ expectedResult["raw_body"] = "Moved Permanently.\n\n"
expectedResult["status"] = "301 Moved Permanently"
expectedResult["status_code"] = http.StatusMovedPermanently
- expectedResult["body"] = nil
resultObj, err := ast.InterfaceToValue(expectedResult)
if err != nil {
@@ -316,6 +362,7 @@ func TestHTTPRedirectEnable(t *testing.T) {
expectedResult["status"] = "200 OK"
expectedResult["status_code"] = http.StatusOK
expectedResult["body"] = nil
+ expectedResult["raw_body"] = ""
resultObj, err := ast.InterfaceToValue(expectedResult)
if err != nil {
@@ -436,6 +483,7 @@ func TestHTTPSClient(t *testing.T) {
expectedResult := map[string]interface{}{
"status": "200 OK",
"status_code": http.StatusOK,
+ "raw_body": "{\"CommonName\":\"my-ca\"}",
}
expectedResult["body"] = bodyMap
@@ -458,6 +506,7 @@ func TestHTTPSClient(t *testing.T) {
"status": "200 OK",
"status_code": http.StatusOK,
"body": nil,
+ "raw_body": "",
}
resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -479,6 +528,7 @@ func TestHTTPSClient(t *testing.T) {
"status": "200 OK",
"status_code": http.StatusOK,
"body": nil,
+ "raw_body": "",
}
resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -500,6 +550,7 @@ func TestHTTPSClient(t *testing.T) {
"status": "200 OK",
"status_code": http.StatusOK,
"body": nil,
+ "raw_body": "",
}
resultObj, err := ast.InterfaceToValue(expectedResult)
@@ -521,6 +572,7 @@ func TestHTTPSClient(t *testing.T) {
"status": "200 OK",
"status_code": http.StatusOK,
"body": nil,
+ "raw_body": "",
}
resultObj, err := ast.InterfaceToValue(expectedResult)