private/protocol/restjson: Define JSONValue marshaling for body and q…

…uerystring (#1640)

Adds support for APIs which use JSONValue for body and querystring targets.

Fix #1636

models/protocol_tests/generate.go

@@ -459,7 +459,7 @@ func GenerateAssertions(out interface{}, shape *api.Shape, prefix string) string
 				code += GenerateAssertions(v, s, prefix+"[\""+k+"\"]")
 			}
 		} else if shape.Type == "jsonvalue" {
-			code += fmt.Sprintf("reflect.DeepEqual(%s, map[string]interface{}%s)", prefix, walkMap(out.(map[string]interface{})))
+			code += fmt.Sprintf("reflect.DeepEqual(%s, map[string]interface{}%s)\n", prefix, walkMap(out.(map[string]interface{})))
 		} else {
 			for _, k := range keys {
 				v := t[k]
@@ -517,6 +517,7 @@ func getType(t string) uint {
 }
 
 func main() {
+	fmt.Println("Generating test suite", os.Args[1:])
 	out := generateTestSuite(os.Args[1])
 	if len(os.Args) == 3 {
 		f, err := os.Create(os.Args[2])

models/protocol_tests/input/rest-json.json

@@ -1310,17 +1310,46 @@
     "shapes": {
       "InputShape": {
         "type": "structure",
+        "payload": "Body",
         "members": {
-          "Attr": {
+          "HeaderField": {
             "shape": "StringType",
-              "jsonvalue": true,
-              "location": "header",
-              "locationName": "X-Amz-Foo"
+            "jsonvalue": true,
+            "location": "header",
+            "locationName": "X-Amz-Foo"
+          },
+          "QueryField": {
+            "shape": "StringType",
+            "jsonvalue": true,
+            "location": "querystring",
+            "locationName": "Bar"
+          },
+          "Body": {
+            "shape": "BodyStructure"
           }
         }
       },
       "StringType": {
         "type": "string"
+      },
+      "ListType": {
+        "type": "list",
+        "member": {
+          "shape": "StringType",
+          "jsonvalue": true
+        }
+      },
+      "BodyStructure": {
+        "type": "structure",
+        "members": {
+          "BodyField": {
+            "shape": "StringType",
+            "jsonvalue": true
+          },
+          "BodyListField": {
+            "shape": "ListType"
+          }
+        }
       }
     },
     "cases": [
@@ -1335,12 +1364,16 @@
           "name": "OperationName"
         },
         "params": {
-          "Attr": {"Foo":"Bar"}
+          "HeaderField": {"Foo":"Bar"},
+          "QueryField": {"Foo":"Bar"},
+          "Body": {
+            "BodyField": {"Foo":"Bar"}
+          }
         },
         "serialized": {
-          "uri": "/",
+          "uri": "/?Bar=%7B%22Foo%22%3A%22Bar%22%7D",
           "headers": {"X-Amz-Foo": "eyJGb28iOiJCYXIifQ=="},
-          "body": ""
+          "body": "{\"BodyField\":\"{\\\"Foo\\\":\\\"Bar\\\"}\"}"
         }
       },
       {
@@ -1353,6 +1386,27 @@
           },
           "name": "OperationName"
         },
+        "params": {
+          "Body": {
+            "BodyListField": [{"Foo":"Bar"}]
+          }
+        },
+        "serialized": {
+          "uri": "/",
+          "headers": {},
+          "body": "{\"BodyListField\":[\"{\\\"Foo\\\":\\\"Bar\\\"}\"]}"
+        }
+      },
+	  {
+        "given": {
+          "input": {
+            "shape": "InputShape"
+          },
+          "http": {
+            "method": "POST"
+          },
+          "name": "OperationName"
+        },
         "params": {
         },
         "serialized": {

models/protocol_tests/output/rest-json.json

@@ -614,16 +614,30 @@
       "OutputShape": {
         "type": "structure",
         "members": {
-          "Attr": {
+          "HeaderField": {
             "shape": "StringType",
-              "jsonvalue": true,
-              "location": "header",
-              "locationName": "X-Amz-Foo"
+            "jsonvalue": true,
+            "location": "header",
+            "locationName": "X-Amz-Foo"
+          },
+          "BodyField":{
+            "shape": "StringType",
+            "jsonvalue": true
+          },
+          "BodyListField": {
+            "shape": "ListType"
           }
         }
       },
       "StringType": {
         "type": "string"
+      },
+      "ListType": {
+        "type": "list",
+        "member": {
+          "shape": "StringType",
+          "jsonvalue": true
+        }
       }
     },
     "cases": [
@@ -635,11 +649,43 @@
           "name": "OperationName"
         },
         "result": {
-          "Attr": {"Foo":"Bar"}
+          "HeaderField": {"Foo":"Bar"},
+          "BodyField": {"Foo":"Bar"}
         },
         "response": {
           "status_code": 200,
           "headers": {"X-Amz-Foo": "eyJGb28iOiJCYXIifQ=="},
+          "body": "{\"BodyField\":\"{\\\"Foo\\\":\\\"Bar\\\"}\"}"
+        }
+      },
+      {
+        "given": {
+          "output": {
+            "shape": "OutputShape"
+          },
+          "name": "OperationName"
+        },
+        "result": {
+          "BodyListField": [{"Foo":"Bar"}]
+        },
+        "response": {
+          "status_code": 200,
+          "headers": {},
+          "body": "{\"BodyListField\":[\"{\\\"Foo\\\":\\\"Bar\\\"}\"]}"
+        }
+      },
+      {
+        "given": {
+          "output": {
+            "shape": "OutputShape"
+          },
+          "name": "OperationName"
+        },
+        "result": {
+        },
+        "response": {
+          "status_code": 200,
+          "headers": {},
           "body": ""
         }
       }

private/model/api/param_filler.go

@@ -4,6 +4,7 @@ package api
 
 import (
 	"fmt"
+	"encoding/json"
 	"reflect"
 	"strings"
 
@@ -79,6 +80,19 @@ func (f paramFiller) paramsStructAny(value interface{}, shape *Shape) string {
 		if v.IsValid() {
 			return fmt.Sprintf("aws.Time(time.Unix(%d, 0))", int(v.Float()))
 		}
+	case "jsonvalue":
+		v, err := json.Marshal(value)
+		if err != nil {
+			panic("failed to marshal JSONValue, "+err.Error())
+		}
+		const tmpl = `func() aws.JSONValue {
+			var m aws.JSONValue
+			if err := json.Unmarshal([]byte(%q), &m); err != nil {
+				panic("failed to unmarshal JSONValue, "+err.Error())
+			}
+			return m
+		}()`
+		return fmt.Sprintf(tmpl, string(v))
 	default:
 		panic("Unhandled type " + shape.Type)
 	}

private/protocol/json/jsonutil/build.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/private/protocol"
 )
 
@@ -49,7 +50,10 @@ func buildAny(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) err
 				t = "list"
 			}
 		case reflect.Map:
-			t = "map"
+			// cannot be a JSONValue map
+			if _, ok := value.Interface().(aws.JSONValue); !ok {
+				t = "map"
+			}
 		}
 	}
 
@@ -210,14 +214,11 @@ func buildScalar(v reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) erro
 		}
 		buf.Write(strconv.AppendFloat(scratch[:0], f, 'f', -1, 64))
 	default:
-		switch value.Type() {
-		case timeType:
-			converted := v.Interface().(*time.Time)
-
+		switch converted := value.Interface().(type) {
+		case time.Time:
 			buf.Write(strconv.AppendInt(scratch[:0], converted.UTC().Unix(), 10))
-		case byteSliceType:
+		case []byte:
 			if !value.IsNil() {
-				converted := value.Interface().([]byte)
 				buf.WriteByte('"')
 				if len(converted) < 1024 {
 					// for small buffers, using Encode directly is much faster.
@@ -233,6 +234,12 @@ func buildScalar(v reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) erro
 				}
 				buf.WriteByte('"')
 			}
+		case aws.JSONValue:
+			str, err := protocol.EncodeJSONValue(converted, protocol.QuotedEscape)
+			if err != nil {
+				return fmt.Errorf("unable to encode JSONValue, %v", err)
+			}
+			buf.WriteString(str)
 		default:
 			return fmt.Errorf("unsupported JSON value %v (%s)", value.Interface(), value.Type())
 		}

private/protocol/json/jsonutil/unmarshal.go

@@ -8,6 +8,9 @@ import (
 	"io/ioutil"
 	"reflect"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/private/protocol"
 )
 
 // UnmarshalJSON reads a stream and unmarshals the results in object v.
@@ -50,7 +53,10 @@ func unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag)
 				t = "list"
 			}
 		case reflect.Map:
-			t = "map"
+			// cannot be a JSONValue map
+			if _, ok := value.Interface().(aws.JSONValue); !ok {
+				t = "map"
+			}
 		}
 	}
 
@@ -183,6 +189,13 @@ func unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTa
 				return err
 			}
 			value.Set(reflect.ValueOf(b))
+		case aws.JSONValue:
+			// No need to use escaping as the value is a non-quoted string.
+			v, err := protocol.DecodeJSONValue(d, protocol.NoEscape)
+			if err != nil {
+				return err
+			}
+			value.Set(reflect.ValueOf(v))
 		default:
 			return errf()
 		}

private/protocol/jsonvalue.go

@@ -0,0 +1,76 @@
+package protocol
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+// EscapeMode is the mode that should be use for escaping a value
+type EscapeMode uint
+
+// The modes for escaping a value before it is marshaled, and unmarshaled.
+const (
+	NoEscape EscapeMode = iota
+	Base64Escape
+	QuotedEscape
+)
+
+// EncodeJSONValue marshals the value into a JSON string, and optionally base64
+// encodes the string before returning it.
+//
+// Will panic if the escape mode is unknown.
+func EncodeJSONValue(v aws.JSONValue, escape EscapeMode) (string, error) {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	switch escape {
+	case NoEscape:
+		return string(b), nil
+	case Base64Escape:
+		return base64.StdEncoding.EncodeToString(b), nil
+	case QuotedEscape:
+		return strconv.Quote(string(b)), nil
+	}
+
+	panic(fmt.Sprintf("EncodeJSONValue called with unknown EscapeMode, %v", escape))
+}
+
+// DecodeJSONValue will attempt to decode the string input as a JSONValue.
+// Optionally decoding base64 the value first before JSON unmarshaling.
+//
+// Will panic if the escape mode is unknown.
+func DecodeJSONValue(v string, escape EscapeMode) (aws.JSONValue, error) {
+	var b []byte
+	var err error
+
+	switch escape {
+	case NoEscape:
+		b = []byte(v)
+	case Base64Escape:
+		b, err = base64.StdEncoding.DecodeString(v)
+	case QuotedEscape:
+		var u string
+		u, err = strconv.Unquote(v)
+		b = []byte(u)
+	default:
+		panic(fmt.Sprintf("DecodeJSONValue called with unknown EscapeMode, %v", escape))
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	m := aws.JSONValue{}
+	err = json.Unmarshal(b, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}