API Marshaling: Add generated marshalers for RESTXML protocol (#1409)

This change is the start of an iterative set of changes that will
replace the SDK's current reflection base protocol marshalers with
generated code for each API type.

This works starts with the RESTXML protocol's encoders. The following
benchmarks show a significant improvement of execution speed and reduced
memory usage.

Benchmarks of RESTXML build and full request lifecycle between master and this change. `Build` benchmarks focus on just the marshaling of the request's input parameters. `Request` benchmarks provide a view of the impact of the changes in context to a full request lifecycle. Does not include network latency, nor umarshaling of a response object.

```
go test -tags bench -run NONE -bench ".*" -benchmem ./private/protocol/restxml
```

```
benchmark                                            old ns/op     new ns/op     delta
BenchmarkRESTXMLBuild_Complex_CFCreateDistro-8       264834        59891         -77.39%
BenchmarkRESTXMLBuild_Simple_CFDeleteDistro-8        9666          5894          -39.02%
BenchmarkRESTXMLBuild_REST_S3HeadObject-8            19723         7340          -62.78%
BenchmarkRESTXMLBuild_XML_S3PutObjectAcl-8           44739         15656         -65.01%
BenchmarkRESTXMLRequest_Complex_CFCreateDistro-8     586238        292755        -50.06%
BenchmarkRESTXMLRequest_Simple_CFDeleteDistro-8      168954        158484        -6.20%
BenchmarkRESTXMLRequest_REST_S3HeadObject-8          228720        199964        -12.57%
BenchmarkRESTXMLRequest_XML_S3PutObjectAcl-8         253821        206911        -18.48%

benchmark                                            old allocs     new allocs     delta
BenchmarkRESTXMLBuild_Complex_CFCreateDistro-8       1332           478            -64.11%
BenchmarkRESTXMLBuild_Simple_CFDeleteDistro-8        63             44             -30.16%
BenchmarkRESTXMLBuild_REST_S3HeadObject-8            134            57             -57.46%
BenchmarkRESTXMLBuild_XML_S3PutObjectAcl-8           247            113            -54.25%
BenchmarkRESTXMLRequest_Complex_CFCreateDistro-8     1641           786            -52.10%
BenchmarkRESTXMLRequest_Simple_CFDeleteDistro-8      230            211            -8.26%
BenchmarkRESTXMLRequest_REST_S3HeadObject-8          443            366            -17.38%
BenchmarkRESTXMLRequest_XML_S3PutObjectAcl-8         466            331            -28.97%

benchmark                                            old bytes     new bytes     delta
BenchmarkRESTXMLBuild_Complex_CFCreateDistro-8       87893         27936         -68.22%
BenchmarkRESTXMLBuild_Simple_CFDeleteDistro-8        8104          7880          -2.76%
BenchmarkRESTXMLBuild_REST_S3HeadObject-8            9544          8800          -7.80%
BenchmarkRESTXMLBuild_XML_S3PutObjectAcl-8           17701         11704         -33.88%
BenchmarkRESTXMLRequest_Complex_CFCreateDistro-8     141892        80852         -43.02%
BenchmarkRESTXMLRequest_Simple_CFDeleteDistro-8      52773         52543         -0.44%
BenchmarkRESTXMLRequest_REST_S3HeadObject-8          61947         61193         -1.22%
BenchmarkRESTXMLRequest_XML_S3PutObjectAcl-8         70800         64671         -8.66%
```

models/protocol_tests/generate.go

@@ -358,6 +358,7 @@ func generateTestSuite(filename string) string {
 		svcPrefix := inout + "Service" + strconv.Itoa(i+1)
 		suite.API.Metadata.ServiceAbbreviation = svcPrefix + "ProtocolTest"
 		suite.API.Operations = map[string]*api.Operation{}
+
 		for idx, c := range suite.Cases {
 			c.Given.ExportedName = svcPrefix + "TestCaseOperation" + strconv.Itoa(idx+1)
 			suite.API.Operations[c.Given.ExportedName] = c.Given
@@ -370,6 +371,9 @@ func generateTestSuite(filename string) string {
 		suite.API.Setup()
 		suite.API.Metadata.EndpointPrefix = suite.API.PackageName()
 
+		// TODO until generated marshalers are all supported
+		suite.API.EnableSelectGeneratedMarshalers()
+
 		// Sort in order for deterministic test generation
 		names := make([]string, 0, len(suite.API.Shapes))
 		for n := range suite.API.Shapes {

private/model/api/api.go

@@ -46,6 +46,10 @@ type API struct {
 	// Set to true to not generate struct field accessors
 	NoGenStructFieldAccessors bool
 
+	// Set to true to not generate (un)marshalers
+	NoGenMarshalers   bool
+	NoGenUnmarshalers bool
+
 	SvcClientImportPath string
 
 	initialized bool
@@ -272,6 +276,9 @@ func (a *API) APIGoCode() string {
 		a.imports["github.com/aws/aws-sdk-go/private/protocol/"+a.ProtocolPackage()] = true
 		a.imports["github.com/aws/aws-sdk-go/private/protocol"] = true
 	}
+	if !a.NoGenMarshalers || !a.NoGenUnmarshalers {
+		a.imports["github.com/aws/aws-sdk-go/private/protocol"] = true
+	}
 
 	for _, op := range a.Operations {
 		if op.AuthType == "none" {

private/model/api/customization_passes.go

@@ -47,6 +47,21 @@ func (a *API) customizationPasses() {
 	if fn := svcCustomizations[a.PackageName()]; fn != nil {
 		fn(a)
 	}
+
+	// TODO until generated marshalers are all supported
+	a.EnableSelectGeneratedMarshalers()
+}
+
+func (a *API) EnableSelectGeneratedMarshalers() {
+	// Selectivily enable generated marshalers as available
+	a.NoGenMarshalers = true
+	a.NoGenUnmarshalers = true
+
+	// Enable generated marshalers
+	switch a.Metadata.Protocol {
+	case "rest-xml":
+		a.NoGenMarshalers = false
+	}
 }
 
 // s3Customizations customizes the API generation to replace values specific to S3.

private/model/api/passes.go

@@ -61,6 +61,15 @@ func (a *API) resolveReferences() {
 			o.ErrorRefs[i].Shape.IsError = true
 		}
 	}
+
+	for _, s := range a.Shapes {
+		switch s.Type {
+		case "list":
+			s.MemberRef.Shape.UsedInList = true
+		case "map":
+			s.ValueRef.Shape.UsedInMap = true
+		}
+	}
 }
 
 // A referenceResolver provides a way to resolve shape references to

private/model/api/shape.go

@@ -78,6 +78,9 @@ type Shape struct {
 
 	OrigShapeName string `json:"-"`
 
+	UsedInMap  bool
+	UsedInList bool
+
 	// Defines if the shape is a placeholder and should not be used directly
 	Placeholder bool
 
@@ -185,10 +188,18 @@ func (s *Shape) GoStructValueType(name string, ref *ShapeRef) string {
 	return v
 }
 
+func (s *Shape) IsRefPayload(name string) bool {
+	return s.Payload == name
+}
+
+func (s *Shape) IsRefPayloadReader(name string, ref *ShapeRef) bool {
+	return (ref.Streaming || ref.Shape.Streaming) && s.IsRefPayload(name)
+}
+
 // GoStructType returns the type of a struct field based on the API
 // model definition.
 func (s *Shape) GoStructType(name string, ref *ShapeRef) string {
-	if (ref.Streaming || ref.Shape.Streaming) && s.Payload == name {
+	if s.IsRefPayloadReader(name, ref) {
 		rtype := "io.ReadSeeker"
 		if strings.HasSuffix(s.ShapeName, "Output") {
 			rtype = "io.ReadCloser"
@@ -513,7 +524,9 @@ func (s *Shape) NestedShape() *Shape {
 }
 
 var structShapeTmpl = template.Must(template.New("StructShape").Funcs(template.FuncMap{
-	"GetCrosslinkURL": GetCrosslinkURL,
+	"GetCrosslinkURL":      GetCrosslinkURL,
+	"MarshalShapeGoCode":   MarshalShapeGoCode,
+	"UnmarshalShapeGoCode": UnmarshalShapeGoCode,
 }).Parse(`
 {{ .Docstring }}
 {{ if ne $.OrigShapeName "" -}}
@@ -597,6 +610,13 @@ func (s *{{ $builderShapeName }}) get{{ $name }}() (v {{ $context.GoStructValueT
 
 {{ end }}
 {{ end }}
+
+{{ if not $.API.NoGenMarshalers -}}
+{{ MarshalShapeGoCode $ }}
+{{- end }}
+{{ if not $.API.NoGenUnmarshalers -}}
+{{ UnmarshalShapeGoCode $ }}
+{{- end }}
 `))
 
 var enumShapeTmpl = template.Must(template.New("EnumShape").Parse(`