Add tags and appliesTo to protocol tests

This commit adds support for applying tags to individual test cases, allowing them to be grouped much more granularly. It also adds support for appliesTo, which allows test cases to indicate if they should only be implemented by clients or servers. Test cases that do not define an appliesTo member are to be implemented by both client and server implementations. Existing AWS protocol tests are updated to now utilize appliesTo, and the tags are left as-is. These tags will be removed in the future once implementations have migrated away from them.
smithy-lang · Jan 28, 2021 · 9034f24 · 9034f24
1 parent 44f5afb
commit 9034f24
Show file tree

Hide file tree

Showing 26 changed files with 478 additions and 31 deletions.
diff --git a/docs/source/1.0/spec/http-protocol-compliance-tests.rst b/docs/source/1.0/spec/http-protocol-compliance-tests.rst
@@ -202,6 +202,21 @@ that support the following members:
       - ``string``
       - A description of the test and what is being asserted defined in
         CommonMark_.
+    * - tags
+      - ``[string]``
+      - Attaches a list of tags that allow test cases to be categorized and
+        grouped.
+    * - appliesTo
+      - ``string``, one of "client" or "server"
+      - Indicates that the test case is only to be implemented by "client" or
+        "server" implementations. This property is useful for identifying and
+        testing edge cases of clients and servers that are impossible or
+        undesirable to test in *both* client and server implementations. For
+        example, a "server" test might be useful to ensure a service can
+        gracefully receive a request that optionally contains a payload.
+
+        Is is assumed that test cases that do not define an ``appliesTo``
+        member are implemented by both client and server implementations.
 
 
 HTTP request example
@@ -410,6 +425,21 @@ structures that support the following members:
       - ``string``
       - A description of the test and what is being asserted defined in
         CommonMark_.
+    * - tags
+      - ``[string]``
+      - Attaches a list of tags that allow test cases to be categorized and
+        grouped.
+    * - appliesTo
+      - ``string``, one of "client" or "server"
+      - Indicates that the test case is only to be implemented by "client" or
+        "server" implementations. This property is useful for identifying and
+        testing edge cases of clients and servers that are impossible or
+        undesirable to test in *both* client and server implementations. For
+        example, a "client" test might be useful to ensure a client can
+        gracefully receive a response that optionally contains a payload.
+
+        Is is assumed that test cases that do not define an ``appliesTo``
+        member are implemented by both client and server implementations.
 
 
 HTTP response example

diff --git a/smithy-aws-protocol-tests/model/awsJson1_0/errors.smithy b/smithy-aws-protocol-tests/model/awsJson1_0/errors.smithy
@@ -121,6 +121,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "FooError",
         },
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorUsingXAmznErrorTypeWithUri",
@@ -135,6 +136,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
         },
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorUsingXAmznErrorTypeWithUriAndNamespace",
@@ -146,6 +148,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "aws.protocoltests.json10#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
         },
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorUsingCode",
@@ -165,6 +168,7 @@ apply FooError @httpResponseTests([
                   "code": "FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorUsingCodeAndNamespace",
@@ -181,6 +185,7 @@ apply FooError @httpResponseTests([
                   "code": "aws.protocoltests.json10#FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorUsingCodeUriAndNamespace",
@@ -198,6 +203,7 @@ apply FooError @httpResponseTests([
                   "code": "aws.protocoltests.json10#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorWithDunderType",
@@ -212,6 +218,7 @@ apply FooError @httpResponseTests([
                   "__type": "FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorWithDunderTypeAndNamespace",
@@ -228,6 +235,7 @@ apply FooError @httpResponseTests([
                   "__type": "aws.protocoltests.json10#FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson10FooErrorWithDunderTypeUriAndNamespace",
@@ -245,5 +253,6 @@ apply FooError @httpResponseTests([
                   "__type": "aws.protocoltests.json10#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     }
 ])
diff --git a/smithy-aws-protocol-tests/model/awsJson1_1/errors.smithy b/smithy-aws-protocol-tests/model/awsJson1_1/errors.smithy
@@ -120,6 +120,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "FooError",
         },
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorUsingXAmznErrorTypeWithUri",
@@ -134,6 +135,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
         },
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorUsingXAmznErrorTypeWithUriAndNamespace",
@@ -145,6 +147,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
         },
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorUsingCode",
@@ -164,6 +167,7 @@ apply FooError @httpResponseTests([
                   "code": "FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorUsingCodeAndNamespace",
@@ -180,6 +184,7 @@ apply FooError @httpResponseTests([
                   "code": "aws.protocoltests.restjson#FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorUsingCodeUriAndNamespace",
@@ -197,6 +202,7 @@ apply FooError @httpResponseTests([
                   "code": "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorWithDunderType",
@@ -211,6 +217,7 @@ apply FooError @httpResponseTests([
                   "__type": "FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorWithDunderTypeAndNamespace",
@@ -227,6 +234,7 @@ apply FooError @httpResponseTests([
                   "__type": "aws.protocoltests.restjson#FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "AwsJson11FooErrorWithDunderTypeUriAndNamespace",
@@ -244,5 +252,6 @@ apply FooError @httpResponseTests([
                   "__type": "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     }
 ])
diff --git a/smithy-aws-protocol-tests/model/awsQuery/input.smithy b/smithy-aws-protocol-tests/model/awsQuery/input.smithy
@@ -300,6 +300,7 @@ apply QueryIdempotencyTokenAutoFill @httpRequestTests([
               &Version=2020-01-08
               &token=00000000-0000-4000-8000-000000000000""",
         bodyMediaType: "application/x-www-form-urlencoded",
+        appliesTo: "client",
     },
     {
         id: "QueryProtocolIdempotencyTokenAutoFillIsSet",
@@ -320,7 +321,8 @@ apply QueryIdempotencyTokenAutoFill @httpRequestTests([
         bodyMediaType: "application/x-www-form-urlencoded",
         params: {
             token: "00000000-0000-4000-8000-000000000123"
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/awsQuery/xml-lists.smithy b/smithy-aws-protocol-tests/model/awsQuery/xml-lists.smithy
@@ -154,7 +154,8 @@ apply XmlEmptyLists @httpResponseTests([
         params: {
             stringList: [],
             stringSet: [],
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/awsQuery/xml-maps.smithy b/smithy-aws-protocol-tests/model/awsQuery/xml-maps.smithy
@@ -83,7 +83,8 @@ apply XmlEmptyMaps @httpResponseTests([
         },
         params: {
             myMap: {}
-        }
+        },
+        appliesTo: "client",
     },
     {
         id: "QueryXmlEmptySelfClosedMaps",
@@ -103,7 +104,8 @@ apply XmlEmptyMaps @httpResponseTests([
         },
         params: {
             myMap: {}
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/awsQuery/xml-structs.smithy b/smithy-aws-protocol-tests/model/awsQuery/xml-structs.smithy
@@ -126,7 +126,8 @@ apply XmlEmptyBlobs @httpResponseTests([
         },
         params: {
             data: ""
-        }
+        },
+        appliesTo: "client",
     },
     {
         id: "QueryXmlEmptySelfClosedBlobs",
@@ -146,7 +147,8 @@ apply XmlEmptyBlobs @httpResponseTests([
         },
         params: {
             data: ""
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/ec2Query/input.smithy b/smithy-aws-protocol-tests/model/ec2Query/input.smithy
@@ -374,6 +374,7 @@ apply QueryIdempotencyTokenAutoFill @httpRequestTests([
               &Version=2020-01-08
               &Token=00000000-0000-4000-8000-000000000000""",
         bodyMediaType: "application/x-www-form-urlencoded",
+        appliesTo: "client",
     },
     {
         id: "Ec2ProtocolIdempotencyTokenAutoFillIsSet",
@@ -394,7 +395,8 @@ apply QueryIdempotencyTokenAutoFill @httpRequestTests([
         bodyMediaType: "application/x-www-form-urlencoded",
         params: {
             token: "00000000-0000-4000-8000-000000000123"
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/ec2Query/xml-lists.smithy b/smithy-aws-protocol-tests/model/ec2Query/xml-lists.smithy
@@ -152,7 +152,8 @@ apply XmlEmptyLists @httpResponseTests([
         params: {
             stringList: [],
             stringSet: [],
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/ec2Query/xml-structs.smithy b/smithy-aws-protocol-tests/model/ec2Query/xml-structs.smithy
@@ -127,7 +127,8 @@ apply XmlEmptyBlobs @httpResponseTests([
         },
         params: {
             data: ""
-        }
+        },
+        appliesTo: "client",
     },
     {
         id: "Ec2XmlEmptySelfClosedBlobs",
@@ -146,7 +147,8 @@ apply XmlEmptyBlobs @httpResponseTests([
         },
         params: {
             data: ""
-        }
+        },
+        appliesTo: "client",
     }
 ])
 

diff --git a/smithy-aws-protocol-tests/model/restJson1/errors.smithy b/smithy-aws-protocol-tests/model/restJson1/errors.smithy
@@ -155,6 +155,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "FooError",
         },
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorUsingXAmznErrorTypeWithUri",
@@ -169,6 +170,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
         },
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorUsingXAmznErrorTypeWithUriAndNamespace",
@@ -180,6 +182,7 @@ apply FooError @httpResponseTests([
         headers: {
             "X-Amzn-Errortype": "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
         },
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorUsingCode",
@@ -199,6 +202,7 @@ apply FooError @httpResponseTests([
                   "code": "FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorUsingCodeAndNamespace",
@@ -215,6 +219,7 @@ apply FooError @httpResponseTests([
                   "code": "aws.protocoltests.restjson#FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorUsingCodeUriAndNamespace",
@@ -232,6 +237,7 @@ apply FooError @httpResponseTests([
                   "code": "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorWithDunderType",
@@ -246,6 +252,7 @@ apply FooError @httpResponseTests([
                   "__type": "FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorWithDunderTypeAndNamespace",
@@ -262,6 +269,7 @@ apply FooError @httpResponseTests([
                   "__type": "aws.protocoltests.restjson#FooError"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     },
     {
         id: "RestJsonFooErrorWithDunderTypeUriAndNamespace",
@@ -279,5 +287,6 @@ apply FooError @httpResponseTests([
                   "__type": "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/"
               }""",
         bodyMediaType: "application/json",
+        appliesTo: "client",
     }
 ])
diff --git a/smithy-aws-protocol-tests/model/restJson1/http-headers.smithy b/smithy-aws-protocol-tests/model/restJson1/http-headers.smithy
@@ -284,7 +284,8 @@ apply NullAndEmptyHeadersClient @httpRequestTests([
             a: null,
             b: "",
             c: [],
-        }
+        },
+        appliesTo: "client",
     },
 ])
 
@@ -309,7 +310,8 @@ apply NullAndEmptyHeadersServer @httpResponseTests([
             a: null,
             b: "",
             c: [],
-        }
+        },
+        appliesTo: "server",
     },
 ])
-Original file line number
+Diff line change
@@ Expand Up / @@ -284,7 +284,8 @@ apply NullAndEmptyHeadersClient @httpRequestTests([ @@
                 a: null,
                 b: "",
                 c: [],
-            }
+            },
+            appliesTo: "client",
         },
     ])
@@ Expand All / @@ -309,7 +310,8 @@ apply NullAndEmptyHeadersServer @httpResponseTests([ @@
                 a: null,
                 b: "",
                 c: [],
-            }
+            },
+            appliesTo: "server",
         },
     ])
@@ Expand Down @@