From cc8cb0b57d67296342c3cab309a37beae5462810 Mon Sep 17 00:00:00 2001 From: Michael Dowling Date: Tue, 29 Sep 2020 00:37:20 -0700 Subject: [PATCH] Add media type parser and mediaType validation A media type parser is needed to parse Smithy mediaType traits both for codegen and also for validation. --- .../validators/MediaTypeValidator.java | 48 +++ ...e.amazon.smithy.model.validation.Validator | 1 + .../validators/mediatype-trait.errors | 7 + .../validators/mediatype-trait.json | 71 +++++ .../amazon/smithy/utils/MediaType.java | 278 ++++++++++++++++++ .../amazon/smithy/utils/MediaTypeTest.java | 158 ++++++++++ 6 files changed, 563 insertions(+) create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/validation/validators/MediaTypeValidator.java create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.errors create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.json create mode 100644 smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java create mode 100644 smithy-utils/src/test/java/software/amazon/smithy/utils/MediaTypeTest.java diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/validation/validators/MediaTypeValidator.java b/smithy-model/src/main/java/software/amazon/smithy/model/validation/validators/MediaTypeValidator.java new file mode 100644 index 00000000000..cff9db911eb --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/validation/validators/MediaTypeValidator.java @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.validation.validators; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.traits.MediaTypeTrait; +import software.amazon.smithy.model.validation.AbstractValidator; +import software.amazon.smithy.model.validation.ValidationEvent; +import software.amazon.smithy.utils.MediaType; + +public final class MediaTypeValidator extends AbstractValidator { + @Override + public List validate(Model model) { + List events = new ArrayList<>(); + for (Shape shape : model.getShapesWithTrait(MediaTypeTrait.class)) { + validateMediaType(shape, shape.expectTrait(MediaTypeTrait.class)).ifPresent(events::add); + } + + return events; + } + + private Optional validateMediaType(Shape shape, MediaTypeTrait trait) { + try { + MediaType.from(trait.getValue()); + return Optional.empty(); + } catch (RuntimeException e) { + return Optional.of(error(shape, trait, String.format( + "Invalid mediaType value, \"%s\": %s", trait.getValue(), e.getMessage()))); + } + } +} diff --git a/smithy-model/src/main/resources/META-INF/services/software.amazon.smithy.model.validation.Validator b/smithy-model/src/main/resources/META-INF/services/software.amazon.smithy.model.validation.Validator index 26a3c2f3b73..c4106c59e47 100644 --- a/smithy-model/src/main/resources/META-INF/services/software.amazon.smithy.model.validation.Validator +++ b/smithy-model/src/main/resources/META-INF/services/software.amazon.smithy.model.validation.Validator @@ -15,6 +15,7 @@ software.amazon.smithy.model.validation.validators.HttpQueryTraitValidator software.amazon.smithy.model.validation.validators.HttpResponseCodeSemanticsValidator software.amazon.smithy.model.validation.validators.HttpUriConflictValidator software.amazon.smithy.model.validation.validators.LengthTraitValidator +software.amazon.smithy.model.validation.validators.MediaTypeValidator software.amazon.smithy.model.validation.validators.NoInlineDocumentSupportValidator software.amazon.smithy.model.validation.validators.PaginatedTraitValidator software.amazon.smithy.model.validation.validators.PrivateAccessValidator diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.errors new file mode 100644 index 00000000000..a131223619b --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.errors @@ -0,0 +1,7 @@ +[ERROR] smithy.example#Invalid1: Invalid mediaType value, " | MediaType +[ERROR] smithy.example#Invalid2: Invalid mediaType value, " | MediaType +[ERROR] smithy.example#Invalid3: Invalid mediaType value, " | MediaType +[ERROR] smithy.example#Invalid4: Invalid mediaType value, " | MediaType +[ERROR] smithy.example#Invalid5: Invalid mediaType value, " | MediaType +[ERROR] smithy.example#Invalid6: Invalid mediaType value, " | MediaType +[ERROR] smithy.example#Invalid7: Invalid mediaType value, " | MediaType diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.json b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.json new file mode 100644 index 00000000000..4a5ffaaada2 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/mediatype-trait.json @@ -0,0 +1,71 @@ +{ + "smithy": "1.0", + "shapes": { + "smithy.example#Valid1": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/json" + } + }, + "smithy.example#Valid2": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/foo+json" + } + }, + "smithy.example#Valid3": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/foo+json; bar=baz" + } + }, + "smithy.example#Valid4": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/foo+json; bar=baz; bam=boozled; foo=\"hi\"" + } + }, + "smithy.example#Invalid1": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application" + } + }, + "smithy.example#Invalid2": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/" + } + }, + "smithy.example#Invalid3": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/json;" + } + }, + "smithy.example#Invalid4": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/json; bar" + } + }, + "smithy.example#Invalid5": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/json; bar=" + } + }, + "smithy.example#Invalid6": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/json; bar=," + } + }, + "smithy.example#Invalid7": { + "type": "string", + "traits": { + "smithy.api#mediaType": "application/json; bar=bam;" + } + } + } +} diff --git a/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java b/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java new file mode 100644 index 00000000000..4a107f23c18 --- /dev/null +++ b/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java @@ -0,0 +1,278 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.utils; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Implements a simple media type parser based on the Content-Type grammar defined in + * RFC 7231. + * + *

The type, subtype, and parameter names are all canonicalized to + * lowercase strings. + */ +public final class MediaType { + + private final String value; + private final String type; + private final String subtype; + private final Map parameters; + + private MediaType(String value, String type, String subtype, Map parameters) { + this.value = value; + this.type = type; + this.subtype = subtype; + this.parameters = Collections.unmodifiableMap(parameters); + } + + /** + * Create a parsed MediaType from the given string. + * + * @param value Media type to parse (e.g., application/json). + * @return Returns the parsed media type. + * @throws RuntimeException if the media type is invalid. + */ + public static MediaType from(String value) { + Parser parser = new Parser(value); + parser.parse(); + return new MediaType(parser.expression(), parser.type, parser.subtype, parser.parameters); + } + + /** + * Detects if the given media type string is JSON, meaning it + * is "application/json" or uses the "+json" structured syntax + * suffix. + * + * @param mediaType Media type to parse and test if it's JSON. + * @return Returns true if the given media type is JSON. + */ + public static boolean isJson(String mediaType) { + MediaType type = from(mediaType); + return (type.getType().equals("application") && type.getSubtypeWithoutSuffix().equals("json")) + || type.getSuffix().filter(s -> s.equals("json")).isPresent(); + } + + /** + * Gets the "type" of the media type. + * + * @return Returns the type (e.g., "application"). + */ + public String getType() { + return type; + } + + /** + * Gets the "subtype" of the media type. + * + * @return Returns the subtype (e.g., "json", "foo+json"). + */ + public String getSubtype() { + return subtype; + } + + /** + * Gets the "subtype" of the media type with no structured syntax suffix. + * + *

For example given, "application/foo+json", this method returns + * "foo". Given "application/foo+baz+json", this method returns + * "foo+baz". + * + * @return Returns the subtype (e.g., "json", "foo+json"). + */ + public String getSubtypeWithoutSuffix() { + int position = subtype.lastIndexOf('+'); + return position == -1 ? getSubtype() : subtype.substring(0, position); + } + + /** + * Gets the immutable map of parameters. + * + * @return Returns the parameters. + */ + public Map getParameters() { + return parameters; + } + + /** + * Gets the optional structured syntax suffix. + * + *

For example given, "application/foo+json", this method returns + * "json". Given "application/foo+baz+json", this method returns + * "json". Given "application/json", this method returns an empty + * {@code Optional}. + * + * @return Returns the optional structured syntax suffix value with no "+". + */ + public Optional getSuffix() { + int position = subtype.lastIndexOf('+'); + return position == -1 || position == subtype.length() - 1 + ? Optional.empty() + : Optional.of(subtype.substring(position + 1)); + } + + @Override + public String toString() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (!(o instanceof MediaType)) { + return false; + } + MediaType mediaType = (MediaType) o; + return mediaType.type.equals(type) + && mediaType.subtype.equals(subtype) + && mediaType.parameters.equals(parameters); + } + + @Override + public int hashCode() { + return Objects.hash(type, subtype, parameters); + } + + private static final class Parser extends SimpleParser { + // See https://tools.ietf.org/html/rfc7230#section-3.2.6 + // token = 1*tchar + // tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + // / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + // / DIGIT / ALPHA + // ; any VCHAR, except delimiters + private static final Set TOKEN = new LinkedHashSet<>(); + + static { + TOKEN.add('!'); + TOKEN.add('#'); + TOKEN.add('$'); + TOKEN.add('%'); + TOKEN.add('&'); + TOKEN.add('\''); + TOKEN.add('*'); + TOKEN.add('+'); + TOKEN.add('-'); + TOKEN.add('.'); + TOKEN.add('^'); + TOKEN.add('_'); + TOKEN.add('`'); + TOKEN.add('|'); + TOKEN.add('~'); + for (char c = '0'; c <= '9'; c++) { + TOKEN.add(c); + } + for (char c = 'a'; c <= 'z'; c++) { + TOKEN.add(c); + } + for (char c = 'A'; c <= 'Z'; c++) { + TOKEN.add(c); + } + } + + private String type; + private String subtype; + private final Map parameters = new LinkedHashMap<>(); + + Parser(String value) { + super(value); + } + + private void parse() { + // From: https://tools.ietf.org/html/rfc7231#section-3.1.1.1 + // The type, subtype, and parameter name tokens are case-insensitive. + // media-type = type "/" subtype *( OWS ";" OWS parameter ) + // type = token. + type = parseToken().toLowerCase(Locale.US); + expect('/'); + + // subtype = token + subtype = parseToken().toLowerCase(Locale.US); + ws(); + + // parameter = token "=" ( token / quoted-string ) + while (!eof()) { + expect(';'); + ws(); + String name = parseToken().toLowerCase(Locale.US); + expect('='); + String value = peek() == '"' + ? parseQuotedString() + : parseToken(); + parameters.put(name, value); + ws(); + } + } + + private String parseToken() { + int start = position(); + consumeUntilNoLongerMatches(TOKEN::contains); + + // Fail if the token was empty. + if (start == position()) { + char[] chars = new char[TOKEN.size()]; + int i = 0; + for (Character character : TOKEN) { + chars[i++] = character; + } + expect(chars); + } + + return sliceFrom(start); + } + + // See https://tools.ietf.org/html/rfc7230#section-3.2.6 + // quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + // qdtext = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text + // obs-text = %x80-FF + // quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + private String parseQuotedString() { + StringBuilder result = new StringBuilder(); + expect('"'); + + while (!eof()) { + char next = peek(); + if (next == '"') { + break; + } + skip(); + + // The backslash octet ("\") can be used as a single-octet quoting + // mechanism within quoted-string and comment constructs. Recipients + // that process the value of a quoted-string MUST handle a quoted-pair + // as if it were replaced by the octet following the backslash. + if (next == '\\') { + if (eof()) { + throw syntax("Expected character after escape"); + } + result.append(peek()); + skip(); + } else { + result.append(next); + } + } + + expect('"'); + return result.toString(); + } + } +} diff --git a/smithy-utils/src/test/java/software/amazon/smithy/utils/MediaTypeTest.java b/smithy-utils/src/test/java/software/amazon/smithy/utils/MediaTypeTest.java new file mode 100644 index 00000000000..6d66df307dc --- /dev/null +++ b/smithy-utils/src/test/java/software/amazon/smithy/utils/MediaTypeTest.java @@ -0,0 +1,158 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.utils; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MediaTypeTest { + @Test + public void requiresNonEmptyString() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("")); + } + + @Test + public void requiresValidTypeToken() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from(",")); + } + + @Test + public void requiresSubtype() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application")); + + assertThat(e.getMessage(), containsString("Expected: '/'")); + } + + @Test + public void requiresSubtypeToken() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application/")); + } + + @Test + public void requiresParametersAfterSemicolon() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application/json;")); + } + + @Test + public void requiresValidParameterName() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application/json; \\")); + } + + @Test + public void requiresEqualsAfterParameterName() { + RuntimeException e = Assertions.assertThrows( + RuntimeException.class, () -> MediaType.from("application/json; foo")); + + assertThat(e.getMessage(), containsString("Expected: '='")); + } + + @Test + public void requiresParameterValueAfterEquals() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application/json; foo=")); + } + + @Test + public void requiresParameterValueClosingQuote() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application/json; foo=\"baz")); + } + + @Test + public void requiresParameterValueAfterOtherParamsAndSemicolon() { + Assertions.assertThrows(RuntimeException.class, () -> MediaType.from("application/json; foo=\"baz\"; ")); + } + + @Test + public void detectsInvalidQuotedEscapes() { + RuntimeException e = Assertions.assertThrows( + RuntimeException.class, () -> MediaType.from("application/json; foo=\"bar\\")); + + assertThat(e.getMessage(), containsString("Expected character after escape")); + } + + @Test + public void detectsJsonMediaTypes() { + assertThat(MediaType.isJson("application/json"), is(true)); + assertThat(MediaType.isJson("application/foo+json"), is(true)); + assertThat(MediaType.isJson("foo/json"), is(false)); + assertThat(MediaType.isJson("application/jsonn"), is(false)); + assertThat(MediaType.isJson("application/foo+jsonn"), is(false)); + } + + @Test + public void parsesSuffix() { + assertThat(MediaType.from("foo/baz+").getSubtypeWithoutSuffix(), equalTo("baz")); + assertThat(MediaType.from("foo/baz+").getSuffix(), equalTo(Optional.empty())); + assertThat(MediaType.from("foo/baz+bam+boo").getSuffix(), equalTo(Optional.of("boo"))); + assertThat(MediaType.from("foo/baz+bam+boo").getSubtypeWithoutSuffix(), equalTo("baz+bam")); + } + + @Test + public void parsesMediaType() { + String typeString = "foo/baz; bar=abc;BAM=\"100\" ; _a=_"; + MediaType type = MediaType.from(typeString); + + assertThat(typeString, equalTo(type.toString())); + + assertThat(type.getType(), equalTo("foo")); + assertThat(type.getSubtype(), equalTo("baz")); + + assertThat(type.getParameters(), hasKey("bar")); + assertThat(type.getParameters(), hasKey("bam")); + assertThat(type.getParameters(), hasKey("_a")); + + assertThat(type.getParameters().get("bar"), equalTo("abc")); + assertThat(type.getParameters().get("bam"), equalTo("100")); + assertThat(type.getParameters().get("_a"), equalTo("_")); + } + + @Test + public void hashCodeAndEquals() { + String typeString = "foo/baz; bar=abc;BAM=\"100\" ; _a=_"; + MediaType type = MediaType.from(typeString); + MediaType type2 = MediaType.from(typeString); + MediaType type3 = MediaType.from("foo/baz; bar=abc"); + + assertThat(type, equalTo(type)); + assertThat(type, equalTo(type2)); + assertThat(type.hashCode(), equalTo(type2.hashCode())); + + assertThat(type, not(equalTo(type3))); + assertThat(type.hashCode(), not(equalTo(type3.hashCode()))); + assertThat(type, not(equalTo("foo"))); + } + + @Test + public void allowsEscapedQuotes() { + MediaType type = MediaType.from("foo/baz; bar=\"foo\\\"baz\""); + + assertThat(type.getParameters().get("bar"), equalTo("foo\"baz")); + } + + @Test + public void allowsSpecialStuffInQuotes() { + MediaType type = MediaType.from("foo/baz; bar=\";\""); + + assertThat(type.getParameters().get("bar"), equalTo(";")); + } +}