From 1cbdb7de4784cb0310083f0b6d3b991de516c032 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Wed, 9 Mar 2022 13:03:39 -0500 Subject: [PATCH] GH-1251: Jackson2JsonMessageConverter Improvements Resolves https://github.com/spring-projects/spring-amqp/issues/1420 - detect and use `charset` in `contentType` when present - allow Jackson to determine the decode `charset` via `ByteSourceJsonBootstrapper.detectEncoding()` - allow configuration of the `MimeType` to use, which can include a `charset` parameter **cherry-pick to main - will require what's new fix** * Fix typo in doc. # Conflicts: # src/reference/asciidoc/whats-new.adoc --- .../AbstractJackson2MessageConverter.java | 93 ++++++++++++++++--- .../Jackson2JsonMessageConverterTests.java | 65 ++++++++++++- src/reference/asciidoc/amqp.adoc | 23 +++++ src/reference/asciidoc/whats-new.adoc | 5 +- 4 files changed, 170 insertions(+), 16 deletions(-) diff --git a/spring-amqp/src/main/java/org/springframework/amqp/support/converter/AbstractJackson2MessageConverter.java b/spring-amqp/src/main/java/org/springframework/amqp/support/converter/AbstractJackson2MessageConverter.java index 8f3e11315e..5c102712d1 100644 --- a/spring-amqp/src/main/java/org/springframework/amqp/support/converter/AbstractJackson2MessageConverter.java +++ b/spring-amqp/src/main/java/org/springframework/amqp/support/converter/AbstractJackson2MessageConverter.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 the original author or authors. + * Copyright 2018-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ import org.springframework.util.Assert; import org.springframework.util.ClassUtils; import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; @@ -61,13 +62,18 @@ public abstract class AbstractJackson2MessageConverter extends AbstractMessageCo */ public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; + protected final ObjectMapper objectMapper; // NOSONAR protected + /** - * The supported content type; only the subtype is checked, e.g. */json, - * */xml. + * The supported content type; only the subtype is checked when decoding, e.g. + * */json, */xml. If this contains a charset parameter, when encoding, the + * contentType header will not be set, when decoding, the raw bytes are passed to + * Jackson which can dynamically determine the encoding; otherwise the contentEncoding + * or default charset is used. */ - private final MimeType supportedContentType; + private MimeType supportedContentType; - protected final ObjectMapper objectMapper; // NOSONAR protected + private String supportedCTCharset; @Nullable private ClassMapper classMapper = null; @@ -93,8 +99,11 @@ public abstract class AbstractJackson2MessageConverter extends AbstractMessageCo /** * Construct with the provided {@link ObjectMapper} instance. * @param objectMapper the {@link ObjectMapper} to use. - * @param contentType supported content type when decoding messages, only the subtype - * is checked, e.g. */json, */xml. + * @param contentType the supported content type; only the subtype is checked when + * decoding, e.g. */json, */xml. If this contains a charset parameter, when + * encoding, the contentType header will not be set, when decoding, the raw bytes are + * passed to Jackson which can dynamically determine the encoding; otherwise the + * contentEncoding or default charset is used. * @param trustedPackages the trusted Java packages for deserialization * @see DefaultJackson2JavaTypeMapper#setTrustedPackages(String...) */ @@ -105,9 +114,41 @@ protected AbstractJackson2MessageConverter(ObjectMapper objectMapper, MimeType c Assert.notNull(contentType, "'contentType' must not be null"); this.objectMapper = objectMapper; this.supportedContentType = contentType; + this.supportedCTCharset = this.supportedContentType.getParameter("charset"); ((DefaultJackson2JavaTypeMapper) this.javaTypeMapper).setTrustedPackages(trustedPackages); } + + /** + * Get the supported content type; only the subtype is checked when decoding, e.g. + * */json, */xml. If this contains a charset parameter, when encoding, the + * contentType header will not be set, when decoding, the raw bytes are passed to + * Jackson which can dynamically determine the encoding; otherwise the contentEncoding + * or default charset is used. + * @return the supportedContentType + * @since 2.4.3 + */ + protected MimeType getSupportedContentType() { + return this.supportedContentType; + } + + + /** + * Set the supported content type; only the subtype is checked when decoding, e.g. + * */json, */xml. If this contains a charset parameter, when encoding, the + * contentType header will not be set, when decoding, the raw bytes are passed to + * Jackson which can dynamically determine the encoding; otherwise the contentEncoding + * or default charset is used. + * @param supportedContentType the supportedContentType to set. + * @since 2.4.3 + */ + public void setSupportedContentType(MimeType supportedContentType) { + Assert.notNull(supportedContentType, "'supportedContentType' cannot be null"); + this.supportedContentType = supportedContentType; + this.supportedCTCharset = this.supportedContentType.getParameter("charset"); + } + + @Nullable public ClassMapper getClassMapper() { return this.classMapper; @@ -264,10 +305,7 @@ public Object fromMessage(Message message, @Nullable Object conversionHint) thro if ((this.assumeSupportedContentType // NOSONAR Boolean complexity && (contentType == null || contentType.equals(MessageProperties.DEFAULT_CONTENT_TYPE))) || (contentType != null && contentType.contains(this.supportedContentType.getSubtype()))) { - String encoding = properties.getContentEncoding(); - if (encoding == null) { - encoding = getDefaultCharset(); - } + String encoding = determineEncoding(properties, contentType); content = doFromMessage(message, conversionHint, properties, encoding); } else { @@ -283,6 +321,24 @@ public Object fromMessage(Message message, @Nullable Object conversionHint) thro return content; } + private String determineEncoding(MessageProperties properties, @Nullable String contentType) { + String encoding = properties.getContentEncoding(); + if (encoding == null && contentType != null) { + try { + MimeType mimeType = MimeTypeUtils.parseMimeType(contentType); + if (mimeType != null) { + encoding = mimeType.getParameter("charset"); + } + } + catch (RuntimeException e) { + } + } + if (encoding == null) { + encoding = this.supportedCTCharset != null ? this.supportedCTCharset : getDefaultCharset(); + } + return encoding; + } + private Object doFromMessage(Message message, Object conversionHint, MessageProperties properties, String encoding) { @@ -348,11 +404,17 @@ private Object tryConverType(Message message, String encoding, JavaType inferred } private Object convertBytesToObject(byte[] body, String encoding, JavaType targetJavaType) throws IOException { + if (this.supportedCTCharset != null) { // Jackson will determine encoding + return this.objectMapper.readValue(body, targetJavaType); + } String contentAsString = new String(body, encoding); return this.objectMapper.readValue(contentAsString, targetJavaType); } private Object convertBytesToObject(byte[] body, String encoding, Class targetClass) throws IOException { + if (this.supportedCTCharset != null) { // Jackson will determine encoding + return this.objectMapper.readValue(body, this.objectMapper.constructType(targetClass)); + } String contentAsString = new String(body, encoding); return this.objectMapper.readValue(contentAsString, this.objectMapper.constructType(targetClass)); } @@ -370,20 +432,23 @@ protected Message createMessage(Object objectToConvert, MessageProperties messag byte[] bytes; try { - if (this.charsetIsUtf8) { + if (this.charsetIsUtf8 && this.supportedCTCharset == null) { bytes = this.objectMapper.writeValueAsBytes(objectToConvert); } else { String jsonString = this.objectMapper .writeValueAsString(objectToConvert); - bytes = jsonString.getBytes(getDefaultCharset()); + String encoding = this.supportedCTCharset != null ? this.supportedCTCharset : getDefaultCharset(); + bytes = jsonString.getBytes(encoding); } } catch (IOException e) { throw new MessageConversionException("Failed to convert Message content", e); } messageProperties.setContentType(this.supportedContentType.toString()); - messageProperties.setContentEncoding(getDefaultCharset()); + if (this.supportedCTCharset == null) { + messageProperties.setContentEncoding(getDefaultCharset()); + } messageProperties.setContentLength(bytes.length); if (getClassMapper() == null) { diff --git a/spring-amqp/src/test/java/org/springframework/amqp/support/converter/Jackson2JsonMessageConverterTests.java b/spring-amqp/src/test/java/org/springframework/amqp/support/converter/Jackson2JsonMessageConverterTests.java index 19ec20ea4b..37e274d314 100644 --- a/spring-amqp/src/test/java/org/springframework/amqp/support/converter/Jackson2JsonMessageConverterTests.java +++ b/spring-amqp/src/test/java/org/springframework/amqp/support/converter/Jackson2JsonMessageConverterTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2020 the original author or authors. + * Copyright 2002-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package org.springframework.amqp.support.converter; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import java.io.IOException; import java.math.BigDecimal; @@ -34,6 +35,7 @@ import org.springframework.core.ParameterizedTypeReference; import org.springframework.data.web.JsonPath; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.util.MimeTypeUtils; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; @@ -399,6 +401,67 @@ void concreteInMapRegression() throws Exception { assertThat(foos.values().iterator().next().getField()).isEqualTo("baz"); } + @Test + void charsetInContentType() { + trade.setUserName("John Doe ∫"); + Jackson2JsonMessageConverter converter = new Jackson2JsonMessageConverter(); + String utf8 = "application/json;charset=utf-8"; + converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf8)); + Message message = converter.toMessage(trade, new MessageProperties()); + int bodyLength8 = message.getBody().length; + assertThat(message.getMessageProperties().getContentEncoding()).isNull(); + assertThat(message.getMessageProperties().getContentType()).isEqualTo(utf8); + SimpleTrade marshalledTrade = (SimpleTrade) converter.fromMessage(message); + assertThat(marshalledTrade).isEqualTo(trade); + + // use content type property + String utf16 = "application/json;charset=utf-16"; + converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf16)); + message = converter.toMessage(trade, new MessageProperties()); + assertThat(message.getBody().length).isNotEqualTo(bodyLength8); + assertThat(message.getMessageProperties().getContentEncoding()).isNull(); + assertThat(message.getMessageProperties().getContentType()).isEqualTo(utf16); + marshalledTrade = (SimpleTrade) converter.fromMessage(message); + assertThat(marshalledTrade).isEqualTo(trade); + + // no encoding in message, use configured default + converter.setSupportedContentType(MimeTypeUtils.parseMimeType("application/json")); + converter.setDefaultCharset("UTF-16"); + message = converter.toMessage(trade, new MessageProperties()); + assertThat(message.getBody().length).isNotEqualTo(bodyLength8); + assertThat(message.getMessageProperties().getContentEncoding()).isNotNull(); + message.getMessageProperties().setContentEncoding(null); + marshalledTrade = (SimpleTrade) converter.fromMessage(message); + assertThat(marshalledTrade).isEqualTo(trade); + + } + + @Test + void noConfigForCharsetInContentType() { + trade.setUserName("John Doe ∫"); + Jackson2JsonMessageConverter converter = new Jackson2JsonMessageConverter(); + Message message = converter.toMessage(trade, new MessageProperties()); + int bodyLength8 = message.getBody().length; + SimpleTrade marshalledTrade = (SimpleTrade) converter.fromMessage(message); + assertThat(marshalledTrade).isEqualTo(trade); + + // no encoding in message; use configured default + message = converter.toMessage(trade, new MessageProperties()); + assertThat(message.getMessageProperties().getContentEncoding()).isNotNull(); + message.getMessageProperties().setContentEncoding(null); + marshalledTrade = (SimpleTrade) converter.fromMessage(message); + assertThat(marshalledTrade).isEqualTo(trade); + + converter.setDefaultCharset("UTF-16"); + Message message2 = converter.toMessage(trade, new MessageProperties()); + message2.getMessageProperties().setContentEncoding(null); + assertThat(message2.getBody().length).isNotEqualTo(bodyLength8); + converter.setDefaultCharset("UTF-8"); + + assertThatExceptionOfType(MessageConversionException.class).isThrownBy( + () -> converter.fromMessage(message2)); + } + public List fooLister() { return null; } diff --git a/src/reference/asciidoc/amqp.adoc b/src/reference/asciidoc/amqp.adoc index 05675d1b1e..9c40006343 100644 --- a/src/reference/asciidoc/amqp.adoc +++ b/src/reference/asciidoc/amqp.adoc @@ -3938,11 +3938,34 @@ public DefaultClassMapper classMapper() { Now, if the sending system sets the header to `thing1`, the converter creates a `Thing1` object, and so on. See the <> sample application for a complete discussion about converting messages from non-Spring applications. +Starting with version 2.4.3, the converter will not add a `contentEncoding` message property if the `supportedMediaType` has a `charset` parameter; this is also used for the encoding. +A new method `setSupportedMediaType` has been added: + +==== +[source, java] +---- +String utf16 = "application/json; charset=utf-16"; +converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf16)); +---- +==== + [[Jackson2JsonMessageConverter-from-message]] ====== Converting from a `Message` Inbound messages are converted to objects according to the type information added to headers by the sending system. +Starting with version 2.4.3, if there is no `contentEncoding` message property, the converter will attempt to detect a `charset` parameter in the `contentType` message property and use that. +If neither exist, if the `supportedMediaType` has a `charset` parameter, it will be used for decoding, with a final fallback to the `defaultCharset` property. +A new method `setSupportedMediaType` has been added: + +==== +[source, java] +---- +String utf16 = "application/json; charset=utf-16"; +converter.setSupportedContentType(MimeTypeUtils.parseMimeType(utf16)); +---- +==== + In versions prior to 1.6, if type information is not present, conversion would fail. Starting with version 1.6, if type information is missing, the converter converts the JSON by using Jackson defaults (usually a map). diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index b9d83a054b..d216446a8c 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -11,4 +11,7 @@ This version requires Spring Framework 6.0 and Java 17 The remoting feature (using RMI) is no longer supported. -See <> for alternatives. +==== Message Converter Changes + +The `Jackson2JsonMessageConverter` can now determine the charset from the `contentEncoding` header. +See <> for more information. \ No newline at end of file