diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java index 141a987f6f..2c9db32c97 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java @@ -62,8 +62,13 @@ import co.elastic.apm.agent.impl.transaction.TransactionImpl; import co.elastic.apm.agent.report.ApmServerClient; import co.elastic.apm.agent.sdk.internal.collections.LongList; +import co.elastic.apm.agent.sdk.internal.pooling.ObjectHandle; +import co.elastic.apm.agent.sdk.internal.pooling.ObjectPool; +import co.elastic.apm.agent.sdk.internal.pooling.ObjectPooling; +import co.elastic.apm.agent.sdk.internal.util.IOUtils; import co.elastic.apm.agent.sdk.logging.Logger; import co.elastic.apm.agent.sdk.logging.LoggerFactory; +import co.elastic.apm.agent.tracer.configuration.WebConfiguration; import co.elastic.apm.agent.tracer.metadata.PotentiallyMultiValuedMap; import co.elastic.apm.agent.tracer.metrics.DslJsonUtil; import co.elastic.apm.agent.tracer.metrics.Labels; @@ -79,8 +84,10 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.CharBuffer; +import java.nio.charset.CoderResult; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -88,6 +95,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -104,6 +112,13 @@ public class DslJsonSerializer { private static final Logger logger = LoggerFactory.getLogger(DslJsonSerializer.class); private static final List excludedStackFramesPrefixes = Arrays.asList("java.lang.reflect.", "com.sun.", "sun.", "jdk.internal."); + private static final ObjectPool> REQUEST_BODY_BUFFER_POOL = ObjectPooling.createWithDefaultFactory(new Callable() { + @Override + public CharBuffer call() throws Exception { + return CharBuffer.allocate(WebConfiguration.MAX_BODY_CAPTURE_BYTES); + } + }); + private final StacktraceConfigurationImpl stacktraceConfiguration; private final ApmServerClient apmServerClient; @@ -1054,22 +1069,7 @@ private void serializeSpanLinks(List spanLinks) { } private void serializeOTel(SpanImpl span) { - serializeOtel(span, Collections.emptyList(), requestBodyToString(span.getContext().getHttp().getRequestBody())); - } - - @Nullable - private CharSequence requestBodyToString(BodyCaptureImpl requestBody) { - //TODO: perform proper, charset aware conversion to string - ByteBuffer buffer = requestBody.getBody(); - if (buffer == null || buffer.position() == 0) { - return null; - } - buffer.flip(); - StringBuilder result = new StringBuilder(); - while (buffer.hasRemaining()) { - result.append((char) buffer.get()); - } - return result; + serializeOtel(span, Collections.emptyList(), span.getContext().getHttp().getRequestBody()); } private void serializeOTel(TransactionImpl transaction) { @@ -1079,11 +1079,12 @@ private void serializeOTel(TransactionImpl transaction) { } } - private void serializeOtel(AbstractSpanImpl span, List profilingStackTraceIds, @Nullable CharSequence httpRequestBody) { + private void serializeOtel(AbstractSpanImpl span, List profilingStackTraceIds, @Nullable BodyCaptureImpl httpRequestBody) { OTelSpanKind kind = span.getOtelKind(); Map attributes = span.getOtelAttributes(); - boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || httpRequestBody != null; + boolean hasRequestBody = httpRequestBody != null && httpRequestBody.getBody() != null; + boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || hasRequestBody; boolean hasKind = kind != null; if (hasKind || hasAttributes) { writeFieldName("otel"); @@ -1133,12 +1134,12 @@ private void serializeOtel(AbstractSpanImpl span, List profilingStack } jw.writeByte(ARRAY_END); } - if (httpRequestBody != null) { + if (hasRequestBody) { if (!isFirstAttrib) { jw.writeByte(COMMA); } writeFieldName("http.request.body.content"); - jw.writeString(httpRequestBody); + writeRequestBodyAsString(jw, httpRequestBody); } jw.writeByte(OBJECT_END); } @@ -1148,6 +1149,38 @@ private void serializeOtel(AbstractSpanImpl span, List profilingStack } } + + private void writeRequestBodyAsString(JsonWriter jw, BodyCaptureImpl requestBody) { + try (ObjectHandle charBufferHandle = REQUEST_BODY_BUFFER_POOL.createInstance()) { + CharBuffer charBuffer = charBufferHandle.get(); + try { + decodeRequestBodyBytes(requestBody, charBuffer); + charBuffer.flip(); + jw.writeString(charBuffer); + } finally { + ((Buffer) charBuffer).clear(); + } + } + } + + private void decodeRequestBodyBytes(BodyCaptureImpl requestBody, CharBuffer charBuffer) { + ByteBuffer bodyBytes = requestBody.getBody(); + ((Buffer) bodyBytes).flip(); //make ready for reading + CharSequence charset = requestBody.getCharset(); + if (charset != null) { + CoderResult result = IOUtils.decode(bodyBytes, charBuffer, charset.toString()); + if (result != null && !result.isMalformed() && !result.isUnmappable()) { + return; + } + } + //fallback to decoding by simply casting bytes to chars + ((Buffer) bodyBytes).position(0); + ((Buffer) charBuffer).clear(); + while (bodyBytes.hasRemaining()) { + charBuffer.put((char) (((int) bodyBytes.get()) & 0xFF)); + } + } + private void serializeNumber(Number n, JsonWriter jw) { if (n instanceof Integer) { NumberConverter.serialize(n.intValue(), jw); diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java index 7006fd0fa8..e2c76649aa 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java @@ -57,6 +57,7 @@ import co.elastic.apm.agent.report.ApmServerClient; import co.elastic.apm.agent.sdk.internal.collections.LongList; import co.elastic.apm.agent.sdk.internal.util.IOUtils; +import co.elastic.apm.agent.tracer.configuration.WebConfiguration; import com.dslplatform.json.JsonWriter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -74,6 +75,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -432,18 +434,53 @@ void testSpanHttpContextSerialization() { } @Test - void testSpanHttpRequestBodySerialization() { - SpanImpl span = new SpanImpl(tracer); + void testSpanHttpRequestBodySerialization() throws UnsupportedEncodingException { + String noBody = extractRequestBodyJson(createSpanWithRequestBody(null, "utf-8")); + assertThat(noBody).isNull(); - BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody(); - bodyCapture.markEligibleForCapturing(); - bodyCapture.startCapture("utf-8", 50); - bodyCapture.append("foobar".getBytes(StandardCharsets.UTF_8), 0, 6); + String emptyBody = extractRequestBodyJson(createSpanWithRequestBody(new byte[0], "utf-8")); + assertThat(emptyBody).isEqualTo(""); + + String invalidCharset = extractRequestBodyJson(createSpanWithRequestBody("testö".getBytes("utf-8"), "bad charset!")); + assertThat(invalidCharset).isEqualTo("testö"); + + String noCharset = extractRequestBodyJson(createSpanWithRequestBody("testö".getBytes("utf-8"), null)); + assertThat(noCharset).isEqualTo("testö"); + + String utf8 = extractRequestBodyJson(createSpanWithRequestBody("special charßßß!äöü".getBytes("utf-8"), "utf-8")); + assertThat(utf8).isEqualTo("special charßßß!äöü"); + + String utf16 = extractRequestBodyJson(createSpanWithRequestBody("special charßßß!äöü".getBytes("utf-16"), "utf-16")); + assertThat(utf16).isEqualTo("special charßßß!äöü"); + + String invalidUtf8Sequence = extractRequestBodyJson(createSpanWithRequestBody(new byte[]{'t', 'e', 's', 't', (byte) 0xC2, (byte) 0xC2}, "utf-8")); + assertThat(invalidUtf8Sequence).isEqualTo("testÂÂ"); + } + private String extractRequestBodyJson(SpanImpl span) { JsonNode spanJson = readJsonString(writer.toJsonString(span)); JsonNode otel = spanJson.get("otel"); + if (otel == null) { + return null; + } JsonNode attribs = otel.get("attributes"); - assertThat(attribs.get("http.request.body.content").textValue()).isEqualTo("foobar"); + JsonNode bodyContent = attribs.get("http.request.body.content"); + if (bodyContent == null) { + return null; + } + return bodyContent.textValue(); + } + + private SpanImpl createSpanWithRequestBody(@Nullable byte[] bodyBytes, @Nullable String charset) { + SpanImpl span = new SpanImpl(tracer); + BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody(); + bodyCapture.markEligibleForCapturing(); + bodyCapture.startCapture(charset, WebConfiguration.MAX_BODY_CAPTURE_BYTES); + + if (bodyBytes != null) { + bodyCapture.append(bodyBytes, 0, bodyBytes.length); + } + return span; } public static boolean[][] getContentCombinations() { diff --git a/apm-agent-plugin-sdk/src/main/java/co/elastic/apm/agent/sdk/internal/util/IOUtils.java b/apm-agent-plugin-sdk/src/main/java/co/elastic/apm/agent/sdk/internal/util/IOUtils.java index d432a08901..5a01194a40 100644 --- a/apm-agent-plugin-sdk/src/main/java/co/elastic/apm/agent/sdk/internal/util/IOUtils.java +++ b/apm-agent-plugin-sdk/src/main/java/co/elastic/apm/agent/sdk/internal/util/IOUtils.java @@ -23,32 +23,78 @@ import co.elastic.apm.agent.sdk.internal.pooling.ObjectPool; import co.elastic.apm.agent.sdk.internal.pooling.ObjectPooling; +import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.CharBuffer; +import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CoderResult; +import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; +import java.util.Collections; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; public class IOUtils { protected static final int BYTE_BUFFER_CAPACITY = 2048; - private static class DecoderWithBuffer { - final ByteBuffer byteBuffer = java.nio.ByteBuffer.allocate(BYTE_BUFFER_CAPACITY); - final CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder(); - } - - private static final ObjectPool> POOL = ObjectPooling.createWithDefaultFactory(new Callable() { + private static final ObjectPool> BYTE_BUFFER_POOL = ObjectPooling.createWithDefaultFactory(new Callable() { @Override - public DecoderWithBuffer call() throws Exception { - return new DecoderWithBuffer(); + public ByteBuffer call() throws Exception { + return ByteBuffer.allocate(BYTE_BUFFER_CAPACITY); } }); + private static final Set UNSUPPORTED_CHARSETS = Collections.newSetFromMap(new ConcurrentHashMap()); + private static final Map>> DECODER_POOLS + = new ConcurrentHashMap>>(); + + private static final String UTF8_CHARSET_NAME = StandardCharsets.UTF_8.name().toLowerCase(); + + @Nullable + private static ObjectHandle getPooledCharsetDecoder(String charsetName) { + if (!isLowerCase(charsetName)) { + charsetName = charsetName.toLowerCase(); + } + ObjectPool> decoderPool = DECODER_POOLS.get(charsetName); + if (decoderPool != null) { + return decoderPool.createInstance(); + } + if (UNSUPPORTED_CHARSETS.contains(charsetName)) { + return null; + } + try { + final Charset charset = Charset.forName(charsetName); + decoderPool = ObjectPooling.createWithDefaultFactory(new Callable() { + @Override + public CharsetDecoder call() throws Exception { + return charset.newDecoder(); + } + }); + DECODER_POOLS.put(charsetName, decoderPool); + return decoderPool.createInstance(); + } catch (IllegalCharsetNameException | UnsupportedCharsetException e) { + UNSUPPORTED_CHARSETS.add(charsetName); + return null; + } + } + + private static boolean isLowerCase(String charsetName) { + for (int i = 0; i < charsetName.length(); i++) { + if (!Character.isLowerCase(charsetName.charAt(i))) { + return false; + } + } + return true; + } + /** * Reads the provided {@link InputStream} into the {@link CharBuffer} without causing allocations. @@ -67,9 +113,12 @@ public DecoderWithBuffer call() throws Exception { */ public static boolean readUtf8Stream(final InputStream is, final CharBuffer charBuffer) throws IOException { // to be compatible with Java 8, we have to cast to buffer because of different return types - try (ObjectHandle pooled = POOL.createInstance()) { - final ByteBuffer buffer = pooled.get().byteBuffer; - final CharsetDecoder charsetDecoder = pooled.get().decoder; + try ( + ObjectHandle bufferHandle = BYTE_BUFFER_POOL.createInstance(); + ObjectHandle decoderHandle = getPooledCharsetDecoder(UTF8_CHARSET_NAME); + ) { + final ByteBuffer buffer = bufferHandle.get(); + final CharsetDecoder charsetDecoder = decoderHandle.get(); try { final byte[] bufferArray = buffer.array(); for (int read = is.read(bufferArray); read != -1; read = is.read(bufferArray)) { @@ -143,9 +192,11 @@ public static CoderResult decodeUtf8Bytes(final byte[] bytes, final CharBuffer c * @return a {@link CoderResult}, indicating the success or failure of the decoding */ public static CoderResult decodeUtf8Bytes(final byte[] bytes, final int offset, final int length, final CharBuffer charBuffer) { - try (ObjectHandle pooled = POOL.createInstance()) { - final ByteBuffer pooledBuffer = pooled.get().byteBuffer; - final CharsetDecoder charsetDecoder = pooled.get().decoder; + try ( + ObjectHandle bufferHandle = BYTE_BUFFER_POOL.createInstance(); + ObjectHandle decoderHandle = getPooledCharsetDecoder(UTF8_CHARSET_NAME); + ) { + final ByteBuffer pooledBuffer = bufferHandle.get(); // to be compatible with Java 8, we have to cast to buffer because of different return types final ByteBuffer buffer; if (pooledBuffer.capacity() < length) { @@ -157,7 +208,7 @@ public static CoderResult decodeUtf8Bytes(final byte[] bytes, final int offset, ((Buffer) buffer).position(0); ((Buffer) buffer).limit(length); } - return decode(charBuffer, buffer, charsetDecoder); + return decode(charBuffer, buffer, decoderHandle.get()); } } @@ -183,21 +234,25 @@ public static CoderResult decodeUtf8Bytes(final byte[] bytes, final int offset, */ public static CoderResult decodeUtf8Byte(final byte b, final CharBuffer charBuffer) { // to be compatible with Java 8, we have to cast to buffer because of different return types - try (ObjectHandle pooled = POOL.createInstance()) { - final ByteBuffer buffer = pooled.get().byteBuffer; - final CharsetDecoder charsetDecoder = pooled.get().decoder; + try ( + ObjectHandle bufferHandle = BYTE_BUFFER_POOL.createInstance(); + ObjectHandle decoderHandle = getPooledCharsetDecoder(UTF8_CHARSET_NAME); + ) { + final ByteBuffer buffer = bufferHandle.get(); buffer.put(b); ((Buffer) buffer).position(0); ((Buffer) buffer).limit(1); - return decode(charBuffer, buffer, charsetDecoder); + return decode(charBuffer, buffer, decoderHandle.get()); } } public static CoderResult decodeUtf8BytesFromSource(ByteSourceReader reader, T src, final CharBuffer dest) { // to be compatible with Java 8, we have to cast to buffer because of different return types - try (ObjectHandle pooled = POOL.createInstance()) { - final ByteBuffer buffer = pooled.get().byteBuffer; - final CharsetDecoder charsetDecoder = pooled.get().decoder; + try ( + ObjectHandle bufferHandle = BYTE_BUFFER_POOL.createInstance(); + ObjectHandle decoderHandle = getPooledCharsetDecoder(UTF8_CHARSET_NAME); + ) { + final ByteBuffer buffer = bufferHandle.get(); int readableBytes = reader.availableBytes(src); CoderResult result = null; while (readableBytes > 0) { @@ -206,7 +261,7 @@ public static CoderResult decodeUtf8BytesFromSource(ByteSourceReader read ((Buffer) buffer).position(0); reader.readInto(src, buffer); ((Buffer) buffer).position(0); - result = decode(dest, buffer, charsetDecoder); + result = decode(dest, buffer, decoderHandle.get()); if (result.isError() || result.isOverflow()) { return result; } @@ -223,6 +278,28 @@ public interface ByteSourceReader { void readInto(S source, ByteBuffer into); } + /** + * @param input the byte data to decode + * @param output the buffer to decode into + * @param charsetName the name of the charset + * @return null, if the charset is not known/supported. Otherwise the result of the decoding operation. + */ + @Nullable + public static CoderResult decode(ByteBuffer input, CharBuffer output, String charsetName) { + try (ObjectHandle decoderHandle = getPooledCharsetDecoder(charsetName)) { + if (decoderHandle == null) { + return null; //charset is unsupported + } + CharsetDecoder charsetDecoder = decoderHandle.get(); + try { + final CoderResult coderResult = charsetDecoder.decode(input, output, true); + charsetDecoder.flush(output); + return coderResult; + } finally { + charsetDecoder.reset(); + } + } + } private static CoderResult decode(CharBuffer charBuffer, ByteBuffer buffer, CharsetDecoder charsetDecoder) { try {