feat(replay): Truncate network bodies to max size (#7875)

---------

Co-authored-by: Abhijeet Prasad <aprasad@sentry.io>

packages/replay/src/constants.ts

@@ -29,5 +29,5 @@ export const ERROR_CHECKOUT_TIME = 60_000;
 export const RETRY_BASE_INTERVAL = 5000;
 export const RETRY_MAX_COUNT = 3;
 
-/* The max (uncompressed) size in bytes of a network body. Any body larger than this will be dropped. */
-export const NETWORK_BODY_MAX_SIZE = 300_000;
+/* The max (uncompressed) size in bytes of a network body. Any body larger than this will be truncated. */
+export const NETWORK_BODY_MAX_SIZE = 150_000;

packages/replay/src/coreHandlers/util/fetchUtils.ts

@@ -3,7 +3,6 @@ import { logger } from '@sentry/utils';
 
 import type {
   FetchHint,
-  NetworkBody,
   ReplayContainer,
   ReplayNetworkOptions,
   ReplayNetworkRequestData,
@@ -15,7 +14,6 @@ import {
   getAllowedHeaders,
   getBodySize,
   getBodyString,
-  getNetworkBody,
   makeNetworkReplayBreadcrumb,
   parseContentLengthHeader,
 } from './networkUtils';
@@ -112,8 +110,8 @@ function _getRequestInfo(
 
   // We only want to transmit string or string-like bodies
   const requestBody = _getFetchRequestArgBody(input);
-  const body = getNetworkBody(getBodyString(requestBody));
-  return buildNetworkRequestOrResponse(headers, requestBodySize, body);
+  const bodyStr = getBodyString(requestBody);
+  return buildNetworkRequestOrResponse(headers, requestBodySize, bodyStr);
 }
 
 async function _getResponseInfo(
@@ -137,15 +135,15 @@ async function _getResponseInfo(
   try {
     // We have to clone this, as the body can only be read once
     const res = response.clone();
-    const { body, bodyText } = await _parseFetchBody(res);
+    const bodyText = await _parseFetchBody(res);
 
     const size =
       bodyText && bodyText.length && responseBodySize === undefined
         ? getBodySize(bodyText, textEncoder)
         : responseBodySize;
 
     if (captureBodies) {
-      return buildNetworkRequestOrResponse(headers, size, body);
+      return buildNetworkRequestOrResponse(headers, size, bodyText);
     }
 
     return buildNetworkRequestOrResponse(headers, size, undefined);
@@ -155,25 +153,12 @@ async function _getResponseInfo(
   }
 }
 
-async function _parseFetchBody(
-  response: Response,
-): Promise<{ body?: NetworkBody | undefined; bodyText?: string | undefined }> {
-  let bodyText: string;
-
+async function _parseFetchBody(response: Response): Promise<string | undefined> {
   try {
-    bodyText = await response.text();
+    return await response.text();
   } catch {
-    return {};
-  }
-
-  try {
-    const body = JSON.parse(bodyText);
-    return { body, bodyText };
-  } catch {
-    // just send bodyText
+    return undefined;
   }
-
-  return { bodyText, body: bodyText };
 }
 
 function _getFetchRequestArgBody(fetchArgs: unknown[] = []): RequestInit['body'] | undefined {

packages/replay/src/coreHandlers/util/networkUtils.ts

@@ -4,11 +4,13 @@ import { dropUndefinedKeys } from '@sentry/utils';
 import { NETWORK_BODY_MAX_SIZE } from '../../constants';
 import type {
   NetworkBody,
+  NetworkMetaWarning,
   NetworkRequestData,
   ReplayNetworkRequestData,
   ReplayNetworkRequestOrResponse,
   ReplayPerformanceEntry,
 } from '../../types';
+import { fixJson } from '../../util/truncateJson/fixJson';
 
 /** Get the size of a body. */
 export function getBodySize(
@@ -122,7 +124,7 @@ export function getNetworkBody(bodyText: string | undefined): NetworkBody | unde
 export function buildNetworkRequestOrResponse(
   headers: Record<string, string>,
   bodySize: number | undefined,
-  body: NetworkBody | undefined,
+  body: string | undefined,
 ): ReplayNetworkRequestOrResponse | undefined {
   if (!bodySize && Object.keys(headers).length === 0) {
     return undefined;
@@ -146,11 +148,11 @@ export function buildNetworkRequestOrResponse(
     size: bodySize,
   };
 
-  if (bodySize < NETWORK_BODY_MAX_SIZE) {
-    info.body = body;
-  } else {
+  const { body: normalizedBody, warnings } = normalizeNetworkBody(body);
+  info.body = normalizedBody;
+  if (warnings.length > 0) {
     info._meta = {
-      errors: ['MAX_BODY_SIZE_EXCEEDED'],
+      warnings,
     };
   }
 
@@ -175,3 +177,46 @@ function _serializeFormData(formData: FormData): string {
   // @ts-ignore passing FormData to URLSearchParams actually works
   return new URLSearchParams(formData).toString();
 }
+
+function normalizeNetworkBody(body: string | undefined): {
+  body: NetworkBody | undefined;
+  warnings: NetworkMetaWarning[];
+} {
+  if (!body || typeof body !== 'string') {
+    return {
+      body,
+      warnings: [],
+    };
+  }
+
+  const exceedsSizeLimit = body.length > NETWORK_BODY_MAX_SIZE;
+
+  if (_strIsProbablyJson(body)) {
+    try {
+      const json = exceedsSizeLimit ? fixJson(body.slice(0, NETWORK_BODY_MAX_SIZE)) : body;
+      const normalizedBody = JSON.parse(json);
+      return {
+        body: normalizedBody,
+        warnings: exceedsSizeLimit ? ['JSON_TRUNCATED'] : [],
+      };
+    } catch {
+      return {
+        body,
+        warnings: ['INVALID_JSON'],
+      };
+    }
+  }
+
+  return {
+    body: exceedsSizeLimit ? `${body.slice(0, NETWORK_BODY_MAX_SIZE)}…` : body,
+    warnings: exceedsSizeLimit ? ['TEXT_TRUNCATED'] : [],
+  };
+}
+
+function _strIsProbablyJson(str: string): boolean {
+  const first = str[0];
+  const last = str[str.length - 1];
+
+  // Simple check: If this does not start & end with {} or [], it's not JSON
+  return (first === '[' && last === ']') || (first === '{' && last === '}');
+}

packages/replay/src/coreHandlers/util/xhrUtils.ts

@@ -8,7 +8,6 @@ import {
   getAllowedHeaders,
   getBodySize,
   getBodyString,
-  getNetworkBody,
   makeNetworkReplayBreadcrumb,
   parseContentLengthHeader,
 } from './networkUtils';
@@ -84,12 +83,12 @@ function _prepareXhrData(
   const request = buildNetworkRequestOrResponse(
     requestHeaders,
     requestBodySize,
-    options.captureBodies ? getNetworkBody(getBodyString(input)) : undefined,
+    options.captureBodies ? getBodyString(input) : undefined,
   );
   const response = buildNetworkRequestOrResponse(
     responseHeaders,
     responseBodySize,
-    options.captureBodies ? getNetworkBody(hint.xhr.responseText) : undefined,
+    options.captureBodies ? hint.xhr.responseText : undefined,
   );
 
   return {

packages/replay/src/types.ts

@@ -512,12 +512,15 @@ export type FetchHint = FetchBreadcrumbHint & {
   response: Response;
 };
 
-export type NetworkBody = Record<string, unknown> | string;
+type JsonObject = Record<string, unknown>;
+type JsonArray = unknown[];
 
-type NetworkMetaError = 'MAX_BODY_SIZE_EXCEEDED';
+export type NetworkBody = JsonObject | JsonArray | string;
+
+export type NetworkMetaWarning = 'JSON_TRUNCATED' | 'TEXT_TRUNCATED' | 'INVALID_JSON';
 
 interface NetworkMeta {
-  errors?: NetworkMetaError[];
+  warnings?: NetworkMetaWarning[];
 }
 
 export interface ReplayNetworkRequestOrResponse {

packages/replay/src/util/truncateJson/completeJson.ts

@@ -0,0 +1,125 @@
+import type { JsonToken } from './constants';
+import {
+  ARR,
+  ARR_VAL,
+  ARR_VAL_COMPLETED,
+  ARR_VAL_STR,
+  OBJ,
+  OBJ_KEY,
+  OBJ_KEY_STR,
+  OBJ_VAL,
+  OBJ_VAL_COMPLETED,
+  OBJ_VAL_STR,
+} from './constants';
+
+const ALLOWED_PRIMITIVES = ['true', 'false', 'null'];
+
+/**
+ * Complete an incomplete JSON string.
+ * This will ensure that the last element always has a `"~~"` to indicate it was truncated.
+ * For example, `[1,2,` will be completed to `[1,2,"~~"]`
+ * and `{"aa":"b` will be completed to `{"aa":"b~~"}`
+ */
+export function completeJson(incompleteJson: string, stack: JsonToken[]): string {
+  if (!stack.length) {
+    return incompleteJson;
+  }
+
+  let json = incompleteJson;
+
+  // Most checks are only needed for the last step in the stack
+  const lastPos = stack.length - 1;
+  const lastStep = stack[lastPos];
+
+  json = _fixLastStep(json, lastStep);
+
+  // Complete remaining steps - just add closing brackets
+  for (let i = lastPos; i >= 0; i--) {
+    const step = stack[i];
+
+    switch (step) {
+      case OBJ:
+        json = `${json}}`;
+        break;
+      case ARR:
+        json = `${json}]`;
+        break;
+    }
+  }
+
+  return json;
+}
+
+function _fixLastStep(json: string, lastStep: JsonToken): string {
+  switch (lastStep) {
+    // Object cases
+    case OBJ:
+      return `${json}"~~":"~~"`;
+    case OBJ_KEY:
+      return `${json}:"~~"`;
+    case OBJ_KEY_STR:
+      return `${json}~~":"~~"`;
+    case OBJ_VAL:
+      return _maybeFixIncompleteObjValue(json);
+    case OBJ_VAL_STR:
+      return `${json}~~"`;
+    case OBJ_VAL_COMPLETED:
+      return `${json},"~~":"~~"`;
+
+    // Array cases
+    case ARR:
+      return `${json}"~~"`;
+    case ARR_VAL:
+      return _maybeFixIncompleteArrValue(json);
+    case ARR_VAL_STR:
+      return `${json}~~"`;
+    case ARR_VAL_COMPLETED:
+      return `${json},"~~"`;
+  }
+
+  return json;
+}
+
+function _maybeFixIncompleteArrValue(json: string): string {
+  const pos = _findLastArrayDelimiter(json);
+
+  if (pos > -1) {
+    const part = json.slice(pos + 1);
+
+    if (ALLOWED_PRIMITIVES.includes(part.trim())) {
+      return `${json},"~~"`;
+    }
+
+    // Everything else is replaced with `"~~"`
+    return `${json.slice(0, pos + 1)}"~~"`;
+  }
+
+  // fallback, this shouldn't happen, to be save
+  return json;
+}
+
+function _findLastArrayDelimiter(json: string): number {
+  for (let i = json.length - 1; i >= 0; i--) {
+    const char = json[i];
+
+    if (char === ',' || char === '[') {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+function _maybeFixIncompleteObjValue(json: string): string {
+  const startPos = json.lastIndexOf(':');
+
+  const part = json.slice(startPos + 1);
+
+  if (ALLOWED_PRIMITIVES.includes(part.trim())) {
+    return `${json},"~~":"~~"`;
+  }
+
+  // Everything else is replaced with `"~~"`
+  // This also means we do not have incomplete numbers, e.g `[1` is replaced with `["~~"]`
+  return `${json.slice(0, startPos + 1)}"~~"`;
+}

packages/replay/src/util/truncateJson/constants.ts

@@ -0,0 +1,23 @@
+export const OBJ = 10;
+export const OBJ_KEY = 11;
+export const OBJ_KEY_STR = 12;
+export const OBJ_VAL = 13;
+export const OBJ_VAL_STR = 14;
+export const OBJ_VAL_COMPLETED = 15;
+
+export const ARR = 20;
+export const ARR_VAL = 21;
+export const ARR_VAL_STR = 22;
+export const ARR_VAL_COMPLETED = 23;
+
+export type JsonToken =
+  | typeof OBJ
+  | typeof OBJ_KEY
+  | typeof OBJ_KEY_STR
+  | typeof OBJ_VAL
+  | typeof OBJ_VAL_STR
+  | typeof OBJ_VAL_COMPLETED
+  | typeof ARR
+  | typeof ARR_VAL
+  | typeof ARR_VAL_STR
+  | typeof ARR_VAL_COMPLETED;