From e64aaf5d4bc70767b2e124baeab273797c9536e5 Mon Sep 17 00:00:00 2001 From: YuqiDu Date: Tue, 30 Jan 2024 09:49:58 -0800 Subject: [PATCH] increase max doc length limit: 1M -> 4M (#841) --- CONFIGURATION.md | 2 +- .../sgv2/jsonapi/config/DocumentLimitsConfig.java | 4 ++-- .../jsonapi/api/v1/InsertIntegrationTest.java | 2 +- .../service/shredding/ShredderDocLimitsTest.java | 15 ++++++++++----- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 57b19b1dd4..db639cae07 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -25,7 +25,7 @@ Here are some Stargate-relevant property groups that are necessary for correct s | Property | Type | Default | Description | |-----------------------------------------------------------------|-------|-------------|--------------------------------------------------------------------------------------| -| `stargate.jsonapi.document.limits.max-size` | `int` | `1_000_000` | The maximum size of (in characters) a single document. | +| `stargate.jsonapi.document.limits.max-size` | `int` | `4_000_000` | The maximum size of (in characters) a single document. | | `stargate.jsonapi.document.limits.max-depth` | `int` | `16` | The maximum document depth (nesting). | | `stargate.jsonapi.document.limits.max-property-name-length` | `int` | `100` | The maximum length of property names in a document for an individual segment. | | `stargate.jsonapi.document.limits.max-property-path-length` | `int` | `250` | The maximum length of property paths in a document (segments and separating periods) | diff --git a/src/main/java/io/stargate/sgv2/jsonapi/config/DocumentLimitsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/config/DocumentLimitsConfig.java index 251ef51de0..79d070ee94 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/config/DocumentLimitsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/config/DocumentLimitsConfig.java @@ -13,7 +13,7 @@ @ConfigMapping(prefix = "stargate.jsonapi.document.limits") public interface DocumentLimitsConfig { /** Defines the default maximum document size. */ - int DEFAULT_MAX_DOCUMENT_SIZE = 1_000_000; + int DEFAULT_MAX_DOCUMENT_SIZE = 4_000_000; /** Defines the default maximum document (nesting) depth */ int DEFAULT_MAX_DOCUMENT_DEPTH = 16; @@ -57,7 +57,7 @@ public interface DocumentLimitsConfig { int DEFAULT_MAX_VECTOR_EMBEDDING_LENGTH = 4096; /** - * @return Defines the maximum document size, defaults to {@code 1 meg} (1 million characters). + * @return Defines the maximum document size, defaults to {@code 4 meg} (4 million characters). */ @Positive @WithDefault("" + DEFAULT_MAX_DOCUMENT_SIZE) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java index d8fbec91fa..688ec9b240 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/InsertIntegrationTest.java @@ -821,7 +821,7 @@ public void tryInsertTooLongDoc() throws Exception { .body("errors[0].errorCode", is("SHRED_DOC_LIMIT_VIOLATION")) .body( "errors[0].message", startsWith("Document size limitation violated: document size (")) - .body("errors[0].message", endsWith(") exceeds maximum allowed (1000000)")); + .body("errors[0].message", endsWith(") exceeds maximum allowed (4000000)")); } @Test diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/shredding/ShredderDocLimitsTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/shredding/ShredderDocLimitsTest.java index 9cca791966..a083afa24e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/shredding/ShredderDocLimitsTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/shredding/ShredderDocLimitsTest.java @@ -39,16 +39,21 @@ public class ShredderDocLimitsTest { class ValidationDocSizeViolations { @Test public void allowBigButNotTooBigDoc() { - // Given we fail at 1 meg, let's try 800k (8 x 10 x 10k) - final ObjectNode bigDoc = createBigDoc(8, 10); + // Given we fail at 4 meg + // let's try 600k (8 x 5 x 7.5k) + final ObjectNode bigDoc = createBigDoc(8, 5); assertThat(shredder.shred(bigDoc)).isNotNull(); + + // let's also try 1m (12 x 12 x 7.5k) + final ObjectNode bigDoc1m = createBigDoc(12, 12); + assertThat(shredder.shred(bigDoc1m)).isNotNull(); } @Test public void catchTooBigDoc() { - // Let's construct document above 1 meg limit (but otherwise legal), with - // 144 x 7.5k String values, divided in 12 sub documents of 12 properties - final ObjectNode bigDoc = createBigDoc(12, 12); + // Let's construct document above 4 meg limit (but otherwise legal), with + // (12x45) x 7.5k String values, divided in 12 sub documents of 45 properties + final ObjectNode bigDoc = createBigDoc(12, 45); Exception e = catchException(() -> shredder.shred(bigDoc)); assertThat(e)