fix(cluster): fix complex plan serde in cluster actions (#15672)

* fix(cluster): fix complex plan serde in cluster actions * fix(cluster): fix complex plan serde in cluster actions * fix(cluster): fix complex plan serde in cluster actions * fix(cluster): fix complex plan serde in cluster actions * fix(cluster): fix complex plan serde in cluster actions
databendlabs · May 29, 2024 · 0b2f70a · 0b2f70a
1 parent 67f360b
commit 0b2f70a
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 21 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -233,8 +233,9 @@ arrow-udf-wasm = { package = "arrow-udf-wasm", git = "https://github.com/datafus
 prost = { version = "0.12.1" }
 prost-build = { version = "0.12.1" }
 serde = { version = "1.0.164", features = ["derive", "rc"] }
+serde_stacker = { version = "0.1" }
 serde_with = { version = "3.8.1" }
-serde_json = { version = "1.0.85", default-features = false, features = ["preserve_order"] }
+serde_json = { version = "1.0.85", default-features = false, features = ["preserve_order", "unbounded_depth"] }
 tonic-build = { version = "0.11" }
 
 # Memory management

diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml
@@ -163,6 +163,7 @@ rustls-pki-types = "1"
 rustyline = "14"
 serde = { workspace = true }
 serde_json = { workspace = true }
+serde_stacker = { workspace = true }
 serde_urlencoded = "0.7.1"
 socket2 = "0.5.3"
 strength_reduce = "0.2.4"

diff --git a/src/query/service/src/servers/flight/flight_client.rs b/src/query/service/src/servers/flight/flight_client.rs
@@ -60,29 +60,40 @@ impl FlightClient {
         T: Serialize,
         Res: for<'a> Deserialize<'a>,
     {
+        let mut body = Vec::with_capacity(512);
+        let mut serializer = serde_json::Serializer::new(&mut body);
+        let serializer = serde_stacker::Serializer::new(&mut serializer);
+        message.serialize(serializer).map_err(|cause| {
+            ErrorCode::BadArguments(format!(
+                "Request payload serialize error while in {:?}, cause: {}",
+                path, cause
+            ))
+        })?;
+
+        drop(message);
         let mut request =
             databend_common_tracing::inject_span_to_tonic_request(Request::new(Action {
+                body,
                 r#type: path.to_string(),
-                body: serde_json::to_vec(&message).map_err(|cause| {
-                    ErrorCode::BadArguments(format!(
-                        "Request payload serialize error while in {:?}, cause: {}",
-                        path, cause
-                    ))
-                })?,
             }));
 
-        drop(message);
         request.set_timeout(Duration::from_secs(timeout));
 
         let response = self.inner.do_action(request).await?;
 
         match response.into_inner().message().await? {
-            Some(response) => serde_json::from_slice::<Res>(&response.body).map_err(|cause| {
-                ErrorCode::BadBytes(format!(
-                    "Response payload deserialize error while in {:?}, cause: {}",
-                    path, cause
-                ))
-            }),
+            Some(response) => {
+                let mut deserializer = serde_json::Deserializer::from_slice(&response.body);
+                deserializer.disable_recursion_limit();
+                let deserializer = serde_stacker::Deserializer::new(&mut deserializer);
+
+                Res::deserialize(deserializer).map_err(|cause| {
+                    ErrorCode::BadBytes(format!(
+                        "Response payload deserialize error while in {:?}, cause: {}",
+                        path, cause
+                    ))
+                })
+            }
             None => Err(ErrorCode::EmptyDataFromServer(format!(
                 "Can not receive data from flight server, action: {:?}",
                 path

diff --git a/src/query/service/src/servers/flight/v1/actions/flight_actions.rs b/src/query/service/src/servers/flight/v1/actions/flight_actions.rs
@@ -64,7 +64,10 @@ impl FlightActions {
         self.actions.insert(
             path.clone(),
             Box::new(move |request| {
-                let request = serde_json::from_slice::<Req>(request).map_err(|cause| {
+                let mut deserializer = serde_json::Deserializer::from_slice(request);
+                deserializer.disable_recursion_limit();
+                let deserializer = serde_stacker::Deserializer::new(&mut deserializer);
+                let request = Req::deserialize(deserializer).map_err(|cause| {
                     ErrorCode::BadArguments(format!(
                         "Cannot parse request for {}, cause: {:?}",
                         path, cause
@@ -80,12 +83,19 @@ impl FlightActions {
 
                     let future = CatchUnwindFuture::create(future);
                     match future.await.flatten() {
-                        Ok(v) => serde_json::to_vec(&v).map_err(|cause| {
-                            ErrorCode::BadBytes(format!(
-                                "Cannot serialize response for {}, cause: {:?}",
-                                path, cause
-                            ))
-                        }),
+                        Ok(v) => {
+                            let mut out = Vec::with_capacity(512);
+                            let mut serializer = serde_json::Serializer::new(&mut out);
+                            let serializer = serde_stacker::Serializer::new(&mut serializer);
+                            v.serialize(serializer).map_err(|cause| {
+                                ErrorCode::BadBytes(format!(
+                                    "Cannot serialize response for {}, cause: {:?}",
+                                    path, cause
+                                ))
+                            })?;
+
+                            Ok(out)
+                        }
                         Err(err) => Err(err),
                     }
                 })

diff --git a/tests/sqllogictests/suites/base/03_common/03_0043_select_large_predicate.test b/tests/sqllogictests/suites/base/03_common/03_0043_select_large_predicate.test
@@ -0,0 +1,19 @@
+statement ok
+DROP TABLE IF EXISTS large_predicate
+
+statement ok
+CREATE TABLE large_predicate(test String)
+
+statement ok
+SET max_inlist_to_or = 300
+
+## stack overflow in debug
+## query IT
+## select * from large_predicate where test in ('8AF62BAE6CC6975AED89520B3B82EE28','46803E9ED0D972DDE983F4C8B9895A93','D4FE78B3A34A0948A2AA1F6A3C5122D0','C90052216B33CB1E0A376FE533A3B93F','C940569772D5663E961BC8F0B5CE170B','C84021A9771FA1E966F2A10F196A231D','6AFFB68A62BFC9E9AF1A9766E470590A','017FE89325960225780F67E91922E0E4','2D000BA4B9EA73BCD805597FB21A9273','7D00510517C73661E01B675A989B0C5F','2C401EA579B9B70E05A4741BC611D3E1','B600261AE9425B98B389C9952DE86D03','86FF06858CD5E2A2367CB160702BBC5C','58BFB1176DC8B465923B0BEFACA66859','26FFBAA83FF32BEF399A90E6B0469AB9','30FFDD92C38FAB06E586AEDC1975B957','D67E2C35F8AF50DEFBA8E1A8E19CDF75','40FFE4ED7D380E5BF1BB218B2AE018D6','01FFED84C7D325AB599F4BFE3778DFA8','7F0018EADAC991CA4E329CE1D5A48403','46C05278015CFBD27B5CBDD699DC9539','C8802E282BEB9351543B0F9C4790B86F','E8818B7D3A7D780C3D1157001D4B2C3D','FB80F860177F1A0582E291ABA1112D9F','237FC8B1F57CC1606E6716F5DCD7E878','4AFFA815A62DBC359347EC0F6F637C85','FBC0FAF178F0E414F01F632E6315F26C','51FFEFF78092616E0AA949DEA6CB143B','36FFF37C6277018E74FCEF3512BBBB9C','5AFF973B6B5C1570029EDD06F49ACBCD','170087FAB02AD5C574D31C43A718B3F2','2F4014A779B4F153148966BC2CB31B7A','FBD0F300B72F0381B345BCDC5ADD8798','D6FE103E76909E50A459177BB4EC4A18','64FFE1D9D029846577D07ADDFA283A76','0278C678700EE881780D238C42B689C5','037D9A3331BBCE9E6EF37AB59ECD7B03','08FFEB7EC8B9E3397DC74F359B1C16DD','47005FC19CBBB0E9DF5C4DFF5F8A316D','62FFA00172E1B76AED2B27C4DF2AAA71','29FFCEE9B9A8D74E65A595A70E74F5B6','450039174B556D4717D35598831B2778','057FEAA00E40779AD91C3B34F5673498','8D7F886F4A9252FE858D5D57FD7BE857','1B0018B6A2F5CEE6A19E5E25A8BFF4BA','D63E24D5B11C4B7AA77CD9634D2D1DED','7960C4311FA551C874387C6C4C64E38E','1EFFD1E9B89300972443B84B67E2428E','A0FFFA27B8E15D519948850E6E016C80','03BEC7C8BAA9CA6B27E29DBEFCA5929B','71004E027B56AAF2B1B359AF39A58B24','D6BE1EB5EC255729AC76E2E8E770A4C9','13004499C141360E37777AA4A991524C','7501044C265B523B1C7036135A6E97B8','59BFC5D6DC0AEB6CD860E4637E647D53','BF800AA9CCDC6558681EFDFD168916DA','B0801EB1D3F6EFB7D4FC829D59817BC3','B9006E4DAB29927920FCC5FD2E464D23','7700F4DAC85934B3DC055A388441CE01','327FD2925EFC0EA310B872AA13FAC4F1','61FF9AF1B5269AF887D9F6AF136F9553','587FAAF8BFCDEE05EB899437C1E1844A','067FDF5E2681A23F69DD13FF4416EFA5','CB0077BF7A8FC30642973A72DF441D70','021C79CD70A36D3854693C74A8EAA003','8CFF5F3BD2E223C48AB6170607410198','38FFDEA1E3F2943F43FD5D970014EE2F','6E7FDE4FA50FB6A942E87FC9432D1DCF','2F00176E3A8BE3B8E2624F6AB2A4FF34','8DBFA24CC3E5281E012FB6736A5AE93F','FF003954E1B02D87DB487D058AFE2F0D','9180487D6042F43E473AA5BB61686081','F1001874CE9E51902E707D87276C65E6','C2005D07369A7290F66DE6DFD6F1C8E8','7B0093D995A0F21D97D22E36515D5AF6','FB40FE2AA81938798B34173290077BA8','0DFFEAE17D36926AC95832444DF72C40','0F0009C213769F258B4A2140807B4139','5DFFA99E89DF5F533EFD21C6574F1624','BF001FBBDD8FC0EB12848E04A63C32FF','06FFCBDCBD6AD2EF2C807989A69874B3','56FFBF10CA7D7360BEBD6784851EC710','4CFFB6F832D96D582DD154ECB9264DB4','21FFFFFF8BFA0C3866BAE41BEC341114','CD00501261AD707E3A0BC500CE5120E2','597FC188ECD5640ECAB3F85D10759C13','43802E881A5C67DFCFE240B9C744AAAF','60FFC775451EA59D1539C8765B3BA3AC','7300C75796A869D067EC6E3A4C585C5E','4EFFE272742BC75B93AE2A9C9CFF16E3','05FFF433D61B5FE97EB6D54D57A6F8B6','FB010C7D3AEFDE1C1B9961DB37CACF1A','5EFFDE7E2F537CB944B0C510053EC1D7','C98066992AA4463550788522EA800918','32BFE23BB4A50E9808CE1FF5FB4EB125','59FFC9124078A4613DD564C9FD5628F8','0F800E848100F069FAF668EE9EE97107','2AFFF85F963EF5063993E7996438903D','E10032B5020C097567D9E2623B249956','90800725445A02893FD7127E5BD4E76A','FE805811086CA17284DE9440B566E68C','E30026A2B008DEDAF658265E49B65FA0','9100342383A1EEB09C3B00E1DEE0134E','D65E22A8BCB161A7829A76C1E55C650E','DAFE9F9342289CEE060DE80EA1AAD198','88FEB977413997F7A7EE467D369626A1','1CFFA9B5CA62B486B0BB6B58706A91A3','6EFFF8DF21C96CA3E43B07E3D06CAE83','84FF9399A3646BCBA1CD16C0FB542F09','32FFFC6B04019349633751AF6C5AA54E','50FFE003BD03585AF2A2DF5358CC5590','1680945CF4752E2110BBFB93497D3AE8','BE0075F3DACD74E0CCFFD20D30E1AA1C',NULL,'3C7FA075FD92C437CE2A553F8B6C7C47','EB00FD3EEDE2608359117DD696A65436','17807BDA1E2730044C81720F1B5B52CA','190048209B6E36205796180EF9EE7A7D','FA811408432770E1F4874787335B5C12','BA006A286CA456EC8F0B293CF4120DC2','00FFE9ABA768763D3A6BEAD9D9A529CA','A03FFF7E10BE8CDB37F5FAE8117CB452','023905531063126E023ADA18B38D8B18','B10010E7E9B4195F1BC08A27445D9D94','15007933280C05EFA26BFE979AE6AFAC','25FFAE66BDE0FF8385473974D1C057F3','58FFB73BA4F4FE7EE52BCF3A043440F1','2E000800D9E624CE5A19416BC57B958A','9D0070F1173D479A67F565181D6FD8AD','617FAD8F65F9CABF3F52EA3BBC4BD7F7','CD8048C8FEB5FBD91EF37E40378D4FEA','4E7FD9AFAB770BC831675015A5B17549','3CFFCCF929EE5542D6548102852975F1','04FFDE679FFE664D6FFC6ADAC9DC503A','6CFFC1409F0BD575A3D063D3DD06EB80','A0800048980DFA9F577CCCAE43C5ADFC','2F800B74B2CF9C799C1D125F8668AB92','2C8019A6C1D635EEB2587D9C11077698','D2FED5E3EB5936D80553F2ED3E472441','B1FFFB02B1B631F0F94ACD68485AE4F3','3AFF9BB5716B07965F4E0237285E16F2','FBE0EF71F14BEF7D177AC36B9CB65C5E','CE003F2362FD830F05BD53680EBADF5F','7940C55FAB2EF0F9152668962F1374CC','E70148A4B485BADD744589BE0EC7F87B','F901073818E21F1DECDC666AF818B27B','7980C3FAFA49C63F9601BEA5C5D92856','C600A308CFABC4AB4A3D784170A07C62','323FCA1A63E3E0A56E44540ECA30FB67','24FFAE791FB17CB7C7829D64A1F3C9AB','7900C134C6DFCEE20E385F236FF56B89','68FFEE397D64999FDF6C8B1066697EDC','FD00A8A8BADE1EBDD2C17EE3B8D935B6','21001F243CC73D9055A245CFF448FE51','0CFFE4BD2E577BA44627B74E48AFCBFF','02FB3E85335F1E487CDBF3460E1E5133','C30074F87A512830177FBF059B906B86','0AFFDDD27A49CE47C9759086457BC91E','063FED28D23794230C1C6BBC99BD186E','01BFEC3C38DFA9071A48B586B57F59EF','5CFFB08C19FE042094483456E0C744D3');
+## ----
+
+statement ok
+UNSET max_inlist_to_or
+
+statement ok
+DROP TABLE large_predicate