From b5335901ea4cca7682414391563f14f450e6b8b7 Mon Sep 17 00:00:00 2001 From: soulbird Date: Tue, 10 May 2022 10:10:14 +0800 Subject: [PATCH] feat(ext-plugin): support hook response body (#6968) --- apisix/constants.lua | 1 + apisix/plugins/ext-plugin-post-resp.lua | 172 ++++++++++ apisix/plugins/ext-plugin/init.lua | 108 +++++- conf/config-default.yaml | 1 + rockspec/apisix-master-0.rockspec | 2 +- t/admin/plugins.t | 1 + t/lib/ext-plugin.lua | 111 ++++++ t/plugin/ext-plugin/response.t | 431 ++++++++++++++++++++++++ t/plugin/ext-plugin/sanity.t | 19 +- 9 files changed, 842 insertions(+), 4 deletions(-) create mode 100644 apisix/plugins/ext-plugin-post-resp.lua create mode 100644 t/plugin/ext-plugin/response.t diff --git a/apisix/constants.lua b/apisix/constants.lua index f49e81d1cc9a8..cf04e890cc8cf 100644 --- a/apisix/constants.lua +++ b/apisix/constants.lua @@ -19,6 +19,7 @@ return { RPC_PREPARE_CONF = 1, RPC_HTTP_REQ_CALL = 2, RPC_EXTRA_INFO = 3, + RPC_HTTP_RESP_CALL = 4, HTTP_ETCD_DIRECTORY = { ["/upstreams"] = true, ["/plugins"] = true, diff --git a/apisix/plugins/ext-plugin-post-resp.lua b/apisix/plugins/ext-plugin-post-resp.lua new file mode 100644 index 0000000000000..e6156804c750f --- /dev/null +++ b/apisix/plugins/ext-plugin-post-resp.lua @@ -0,0 +1,172 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ext = require("apisix.plugins.ext-plugin.init") +local constants = require("apisix.constants") +local http = require("resty.http") + +local ngx = ngx +local ngx_print = ngx.print +local ngx_flush = ngx.flush +local string = string +local str_sub = string.sub + + +local name = "ext-plugin-post-resp" +local _M = { + version = 0.1, + priority = -4000, + name = name, + schema = ext.schema, +} + + +local function include_req_headers(ctx) + -- TODO: handle proxy_set_header + return core.request.headers(ctx) +end + + +local function close(http_obj) + -- TODO: keepalive + local ok, err = http_obj:close() + if not ok then + core.log.error("close http object failed: ", err) + end +end + + +local function get_response(ctx, http_obj) + local ok, err = http_obj:connect({ + scheme = ctx.upstream_scheme, + host = ctx.picked_server.host, + port = ctx.picked_server.port, + }) + + if not ok then + return nil, err + end + -- TODO: set timeout + local uri, args + if ctx.var.upstream_uri == "" then + -- use original uri instead of rewritten one + uri = ctx.var.uri + else + uri = ctx.var.upstream_uri + + -- the rewritten one may contain new args + local index = core.string.find(uri, "?") + if index then + local raw_uri = uri + uri = str_sub(raw_uri, 1, index - 1) + args = str_sub(raw_uri, index + 1) + end + end + local params = { + path = uri, + query = args or ctx.var.args, + headers = include_req_headers(ctx), + method = core.request.get_method(), + } + + local body, err = core.request.get_body() + if err then + return nil, err + end + + if body then + params["body"] = body + end + + local res, err = http_obj:request(params) + if not res then + return nil, err + end + + return res, err +end + + +local function send_response(res, code) + ngx.status = code or res.status + + local reader = res.body_reader + repeat + local chunk, ok, read_err, print_err, flush_err + -- TODO: HEAD or 304 + chunk, read_err = reader() + if read_err then + return "read response failed: ".. (read_err or "") + end + + if chunk then + ok, print_err = ngx_print(chunk) + if not ok then + return "output response failed: ".. (print_err or "") + end + ok, flush_err = ngx_flush(true) + if not ok then + core.log.warn("flush response failed: ", flush_err) + end + end + until not chunk + + return nil +end + + + +function _M.check_schema(conf) + return core.schema.check(_M.schema, conf) +end + + +function _M.before_proxy(conf, ctx) + local http_obj = http.new() + local res, err = get_response(ctx, http_obj) + if not res or err then + core.log.error("failed to request: ", err or "") + close(http_obj) + return 502 + end + ctx.runner_ext_response = res + + core.log.info("response info, status: ", res.status) + core.log.info("response info, headers: ", core.json.delay_encode(res.headers)) + + local code, body = ext.communicate(conf, ctx, name, constants.RPC_HTTP_RESP_CALL) + if body then + close(http_obj) + -- if the body is changed, the code will be set. + return code, body + end + core.log.info("ext-plugin will send response") + + -- send origin response, status maybe changed. + err = send_response(res, code) + close(http_obj) + + if err then + core.log.error(err) + return not ngx.headers_sent and 502 or nil + end + + core.log.info("ext-plugin send response succefully") +end + + +return _M diff --git a/apisix/plugins/ext-plugin/init.lua b/apisix/plugins/ext-plugin/init.lua index c37b4ab836606..b575ba45f4358 100644 --- a/apisix/plugins/ext-plugin/init.lua +++ b/apisix/plugins/ext-plugin/init.lua @@ -24,6 +24,8 @@ local http_req_call_resp = require("A6.HTTPReqCall.Resp") local http_req_call_action = require("A6.HTTPReqCall.Action") local http_req_call_stop = require("A6.HTTPReqCall.Stop") local http_req_call_rewrite = require("A6.HTTPReqCall.Rewrite") +local http_resp_call_req = require("A6.HTTPRespCall.Req") +local http_resp_call_resp = require("A6.HTTPRespCall.Resp") local extra_info = require("A6.ExtraInfo.Info") local extra_info_req = require("A6.ExtraInfo.Req") local extra_info_var = require("A6.ExtraInfo.Var") @@ -680,6 +682,107 @@ local rpc_handlers = { return true end, + nil, -- ignore RPC_EXTRA_INFO, already processed during RPC_HTTP_REQ_CALL interaction + function (conf, ctx, sock, entry) + local lrucache_id = core.lrucache.plugin_ctx_id(ctx, entry) + local token, err = core.lrucache.plugin_ctx(lrucache, ctx, entry, rpc_call, + constants.RPC_PREPARE_CONF, conf, ctx, + lrucache_id) + if not token then + return nil, err + end + + builder:Clear() + local var = ctx.var + + local res = ctx.runner_ext_response + local textEntries = {} + local hdrs = res.headers + for key, val in pairs(hdrs) do + local ty = type(val) + if ty == "table" then + for _, v in ipairs(val) do + core.table.insert(textEntries, build_headers(var, builder, key, v)) + end + else + core.table.insert(textEntries, build_headers(var, builder, key, val)) + end + end + local len = #textEntries + http_resp_call_req.StartHeadersVector(builder, len) + for i = len, 1, -1 do + builder:PrependUOffsetTRelative(textEntries[i]) + end + local hdrs_vec = builder:EndVector(len) + + local id = generate_id() + local status = res.status + + http_resp_call_req.Start(builder) + http_resp_call_req.AddId(builder, id) + http_resp_call_req.AddStatus(builder, status) + http_resp_call_req.AddConfToken(builder, token) + http_resp_call_req.AddHeaders(builder, hdrs_vec) + + local req = http_resp_call_req.End(builder) + builder:Finish(req) + + local ok, err = send(sock, constants.RPC_HTTP_RESP_CALL, builder:Output()) + if not ok then + return nil, "failed to send RPC_HTTP_RESP_CALL: " .. err + end + + local ty, resp = receive(sock) + if ty == nil then + return nil, "failed to receive RPC_HTTP_RESP_CALL: " .. resp + end + + if ty ~= constants.RPC_HTTP_RESP_CALL then + return nil, "failed to receive RPC_HTTP_RESP_CALL: unexpected type " .. ty + end + + local buf = flatbuffers.binaryArray.New(resp) + local call_resp = http_resp_call_resp.GetRootAsResp(buf, 0) + local len = call_resp:HeadersLength() + if len > 0 then + local resp_headers = {} + for i = 1, len do + local entry = call_resp:Headers(i) + local name = str_lower(entry:Name()) + if not exclude_resp_header[name] then + if resp_headers[name] == nil then + core.response.set_header(name, entry:Value()) + resp_headers[name] = true + else + core.response.add_header(name, entry:Value()) + end + end + end + else + -- Filter out origin headeres + for k, v in pairs(res.headers) do + if not exclude_resp_header[str_lower(k)] then + core.response.set_header(k, v) + end + end + end + + local body + local len = call_resp:BodyLength() + if len > 0 then + -- TODO: support empty body + body = call_resp:BodyAsString() + end + local code = call_resp:Status() + core.log.info("recv resp, code: ", code, " body: ", body, " len: ", len) + + if code == 0 then + -- runner changes body only, we should set code. + code = body and res.status or nil + end + + return true, nil, code, body + end } @@ -719,12 +822,13 @@ local function recreate_lrucache() end -function _M.communicate(conf, ctx, plugin_name) +function _M.communicate(conf, ctx, plugin_name, rpc_cmd) local ok, err, code, body local tries = 0 + local ty = rpc_cmd and rpc_cmd or constants.RPC_HTTP_REQ_CALL while tries < 3 do tries = tries + 1 - ok, err, code, body = rpc_call(constants.RPC_HTTP_REQ_CALL, conf, ctx, plugin_name) + ok, err, code, body = rpc_call(ty, conf, ctx, plugin_name) if ok then if code then return code, body diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 0888743dd84c5..ae3f151a7f42c 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -402,6 +402,7 @@ plugins: # plugin list (sorted by priority) - openwhisk # priority: -1901 - serverless-post-function # priority: -2000 - ext-plugin-post-req # priority: -3000 + - ext-plugin-post-resp # priority: -4000 stream_plugins: # sorted by priority - ip-restriction # priority: 3000 diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index e75c13e13eaae..af447f1bdb012 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -67,7 +67,7 @@ dependencies = { "luasec = 0.9-1", "lua-resty-consul = 0.3-2", "penlight = 1.9.2-1", - "ext-plugin-proto = 0.4.0", + "ext-plugin-proto = 0.5.0", "casbin = 1.26.0", "api7-snowflake = 2.0-1", "inspect == 3.1.1", diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 2557da7505c54..a639d3af74252 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -127,6 +127,7 @@ azure-functions openwhisk serverless-post-function ext-plugin-post-req +ext-plugin-post-resp diff --git a/t/lib/ext-plugin.lua b/t/lib/ext-plugin.lua index c74464eaa7a37..33bb32b15ed56 100644 --- a/t/lib/ext-plugin.lua +++ b/t/lib/ext-plugin.lua @@ -29,6 +29,8 @@ local http_req_call_resp = require("A6.HTTPReqCall.Resp") local http_req_call_action = require("A6.HTTPReqCall.Action") local http_req_call_stop = require("A6.HTTPReqCall.Stop") local http_req_call_rewrite = require("A6.HTTPReqCall.Rewrite") +local http_resp_call_req = require("A6.HTTPRespCall.Req") +local http_resp_call_resp = require("A6.HTTPRespCall.Resp") local extra_info = require("A6.ExtraInfo.Info") local extra_info_req = require("A6.ExtraInfo.Req") local extra_info_var = require("A6.ExtraInfo.Var") @@ -440,6 +442,115 @@ function _M.go(case) local req = http_req_call_resp.End(builder) builder:Finish(req) data = builder:Output() + + elseif ty == constants.RPC_HTTP_RESP_CALL then + local buf = flatbuffers.binaryArray.New(data) + local call_req = http_resp_call_req.GetRootAsReq(buf, 0) + if case.check_input then + assert(call_req:Id() == 0) + assert(call_req:ConfToken() == 233) + assert(call_req:Status() == 200) + local len = call_req:HeadersLength() + + local headers = {} + for i = 1, len do + local entry = call_req:Headers(i) + local r = headers[entry:Name()] + if r then + headers[entry:Name()] = {r, entry:Value()} + else + headers[entry:Name()] = entry:Value() or true + end + end + assert(json.encode(headers), '{"Connection":"close","Content-Length":"12",' .. + '"Content-Type":"text/plain","Server":"openresty"}') + http_resp_call_resp.Start(builder) + + elseif case.modify_body then + local len = 3 + http_resp_call_resp.StartBodyVector(builder, len) + builder:PrependByte(string.byte("t")) + builder:PrependByte(string.byte("a")) + builder:PrependByte(string.byte("c")) + local b = builder:EndVector(len) + + + http_resp_call_resp.Start(builder) + http_resp_call_resp.AddBody(builder, b) + + elseif case.modify_header then + local len = call_req:HeadersLength() + + local headers = {} + for i = 1, len do + local entry = call_req:Headers(i) + local r = headers[entry:Name()] + if r then + headers[entry:Name()] = {r, entry:Value()} + else + headers[entry:Name()] = entry:Value() or true + end + end + + if case.same_header then + headers["x-same"] = {"one", "two"} + else + local runner = headers["x-runner"] + if runner and runner == "Go-runner" then + headers["x-runner"] = "Test-Runner" + end + end + + local i = 1 + local textEntries = {} + for k, v in pairs(headers) do + local name = builder:CreateString(k) + if type(v) == "table" then + for j = 1, #v do + local value = builder:CreateString(v[j]) + text_entry.Start(builder) + text_entry.AddName(builder, name) + text_entry.AddValue(builder, value) + local c = text_entry.End(builder) + textEntries[i] = c + i = i + 1 + end + else + local value = builder:CreateString(v) + text_entry.Start(builder) + text_entry.AddName(builder, name) + text_entry.AddValue(builder, value) + local c = text_entry.End(builder) + textEntries[i] = c + i = i + 1 + end + end + + len = #textEntries + http_resp_call_resp.StartHeadersVector(builder, len) + for i = len, 1, -1 do + builder:PrependUOffsetTRelative(textEntries[i]) + end + local vec = builder:EndVector(len) + + http_resp_call_resp.Start(builder) + http_resp_call_resp.AddHeaders(builder, vec) + + elseif case.modify_status then + local status = call_req:Status() + if status == 200 then + status = 304 + end + http_resp_call_resp.Start(builder) + http_resp_call_resp.AddStatus(builder, status) + + else + http_resp_call_resp.Start(builder) + end + + local resp = http_resp_call_resp.End(builder) + builder:Finish(resp) + data = builder:Output() end local ok, err = ext.send(sock, ty, data) diff --git a/t/plugin/ext-plugin/response.t b/t/plugin/ext-plugin/response.t new file mode 100644 index 0000000000000..2500e5db0e96f --- /dev/null +++ b/t/plugin/ext-plugin/response.t @@ -0,0 +1,431 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + $block->set_value("stream_conf_enable", 1); + + if (!defined $block->extra_stream_config) { + my $stream_config = <<_EOC_; + server { + listen unix:\$TEST_NGINX_HTML_DIR/nginx.sock; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({}) + } + } + +_EOC_ + $block->set_value("extra_stream_config", $stream_config); + } + + my $unix_socket_path = $ENV{"TEST_NGINX_HTML_DIR"} . "/nginx.sock"; + my $cmd = $block->ext_plugin_cmd // "['sleep', '5s']"; + my $extra_yaml_config = <<_EOC_; +ext-plugin: + path_for_test: $unix_socket_path + cmd: $cmd +_EOC_ + + $block->set_value("extra_yaml_config", $extra_yaml_config); + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: add route +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + + local code, message, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/*", + "plugins": { + "ext-plugin-post-resp": { + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(message) + } + } +--- response_body +passed + + + +=== TEST 2: check input +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({check_input = true}) + } + } +--- error_code: 200 +--- response_body +hello world + + + +=== TEST 3: modify body +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({modify_body = true}) + } + } +--- error_code: 200 +--- response_body chomp +cat + + + +=== TEST 4: modify header +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({modify_header = true}) + } + } +--- more_headers +resp-X-Runner: Go-runner +--- error_code: 200 +--- response_headers +X-Runner: Test-Runner +--- response_body +hello world + + + +=== TEST 5: modify same response headers +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({modify_header = true, same_header = true}) + } + } +--- error_code: 200 +--- response_headers +X-Same: one, two +--- response_body +hello world + + + +=== TEST 6: modify status +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({modify_status = true}) + } + } +--- error_code: 304 + + + +=== TEST 7: default allow_degradation +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + + local code, message, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "plugins": { + "ext-plugin-post-resp": { + "conf": [ + {"name":"foo", "value":"bar"} + ] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(message) + } + } +--- response_body +passed + + + +=== TEST 8: ext-plugin-resp wrong, req reject +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock1; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({}) + } + } +--- error_code: 503 +--- error_log eval +qr/failed to connect to the unix socket/ + + + +=== TEST 9: open allow_degradation +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + + local code, message, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "plugins": { + "ext-plugin-post-req": { + "conf": [ + {"name":"foo", "value":"bar"} + ], + "allow_degradation": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(message) + } + } +--- response_body +passed + + + +=== TEST 10: ext-plugin-resp wrong, req access +--- request +GET /hello +--- extra_stream_config + server { + listen unix:$TEST_NGINX_HTML_DIR/nginx.sock1; + + content_by_lua_block { + local ext = require("lib.ext-plugin") + ext.go({}) + } + } +--- response_body +hello world +--- error_log eval +qr/Plugin Runner.*allow degradation/ + + + +=== TEST 11: add route: wrong upstream +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + + local code, message, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/*", + "plugins": { + "ext-plugin-post-resp": { + } + }, + "upstream": { + "nodes": { + "127.0.0.1:3980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(message) + } + } +--- response_body +passed + + + +=== TEST 12: request upstream failed +--- request +GET /hello +--- error_code: 502 +--- error_log eval +qr/failed to request/ + + + +=== TEST 13: add route +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + + local code, message, res = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/*", + "plugins": { + "ext-plugin-post-resp": { + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + ngx.say(message) + } + } +--- response_body +passed + + + +=== TEST 14: body_reader error +--- request +GET /hello1 +--- more_headers +resp-Content-Length: 14 +--- error_code: 502 +--- error_log eval +qr/read response failed/ + + + +=== TEST 15: response chunked +--- request +GET /hello_chunked +--- error_code: 200 +--- response_body +hello world + + + +=== TEST 16: check upstream uri with args +--- request +GET /plugin_proxy_rewrite_args?aaa=bbb&ccc=ddd +--- error_code: 200 +--- response_body +uri: /plugin_proxy_rewrite_args +aaa: bbb +ccc: ddd diff --git a/t/plugin/ext-plugin/sanity.t b/t/plugin/ext-plugin/sanity.t index a665f1cf8424d..707c9db1d7526 100644 --- a/t/plugin/ext-plugin/sanity.t +++ b/t/plugin/ext-plugin/sanity.t @@ -82,7 +82,8 @@ __DATA__ "uri": "/hello", "plugins": { "ext-plugin-pre-req": {"a":"b"}, - "ext-plugin-post-req": {"c":"d"} + "ext-plugin-post-req": {"c":"d"}, + "ext-plugin-post-resp": {"e":"f"} }, "upstream": { "nodes": { @@ -136,6 +137,14 @@ sending rpc type: 2 data length: receiving rpc type: 2 data length: sending rpc type: 2 data length: receiving rpc type: 2 data length: +sending rpc type: 1 data length: +receiving rpc type: 1 data length: +sending rpc type: 1 data length: +receiving rpc type: 1 data length: +sending rpc type: 4 data length: +receiving rpc type: 4 data length: +sending rpc type: 4 data length: +receiving rpc type: 4 data length: @@ -268,6 +277,14 @@ sending rpc type: 1 data length: receiving rpc type: 1 data length: sending rpc type: 1 data length: receiving rpc type: 1 data length: +sending rpc type: 1 data length: +receiving rpc type: 1 data length: +sending rpc type: 1 data length: +receiving rpc type: 1 data length: +sending rpc type: 1 data length: +receiving rpc type: 1 data length: +sending rpc type: 1 data length: +receiving rpc type: 1 data length: --- error_log flush conf token lrucache flush conf token in shared dict