feat: interact via gRPC in APISIX Admin API

Signed-off-by: spacewander <spacewanderlzx@gmail.com>

apisix/admin/init.lua

@@ -25,6 +25,7 @@ local ngx_time = ngx.time
 local ngx_timer_at = ngx.timer.at
 local ngx_worker_id = ngx.worker.id
 local tonumber = tonumber
+local tostring = tostring
 local str_lower = string.lower
 local reload_event = "/apisix/admin/plugins/reload"
 local ipairs = ipairs
@@ -111,6 +112,16 @@ local function strip_etcd_resp(data)
             data.node.createdIndex = nil
             data.node.modifiedIndex = nil
         end
+
+        data.count = nil
+        data.more = nil
+        data.prev_kvs = nil
+
+        if data.deleted then
+            -- We used to treat the type incorrectly. But for compatibility we follow
+            -- the existing type.
+            data.deleted = tostring(data.deleted)
+        end
     end
 
     return data

apisix/cli/schema.lua

@@ -61,7 +61,12 @@ local etcd_schema = {
             default = 30,
             minimum = 1,
             description = "etcd connection timeout in seconds",
-        }
+        },
+        use_grpc = {
+            type = "boolean",
+            -- TODO: set true by default in v3.2
+            default = false,
+        },
     },
     required = {"prefix", "host"}
 }

apisix/core/config_etcd.lua

@@ -506,8 +506,8 @@ do
             return etcd_cli
         end
 
-        local err
-        etcd_cli, err = etcd_apisix.switch_proxy()
+        local _, err
+        etcd_cli, _, err = etcd_apisix.switch_proxy(true)
         return etcd_cli, err
     end
 end
@@ -693,7 +693,7 @@ function _M.new(key, opts)
     else
         local etcd_cli, err = get_etcd()
         if not etcd_cli then
-            return nil, "failed to start a etcd instance: " .. err
+            return nil, "failed to start an etcd instance: " .. err
         end
         obj.etcd_cli = etcd_cli
     end

apisix/core/etcd.lua

@@ -19,6 +19,7 @@
 --
 -- @module core.etcd
 
+local require           = require
 local fetch_local_conf  = require("apisix.core.config_local").local_conf
 local array_mt          = require("apisix.core.json").array_mt
 local v3_adapter        = require("apisix.admin.v3_adapter")
@@ -27,6 +28,7 @@ local clone_tab         = require("table.clone")
 local health_check      = require("resty.etcd.health_check")
 local pl_path           = require("pl.path")
 local ipairs            = ipairs
+local pcall             = pcall
 local setmetatable      = setmetatable
 local string            = string
 local tonumber          = tonumber
@@ -70,6 +72,18 @@ local function _new(etcd_conf)
         end
     end
 
+    if etcd_conf.use_grpc then
+        -- TODO: let lua-resty-etcd support more use cases
+        if etcd.user or ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
+            etcd_conf.use_grpc = false
+        else
+            local ok = pcall(require, "resty.grpc")
+            if not ok then
+                etcd_conf.use_grpc = false
+            end
+        end
+    end
+
     local etcd_cli, err = etcd.new(etcd_conf)
     if not etcd_cli then
         return nil, nil, err
@@ -79,13 +93,41 @@ local function _new(etcd_conf)
 end
 
 
-local function new()
+---
+-- Create an etcd client which will connect to etcd without being proxyed by conf server.
+-- This method is used in init_worker phase when the conf server is not ready.
+--
+-- @function core.etcd.new_without_proxy
+-- @treturn table|nil the etcd client, or nil if failed.
+-- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
+-- @treturn nil|string the error message.
+local function new_without_proxy()
+    local local_conf, err = fetch_local_conf()
+    if not local_conf then
+        return nil, nil, err
+    end
+
+    local etcd_conf = clone_tab(local_conf.etcd)
+    return _new(etcd_conf)
+end
+_M.new_without_proxy = new_without_proxy
+
+
+local function new(use_http)
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
     end
 
     local etcd_conf = clone_tab(local_conf.etcd)
+    if use_http then
+        etcd_conf.use_grpc = false
+    end
+    if etcd_conf.use_grpc then
+        -- TODO: add grpc proxy support later
+        return new_without_proxy()
+    end
+
     local proxy_by_conf_server = false
 
     if local_conf.deployment then
@@ -151,32 +193,13 @@ end
 _M.new = new
 
 
----
--- Create an etcd client which will connect to etcd without being proxyed by conf server.
--- This method is used in init_worker phase when the conf server is not ready.
---
--- @function core.etcd.new_without_proxy
--- @treturn table|nil the etcd client, or nil if failed.
--- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
--- @treturn nil|string the error message.
-local function new_without_proxy()
-    local local_conf, err = fetch_local_conf()
-    if not local_conf then
-        return nil, nil, err
-    end
-
-    local etcd_conf = clone_tab(local_conf.etcd)
-    return _new(etcd_conf)
-end
-_M.new_without_proxy = new_without_proxy
-
-
-local function switch_proxy()
+-- TODO: add grpc proxy support later, then we can remove "use_http" scaffold
+local function switch_proxy(use_http)
     if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
         return new_without_proxy()
     end
 
-    local etcd_cli, prefix, err = new()
+    local etcd_cli, prefix, err = new(use_http)
     if not etcd_cli or err then
         return etcd_cli, prefix, err
     end

apisix/core/pubsub.lua

@@ -59,7 +59,9 @@ local function get_cmd(data)
     for key, value in pairs(data) do
         -- There are sequence and command properties in the data,
         -- select the handler according to the command value.
-        if key ~= "sequence" then
+        if key ~= "sequence" and key ~= "req" then
+            -- new version of lua-protobuf will add a new field 'oneof_name = oneof_type'
+            -- so we also need to filter it out (in this case, the 'req' key)
             return key, value
         end
     end

apisix/plugins/server-info.lua

@@ -172,7 +172,11 @@ local function report(premature, report_ttl)
             return
 
         else
-            server_info.etcd_version = res.body.etcdcluster
+            if res.body.etcdcluster == "" then
+                server_info.etcd_version = res.body.etcdserver
+            else
+                server_info.etcd_version = res.body.etcdcluster
+            end
         end
     end
 

ci/linux_openresty_common_runner.sh

@@ -38,7 +38,7 @@ do_install() {
     # sudo apt-get install tree -y
     # tree deps
 
-    git clone https://github.com/openresty/test-nginx.git test-nginx
+    git clone --depth 1 https://github.com/openresty/test-nginx.git test-nginx
     make utils
 
     mkdir -p build-cache

ci/linux_openresty_runner.sh

@@ -18,4 +18,5 @@
 
 
 export OPENRESTY_VERSION=source
+export TEST_CI_USE_GRPC=true
 . ./ci/linux_openresty_common_runner.sh

rockspec/apisix-master-0.rockspec

@@ -34,7 +34,7 @@ dependencies = {
     "lua-resty-ctxdump = 0.1-0",
     "lua-resty-dns-client = 6.0.2",
     "lua-resty-template = 2.0",
-    "lua-resty-etcd = 1.9.0",
+    "lua-resty-etcd = 1.10.0",
     "api7-lua-resty-http = 0.2.0",
     "lua-resty-balancer = 0.04",
     "lua-resty-ngxvar = 0.5.2",
@@ -46,7 +46,7 @@ dependencies = {
     "lua-resty-session = 3.10",
     "opentracing-openresty = 0.1",
     "lua-resty-radixtree = 2.8.2",
-    "lua-protobuf = 0.3.4",
+    "api7-lua-protobuf = 0.1.1",
     "lua-resty-openidc = 1.7.5",
     "luafilesystem = 1.7.0-2",
     "api7-lua-tinyyaml = 0.4.2",

t/APISIX.pm

@@ -837,6 +837,17 @@ deployment:
 _EOC_
 
     if ($yaml_config !~ m/deployment:/) {
+        # TODO: remove this temporary option once we have using gRPC by default
+        if ($ENV{TEST_CI_USE_GRPC}) {
+            $default_deployment .= <<_EOC_;
+  etcd:
+    host:
+      - "http://127.0.0.1:2379"
+    prefix: /apisix
+    use_grpc: true
+_EOC_
+        }
+
         $yaml_config = $default_deployment . $yaml_config;
     }
 

t/plugin/grpc-transcode2.t

@@ -565,7 +565,10 @@ qr/request log: \{.*body":\"\{\\"result\\":3}/
     local pb = require("pb")
     local old_f = pb.option
     pb.option = function(o)
-        ngx.log(ngx.WARN, "set protobuf option: ", o)
+        if o ~= "int64_as_string" then
+            -- filter out options set by other components
+            ngx.log(ngx.WARN, "set protobuf option: ", o)
+        end
         return old_f(o)
     end
 --- config
@@ -669,13 +672,11 @@ qr/request log: \{.*body":\"\{\\"result\\":3}/
 qr/set protobuf option: \w+/
 --- grep_error_log_out
 set protobuf option: enum_as_name
-set protobuf option: int64_as_string
 set protobuf option: auto_default_values
 set protobuf option: disable_hooks
 set protobuf option: enum_as_name
 set protobuf option: int64_as_number
 set protobuf option: enum_as_name
-set protobuf option: int64_as_string
 
 
 
@@ -684,7 +685,10 @@ set protobuf option: int64_as_string
     local pb = require("pb")
     local old_f = pb.option
     pb.option = function(o)
-        ngx.log(ngx.WARN, "set protobuf option: ", o)
+        if o ~= "int64_as_string" then
+            -- filter out options set by other components
+            ngx.log(ngx.WARN, "set protobuf option: ", o)
+        end
         return old_f(o)
     end
 --- config