feat(deployment): add structure of traditional role (#7249)

Signed-off-by: spacewander <spacewanderlzx@gmail.com>

apisix/cli/file.lua

@@ -237,6 +237,14 @@ function _M.read_yaml_conf(apisix_home)
         end
     end
 
+    if default_conf.deployment
+        and default_conf.deployment.role == "traditional"
+        and default_conf.deployment.etcd
+    then
+        default_conf.etcd = default_conf.deployment.etcd
+        default_conf.etcd.unix_socket_proxy = "unix:./conf/config_listen.sock"
+    end
+
     if default_conf.apisix.config_center == "yaml" then
         local apisix_conf_path = profile:yaml_path("apisix")
         local apisix_conf_yaml, _ = util.read_file(apisix_conf_path)

apisix/cli/ngx_tpl.lua

@@ -64,8 +64,9 @@ lua {
     {% end %}
 }
 
-{% if enabled_stream_plugins["prometheus"] and not enable_http then %}
+{% if (enabled_stream_plugins["prometheus"] or conf_server) and not enable_http then %}
 http {
+    {% if enabled_stream_plugins["prometheus"] then %}
     init_worker_by_lua_block {
         require("apisix.plugins.prometheus.exporter").http_init(true)
     }
@@ -88,6 +89,11 @@ http {
             stub_status;
         }
     }
+    {% end %}
+
+    {% if conf_server then %}
+    {* conf_server *}
+    {% end %}
 }
 {% end %}
 
@@ -570,6 +576,10 @@ http {
     }
     {% end %}
 
+    {% if conf_server then %}
+    {* conf_server *}
+    {% end %}
+
     server {
         {% for _, item in ipairs(node_listen) do %}
         listen {* item.ip *}:{* item.port *} default_server {% if item.enable_http2 then %} http2 {% end %} {% if enable_reuseport then %} reuseport {% end %};

apisix/cli/ops.lua

@@ -21,6 +21,7 @@ local file = require("apisix.cli.file")
 local schema = require("apisix.cli.schema")
 local ngx_tpl = require("apisix.cli.ngx_tpl")
 local cli_ip = require("apisix.cli.ip")
+local snippet = require("apisix.cli.snippet")
 local profile = require("apisix.core.profile")
 local template = require("resty.template")
 local argparse = require("argparse")
@@ -538,6 +539,8 @@ Please modify "admin_key" in conf/config.yaml .
         proxy_mirror_timeouts = yaml_conf.plugin_attr["proxy-mirror"].timeout
     end
 
+    local conf_server = snippet.generate_conf_server(yaml_conf)
+
     -- Using template.render
     local sys_conf = {
         use_openresty_1_17 = use_openresty_1_17,
@@ -557,6 +560,7 @@ Please modify "admin_key" in conf/config.yaml .
         control_server_addr = control_server_addr,
         prometheus_server_addr = prometheus_server_addr,
         proxy_mirror_timeouts = proxy_mirror_timeouts,
+        conf_server = conf_server,
     }
 
     if not yaml_conf.apisix then

apisix/cli/schema.lua

@@ -22,6 +22,43 @@ local require = require
 
 
 local _M = {}
+local etcd_schema = {
+    type = "object",
+    properties = {
+        resync_delay = {
+            type = "integer",
+        },
+        user = {
+            type = "string",
+        },
+        password = {
+            type = "string",
+        },
+        tls = {
+            type = "object",
+            properties = {
+                cert = {
+                    type = "string",
+                },
+                key = {
+                    type = "string",
+                },
+            }
+        },
+        prefix = {
+            type = "string",
+            pattern = [[^/[^/]+$]]
+        },
+        host = {
+            type = "array",
+            items = {
+                type = "string",
+                pattern = [[^https?://]]
+            }
+        }
+    },
+    required = {"prefix", "host"}
+}
 local config_schema = {
     type = "object",
     properties = {
@@ -190,43 +227,7 @@ local config_schema = {
                 }
             }
         },
-        etcd = {
-            type = "object",
-            properties = {
-                resync_delay = {
-                    type = "integer",
-                },
-                user = {
-                    type = "string",
-                },
-                password = {
-                    type = "string",
-                },
-                tls = {
-                    type = "object",
-                    properties = {
-                        cert = {
-                            type = "string",
-                        },
-                        key = {
-                            type = "string",
-                        },
-                    }
-                },
-                prefix = {
-                    type = "string",
-                    pattern = [[^/[^/]+$]]
-                },
-                host = {
-                    type = "array",
-                    items = {
-                        type = "string",
-                        pattern = [[^https?://]]
-                    }
-                }
-            },
-            required = {"prefix", "host"}
-        },
+        etcd = etcd_schema,
         wasm = {
             type = "object",
             properties = {
@@ -255,8 +256,25 @@ local config_schema = {
                 }
             }
         },
+        deployment = {
+            type = "object",
+            properties = {
+                role = {
+                    enum = {"traditional", "control_plane", "data_plane", "standalone"}
+                }
+            },
+            required = {"role"},
+        },
     }
 }
+local deployment_schema = {
+    traditional = {
+        properties = {
+            etcd = etcd_schema,
+        },
+        required = {"etcd"}
+    },
+}
 
 
 function _M.validate(yaml_conf)
@@ -279,6 +297,15 @@ function _M.validate(yaml_conf)
         end
     end
 
+    if yaml_conf.deployment then
+        local role = yaml_conf.deployment.role
+        local validator = jsonschema.generate_validator(deployment_schema[role])
+        local ok, err = validator(yaml_conf.deployment)
+        if not ok then
+            return false, "invalid deployment " .. role .. " configuration: " .. err
+        end
+    end
+
     return true
 end
 

apisix/cli/snippet.lua

@@ -0,0 +1,65 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local template = require("resty.template")
+local ipairs = ipairs
+
+
+-- this module provide methods to generate snippets which will be used in the nginx.conf template
+local _M = {}
+
+
+function _M.generate_conf_server(conf)
+    if not (conf.deployment and conf.deployment.role == "traditional") then
+        return nil
+    end
+
+    -- we use proxy even the role is traditional so that we can test the proxy in daily dev
+    local servers = conf.deployment.etcd.host
+    for i, s in ipairs(servers) do
+        local prefix = "http://"
+        -- TODO: support https
+        if s:find(prefix, 1, true) then
+            servers[i] = s:sub(#prefix + 1)
+        end
+    end
+
+    local conf_render = template.compile([[
+    upstream apisix_conf_backend {
+        {% for _, addr in ipairs(servers) do %}
+        server {* addr *};
+        {% end %}
+    }
+    server {
+        listen unix:./conf/config_listen.sock;
+        access_log off;
+        location / {
+            set $upstream_scheme             'http';
+
+            proxy_pass $upstream_scheme://apisix_conf_backend;
+
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+        }
+    }
+    ]])
+    return conf_render({
+        servers = servers
+    })
+end
+
+
+return _M

apisix/core/config_etcd.lua

@@ -816,9 +816,13 @@ function _M.init()
         return nil, "failed to start a etcd instance: " .. err
     end
 
+    -- don't go through proxy during start because the proxy is not available
+    local proxy = etcd_cli.unix_socket_proxy
+    etcd_cli.unix_socket_proxy = nil
     local etcd_conf = local_conf.etcd
     local prefix = etcd_conf.prefix
     local res, err = readdir(etcd_cli, prefix, create_formatter(prefix))
+    etcd_cli.unix_socket_proxy = proxy
     if not res then
         return nil, err
     end

conf/config-default.yaml

@@ -477,3 +477,14 @@ plugin_attr:
       send: 60s
 #  redirect:
 #    https_port: 8443   # the default port for use by HTTP redirects to HTTPS
+
+#deployment:
+#    role: traditional
+#    role_traditional:
+#        config_provider: etcd
+#    etcd:
+#        host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
+#          - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
+#                                        # e.g. https://127.0.0.1:2379.
+#        prefix: /apisix                 # configuration prefix in etcd
+#        timeout: 30                     # 30 seconds

rockspec/apisix-master-0.rockspec

@@ -34,7 +34,7 @@ dependencies = {
     "lua-resty-ctxdump = 0.1-0",
     "lua-resty-dns-client = 6.0.2",
     "lua-resty-template = 2.0",
-    "lua-resty-etcd = 1.6.2",
+    "lua-resty-etcd = 1.7.0",
     "api7-lua-resty-http = 0.2.0",
     "lua-resty-balancer = 0.04",
     "lua-resty-ngxvar = 0.5.2",