diff --git a/apisix/cli/file.lua b/apisix/cli/file.lua index 66600b54b41b..5bd64a682e96 100644 --- a/apisix/cli/file.lua +++ b/apisix/cli/file.lua @@ -237,6 +237,14 @@ function _M.read_yaml_conf(apisix_home) end end + if default_conf.deployment + and default_conf.deployment.role == "traditional" + and default_conf.deployment.etcd + then + default_conf.etcd = default_conf.deployment.etcd + default_conf.etcd.unix_socket_proxy = "unix:./conf/config_listen.sock" + end + if default_conf.apisix.config_center == "yaml" then local apisix_conf_path = profile:yaml_path("apisix") local apisix_conf_yaml, _ = util.read_file(apisix_conf_path) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index cf8c08b38bb7..161c530b8d74 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -64,8 +64,9 @@ lua { {% end %} } -{% if enabled_stream_plugins["prometheus"] and not enable_http then %} +{% if (enabled_stream_plugins["prometheus"] or conf_server) and not enable_http then %} http { + {% if enabled_stream_plugins["prometheus"] then %} init_worker_by_lua_block { require("apisix.plugins.prometheus.exporter").http_init(true) } @@ -88,6 +89,11 @@ http { stub_status; } } + {% end %} + + {% if conf_server then %} + {* conf_server *} + {% end %} } {% end %} @@ -570,6 +576,10 @@ http { } {% end %} + {% if conf_server then %} + {* conf_server *} + {% end %} + server { {% for _, item in ipairs(node_listen) do %} listen {* item.ip *}:{* item.port *} default_server {% if item.enable_http2 then %} http2 {% end %} {% if enable_reuseport then %} reuseport {% end %}; diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 0be0701f6827..1e27d9a206d5 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -21,6 +21,7 @@ local file = require("apisix.cli.file") local schema = require("apisix.cli.schema") local ngx_tpl = require("apisix.cli.ngx_tpl") local cli_ip = require("apisix.cli.ip") +local snippet = require("apisix.cli.snippet") local profile = require("apisix.core.profile") local template = require("resty.template") local argparse = require("argparse") @@ -538,6 +539,8 @@ Please modify "admin_key" in conf/config.yaml . proxy_mirror_timeouts = yaml_conf.plugin_attr["proxy-mirror"].timeout end + local conf_server = snippet.generate_conf_server(yaml_conf) + -- Using template.render local sys_conf = { use_openresty_1_17 = use_openresty_1_17, @@ -557,6 +560,7 @@ Please modify "admin_key" in conf/config.yaml . control_server_addr = control_server_addr, prometheus_server_addr = prometheus_server_addr, proxy_mirror_timeouts = proxy_mirror_timeouts, + conf_server = conf_server, } if not yaml_conf.apisix then diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua index 7afece3ab239..ab053a0a727e 100644 --- a/apisix/cli/schema.lua +++ b/apisix/cli/schema.lua @@ -22,6 +22,43 @@ local require = require local _M = {} +local etcd_schema = { + type = "object", + properties = { + resync_delay = { + type = "integer", + }, + user = { + type = "string", + }, + password = { + type = "string", + }, + tls = { + type = "object", + properties = { + cert = { + type = "string", + }, + key = { + type = "string", + }, + } + }, + prefix = { + type = "string", + pattern = [[^/[^/]+$]] + }, + host = { + type = "array", + items = { + type = "string", + pattern = [[^https?://]] + } + } + }, + required = {"prefix", "host"} +} local config_schema = { type = "object", properties = { @@ -190,43 +227,7 @@ local config_schema = { } } }, - etcd = { - type = "object", - properties = { - resync_delay = { - type = "integer", - }, - user = { - type = "string", - }, - password = { - type = "string", - }, - tls = { - type = "object", - properties = { - cert = { - type = "string", - }, - key = { - type = "string", - }, - } - }, - prefix = { - type = "string", - pattern = [[^/[^/]+$]] - }, - host = { - type = "array", - items = { - type = "string", - pattern = [[^https?://]] - } - } - }, - required = {"prefix", "host"} - }, + etcd = etcd_schema, wasm = { type = "object", properties = { @@ -255,8 +256,25 @@ local config_schema = { } } }, + deployment = { + type = "object", + properties = { + role = { + enum = {"traditional", "control_plane", "data_plane", "standalone"} + } + }, + required = {"role"}, + }, } } +local deployment_schema = { + traditional = { + properties = { + etcd = etcd_schema, + }, + required = {"etcd"} + }, +} function _M.validate(yaml_conf) @@ -279,6 +297,15 @@ function _M.validate(yaml_conf) end end + if yaml_conf.deployment then + local role = yaml_conf.deployment.role + local validator = jsonschema.generate_validator(deployment_schema[role]) + local ok, err = validator(yaml_conf.deployment) + if not ok then + return false, "invalid deployment " .. role .. " configuration: " .. err + end + end + return true end diff --git a/apisix/cli/snippet.lua b/apisix/cli/snippet.lua new file mode 100644 index 000000000000..014719511faa --- /dev/null +++ b/apisix/cli/snippet.lua @@ -0,0 +1,65 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local template = require("resty.template") +local ipairs = ipairs + + +-- this module provide methods to generate snippets which will be used in the nginx.conf template +local _M = {} + + +function _M.generate_conf_server(conf) + if not (conf.deployment and conf.deployment.role == "traditional") then + return nil + end + + -- we use proxy even the role is traditional so that we can test the proxy in daily dev + local servers = conf.deployment.etcd.host + for i, s in ipairs(servers) do + local prefix = "http://" + -- TODO: support https + if s:find(prefix, 1, true) then + servers[i] = s:sub(#prefix + 1) + end + end + + local conf_render = template.compile([[ + upstream apisix_conf_backend { + {% for _, addr in ipairs(servers) do %} + server {* addr *}; + {% end %} + } + server { + listen unix:./conf/config_listen.sock; + access_log off; + location / { + set $upstream_scheme 'http'; + + proxy_pass $upstream_scheme://apisix_conf_backend; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + } + ]]) + return conf_render({ + servers = servers + }) +end + + +return _M diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua index f022fa96552e..8736059f7bf5 100644 --- a/apisix/core/config_etcd.lua +++ b/apisix/core/config_etcd.lua @@ -816,9 +816,13 @@ function _M.init() return nil, "failed to start a etcd instance: " .. err end + -- don't go through proxy during start because the proxy is not available + local proxy = etcd_cli.unix_socket_proxy + etcd_cli.unix_socket_proxy = nil local etcd_conf = local_conf.etcd local prefix = etcd_conf.prefix local res, err = readdir(etcd_cli, prefix, create_formatter(prefix)) + etcd_cli.unix_socket_proxy = proxy if not res then return nil, err end diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 8f9e58e5df17..c33e4a731231 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -477,3 +477,14 @@ plugin_attr: send: 60s # redirect: # https_port: 8443 # the default port for use by HTTP redirects to HTTPS + +#deployment: +# role: traditional +# role_traditional: +# config_provider: etcd +# etcd: +# host: # it's possible to define multiple etcd hosts addresses of the same etcd cluster. +# - "http://127.0.0.1:2379" # multiple etcd address, if your etcd cluster enables TLS, please use https scheme, +# # e.g. https://127.0.0.1:2379. +# prefix: /apisix # configuration prefix in etcd +# timeout: 30 # 30 seconds diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 9fc43aaa07bf..f5bfdae206f7 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -34,7 +34,7 @@ dependencies = { "lua-resty-ctxdump = 0.1-0", "lua-resty-dns-client = 6.0.2", "lua-resty-template = 2.0", - "lua-resty-etcd = 1.6.2", + "lua-resty-etcd = 1.7.0", "api7-lua-resty-http = 0.2.0", "lua-resty-balancer = 0.04", "lua-resty-ngxvar = 0.5.2", diff --git a/t/cli/test_deployment_traditional.sh b/t/cli/test_deployment_traditional.sh new file mode 100755 index 000000000000..f6d7d62c981b --- /dev/null +++ b/t/cli/test_deployment_traditional.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. ./t/cli/common.sh + +echo ' +deployment: + role: traditional + role_traditional: + config_provider: etcd +' > conf/config.yaml + +out=$(make init 2>&1 || true) +if ! echo "$out" | grep 'invalid deployment traditional configuration: property "etcd" is required'; then + echo "failed: should check deployment schema during init" + exit 1 +fi + +echo "passed: should check deployment schema during init" + +# HTTP +echo ' +deployment: + role: traditional + role_traditional: + config_provider: etcd + etcd: + prefix: "/apisix" + host: + - http://127.0.0.1:2379 +' > conf/config.yaml + +make run +sleep 1 + +code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1') +make stop + +if [ ! $code -eq 200 ]; then + echo "failed: could not connect to etcd with http enabled" + exit 1 +fi + +# Both HTTP and Stream +echo ' +apisix: + enable_admin: true + stream_proxy: + tcp: + - addr: 9100 +deployment: + role: traditional + role_traditional: + config_provider: etcd + etcd: + prefix: "/apisix" + host: + - http://127.0.0.1:2379 +' > conf/config.yaml + +make run +sleep 1 + +code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1') +make stop + +if [ ! $code -eq 200 ]; then + echo "failed: could not connect to etcd with http & stream enabled" + exit 1 +fi + +# Stream +echo ' +apisix: + enable_admin: false + stream_proxy: + tcp: + - addr: 9100 +deployment: + role: traditional + role_traditional: + config_provider: etcd + etcd: + prefix: "/apisix" + host: + - http://127.0.0.1:2379 +' > conf/config.yaml + +make run +sleep 1 + +if grep '\[error\]' logs/error.log; then + echo "failed: could not connect to etcd with stream enabled" + exit 1 +fi + +echo "passed: could connect to etcd"