From 079eefb1da5946a22d1fe91c75e919736e98d527 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 14:42:26 +0800 Subject: [PATCH 01/28] feat: support reading configuration form shdict(mvp) --- apisix/cli/ngx_tpl.lua | 4 ++ apisix/cli/ops.lua | 1 + apisix/cli/schema.lua | 2 +- apisix/core/config_shdict.lua | 126 ++++++++++++++++++++++++++++++++++ apisix/init.lua | 5 ++ t/APISIX.pm | 3 +- t/amesh-agent/config_shdict.t | 84 +++++++++++++++++++++++ t/amesh-agent/main.go | 91 ++++++++++++++++++++++++ 8 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 apisix/core/config_shdict.lua create mode 100644 t/amesh-agent/config_shdict.t create mode 100644 t/amesh-agent/main.go diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index aaac144adb65..7423dcec56cf 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -235,6 +235,10 @@ http { lua_shared_dict ext-plugin {* http.lua_shared_dict["ext-plugin"] *}; # cache for ext-plugin {% end %} + {% if config_center == "shdict" then %} + lua_shared_dict router-config 10m; + {% end %} + # for custom shared dict {% if http.custom_lua_shared_dict then %} {% for cache_key, cache_size in pairs(http.custom_lua_shared_dict) do %} diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 219afc42046f..36888ccef108 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -554,6 +554,7 @@ Please modify "admin_key" in conf/config.yaml . end sys_conf["wasm"] = yaml_conf.wasm + sys_conf["config_center"] = yaml_conf.apisix.config_center local wrn = sys_conf["worker_rlimit_nofile"] local wc = sys_conf["event"]["worker_connections"] diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua index e479074560e0..dae703c25faa 100644 --- a/apisix/cli/schema.lua +++ b/apisix/cli/schema.lua @@ -28,7 +28,7 @@ local config_schema = { apisix = { properties = { config_center = { - enum = {"etcd", "yaml"}, + enum = {"etcd", "yaml", "shdict"}, }, lua_module_hook = { pattern = "^[a-zA-Z._-]+$", diff --git a/apisix/core/config_shdict.lua b/apisix/core/config_shdict.lua new file mode 100644 index 000000000000..49499458ae51 --- /dev/null +++ b/apisix/core/config_shdict.lua @@ -0,0 +1,126 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Get configuration form ngx.shared.DICT. +-- +-- @module core.config_shdict + +local base = require("resty.core.base") +local config_local = require("apisix.core.config_local") +local table = table +local is_http = ngx.config.subsystem == "http" +local string = string +local io = io +local package = package +local new_tab = base.new_tab +local ngx_timer_at = ngx.timer.at +local ffi = require 'ffi' +local C = ffi.C +local router_config = ngx.shared["router-config"] + +local process +if is_http then + process = require("ngx.process") +end + + +ffi.cdef[[ +extern void CreateMock(void* writeRoute); +]] + + +local _M = { + version = 0.1, + local_conf = config_local.local_conf, +} + + +-- todo: refactor this function in chash.lua and radixtree.lua +local function load_shared_lib(lib_name) + local string_gmatch = string.gmatch + local string_match = string.match + local io_open = io.open + local io_close = io.close + + local cpath = package.cpath + local tried_paths = new_tab(32, 0) + local i = 1 + + for k, _ in string_gmatch(cpath, "[^;]+") do + local fpath = string_match(k, "(.*/)") + fpath = fpath .. lib_name + + local f = io_open(fpath) + if f ~= nil then + io_close(f) + return ffi.load(fpath) + end + tried_paths[i] = fpath + i = i + 1 + end + + return nil, tried_paths +end + + +local function load_ameshagent(lib_name) + ngx_timer_at(0, function(premature) + if premature then + return + end + + local ameshagent, tried_paths = load_shared_lib(lib_name) + + if not ameshagent then + tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' + .. lib_name + error("can not load amesh agent, tried paths: " .. + table.concat(tried_paths, '\r\n', 1, #tried_paths)) + end + + local router_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(router_config[1]) + local router_shd_cdata = ffi.cast("void*", router_zone) + ameshagent.CreateMock(router_shd_cdata) + end) +end + + + +function _M.init_worker() + local lib_name = "libameshagent" + + local postfix = ".so" + if ffi.os == "OSX" then + postfix = ".dylib" + end + lib_name = lib_name .. postfix + + if process.type() == "privileged agent" then + load_ameshagent(lib_name) + end + + return true +end + + +function _M.new(key, opts) + -- mock for test + return { true } +end + + +return _M diff --git a/apisix/init.lua b/apisix/init.lua index 1298e1c20683..6157411ce0ef 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -110,6 +110,11 @@ function _M.http_init_worker() end require("apisix.balancer").init_worker() load_balancer = require("apisix.balancer") + + if core.config == require("apisix.core.config_shdict") then + core.config.init_worker() + end + require("apisix.admin.init").init_worker() require("apisix.timers").init_worker() diff --git a/t/APISIX.pm b/t/APISIX.pm index 671ffe36b01d..e461380124d8 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -239,7 +239,7 @@ apisix: _EOC_ } - my $lua_deps_path = <<_EOC_; + my $lua_deps_path = $block->lua_deps_path // <<_EOC_; lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; lua_package_cpath "$apisix_home/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; _EOC_ @@ -507,6 +507,7 @@ _EOC_ lua_capture_error_log 1m; # plugin error-log-logger lua_shared_dict etcd-cluster-health-check 10m; # etcd health check lua_shared_dict ext-plugin 1m; + lua_shared_dict router-config 1m; proxy_ssl_name \$upstream_host; proxy_ssl_server_name on; diff --git a/t/amesh-agent/config_shdict.t b/t/amesh-agent/config_shdict.t new file mode 100644 index 000000000000..565c9355852f --- /dev/null +++ b/t/amesh-agent/config_shdict.t @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +use Cwd qw(cwd); +my $apisix_home = $ENV{APISIX_HOME} || cwd(); + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } + + my $lua_deps_path = $block->lua_deps_path // <<_EOC_; + lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; + lua_package_cpath "$apisix_home/?.so;$apisix_home/t/amesh-agent/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; +_EOC_ + + $block->set_value("lua_deps_path", $lua_deps_path); +}); + +run_tests; + +__DATA__ + +=== TEST 1: load amesh agent so successfully +--- yaml_config +apisix: + node_listen: 1984 + config_center: shdict + enable_admin: false +--- config + location /t { + content_by_lua_block { + ngx.say("ok") + } + } +--- no_error_log eval +qr/can not load amesh agent/ + + + +=== TEST 2: read data form shdict that wirted by amesh agent +--- yaml_config +apisix: + node_listen: 1984 + config_center: shdict + enable_admin: false +--- config + location /t { + content_by_lua_block { + -- wait for amesh agent sync data + ngx.sleep(1.5) + local value = ngx.shared["router-config"]:get("/apisix/routes/1") + local json_encode = require("toolkit.json").encode + ngx.say(json_encode(value)) + } + } +--- response_body +"{\n\"status\": 1,\n\"update_time\": 1647250524,\n\"create_time\": 1646972532,\n\"uri\": \"/hello\",\n\"priority\": 0,\n\"id\": \"1\",\n\"upstream\": {\n\t\"nodes\": [\n\t\t{\n\t\t\t\"port\": 80,\n\t\t\t\"priority\": 0,\n\t\t\t\"host\": \"127.0.0.1\",\n\t\t\t\"weight\": 1\n\t\t}\n\t],\n\t\"type\": \"roundrobin\",\n\t\"hash_on\": \"vars\",\n\t\"pass_host\": \"pass\",\n\t\"scheme\": \"http\"\n}\n}%!(EXTRA int64=1647326314)" diff --git a/t/amesh-agent/main.go b/t/amesh-agent/main.go new file mode 100644 index 000000000000..21c2244b14a0 --- /dev/null +++ b/t/amesh-agent/main.go @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +/* +#cgo LDFLAGS: -shared +#include + +extern void ngx_http_lua_ffi_shdict_store(void *zone, int op, + const unsigned char *key, size_t key_len, + int value_type, + const unsigned char *str_value_buf, size_t str_value_len, + double num_value, long exptime, int user_flags, char **errmsg, + int *forcible); +*/ +import "C" + +import ( + "fmt" + "time" + "unsafe" +) + +func main() { +} + +//export CreateMock +func CreateMock(zone unsafe.Pointer) { + time.Sleep(time.Second) + value := fmt.Sprintf(`{ +"status": 1, +"update_time": 1647250524, +"create_time": 1646972532, +"uri": "/hello", +"priority": 0, +"id": "1", +"upstream": { + "nodes": [ + { + "port": 80, + "priority": 0, + "host": "127.0.0.1", + "weight": 1 + } + ], + "type": "roundrobin", + "hash_on": "vars", + "pass_host": "pass", + "scheme": "http" +} +}`, time.Now().Unix()) + + writeShdict(zone, "/apisix/routes/1", value) +} + +func writeShdict(zone unsafe.Pointer, key, value string) { + var keyCStr = C.CString(key) + defer C.free(unsafe.Pointer(keyCStr)) + var keyLen = C.size_t(len(key)) + + var valueCStr = C.CString(value) + defer C.free(unsafe.Pointer(valueCStr)) + var valueLen = C.size_t(len(value)) + + errMsgBuf := make([]*C.char, 1) + var forcible = 0 + + C.ngx_http_lua_ffi_shdict_store(zone, 0x0004, + (*C.uchar)(unsafe.Pointer(keyCStr)), keyLen, + 4, + (*C.uchar)(unsafe.Pointer(valueCStr)), valueLen, + 0, 0, 0, + (**C.char)(unsafe.Pointer(&errMsgBuf[0])), + (*C.int)(unsafe.Pointer(&forcible)), + ) +} From 808bd6e1b54792d85b75194ecc779ed6dc010e90 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 15:49:16 +0800 Subject: [PATCH 02/28] ci --- .github/workflows/build.yml | 8 +++++++- t/amesh-agent/config_shdict.t | 6 ++++-- t/amesh-agent/main.go | 4 ++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef861c828288..49a25292cf5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: test_dir: - t/plugin - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/discovery t/error_page t/misc - - t/node t/router t/script t/stream-node t/utils t/wasm + - t/node t/router t/script t/stream-node t/utils t/wasm t/amesh-agent runs-on: ${{ matrix.platform }} timeout-minutes: 90 @@ -90,6 +90,12 @@ jobs: sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p + - name: Build amesh agent lib + if: matrix.os_name == 'linux_openresty' + run: | + cd t/amesh-agent + go build -o libameshagent.so -buildmode=c-shared main.go + - name: Linux Before install run: sudo ./ci/${{ matrix.os_name }}_runner.sh before_install diff --git a/t/amesh-agent/config_shdict.t b/t/amesh-agent/config_shdict.t index 565c9355852f..2ace37f1a28e 100644 --- a/t/amesh-agent/config_shdict.t +++ b/t/amesh-agent/config_shdict.t @@ -75,10 +75,12 @@ apisix: content_by_lua_block { -- wait for amesh agent sync data ngx.sleep(1.5) + local core = require("apisix.core") local value = ngx.shared["router-config"]:get("/apisix/routes/1") + local route_conf, err = core.json.decode(value) local json_encode = require("toolkit.json").encode - ngx.say(json_encode(value)) + ngx.say(json_encode(route_conf)) } } --- response_body -"{\n\"status\": 1,\n\"update_time\": 1647250524,\n\"create_time\": 1646972532,\n\"uri\": \"/hello\",\n\"priority\": 0,\n\"id\": \"1\",\n\"upstream\": {\n\t\"nodes\": [\n\t\t{\n\t\t\t\"port\": 80,\n\t\t\t\"priority\": 0,\n\t\t\t\"host\": \"127.0.0.1\",\n\t\t\t\"weight\": 1\n\t\t}\n\t],\n\t\"type\": \"roundrobin\",\n\t\"hash_on\": \"vars\",\n\t\"pass_host\": \"pass\",\n\t\"scheme\": \"http\"\n}\n}%!(EXTRA int64=1647326314)" +{"create_time":1646972532,"id":"1","priority":0,"status":1,"update_time":1647250524,"upstream":{"hash_on":"vars","nodes":[{"host":"127.0.0.1","port":80,"priority":0,"weight":1}],"pass_host":"pass","scheme":"http","type":"roundrobin"},"uri":"/hello"} diff --git a/t/amesh-agent/main.go b/t/amesh-agent/main.go index 21c2244b14a0..899e1a12c42a 100644 --- a/t/amesh-agent/main.go +++ b/t/amesh-agent/main.go @@ -62,8 +62,8 @@ func CreateMock(zone unsafe.Pointer) { "hash_on": "vars", "pass_host": "pass", "scheme": "http" -} -}`, time.Now().Unix()) + } +}`) writeShdict(zone, "/apisix/routes/1", value) } From 5ffd2bf2f5e93a9e4402c0e140e3e29d967678f9 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 16:09:00 +0800 Subject: [PATCH 03/28] fix code lint --- t/amesh-agent/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/amesh-agent/main.go b/t/amesh-agent/main.go index 899e1a12c42a..7340f2814b5d 100644 --- a/t/amesh-agent/main.go +++ b/t/amesh-agent/main.go @@ -62,7 +62,7 @@ func CreateMock(zone unsafe.Pointer) { "hash_on": "vars", "pass_host": "pass", "scheme": "http" - } +} }`) writeShdict(zone, "/apisix/routes/1", value) From 815074d03785d0613d80ad91a3d56beaefc34c7c Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 16:27:30 +0800 Subject: [PATCH 04/28] fix code lint --- apisix/core/config_shdict.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/apisix/core/config_shdict.lua b/apisix/core/config_shdict.lua index 49499458ae51..eb9e5629f845 100644 --- a/apisix/core/config_shdict.lua +++ b/apisix/core/config_shdict.lua @@ -22,6 +22,7 @@ local base = require("resty.core.base") local config_local = require("apisix.core.config_local") local table = table +local error = error local is_http = ngx.config.subsystem == "http" local string = string local io = io From 1ca0254b5cf3fbc2a2f5b6b644c8ad7b4bbcf204 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 17:16:31 +0800 Subject: [PATCH 05/28] fix CI --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49a25292cf5c..ece9a6bd39c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,7 +91,6 @@ jobs: cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p - name: Build amesh agent lib - if: matrix.os_name == 'linux_openresty' run: | cd t/amesh-agent go build -o libameshagent.so -buildmode=c-shared main.go From 6fa5a6b6c471c7522d8581b5c2d554c85d3d7cfa Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 21:18:07 +0800 Subject: [PATCH 06/28] fix code review --- .github/workflows/build.yml | 6 +++--- apisix/core/config_shdict.lua | 20 +++++++------------ .../config_shdict.t | 10 +++++----- t/{amesh-agent => amesh-library}/main.go | 4 ++-- 4 files changed, 17 insertions(+), 23 deletions(-) rename t/{amesh-agent => amesh-library}/config_shdict.t (90%) rename t/{amesh-agent => amesh-library}/main.go (97%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ece9a6bd39c0..89da4d2eecca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,10 +90,10 @@ jobs: sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p - - name: Build amesh agent lib + - name: Build Amesh library run: | - cd t/amesh-agent - go build -o libameshagent.so -buildmode=c-shared main.go + cd t/amesh-library + go build -o libamesh.so -buildmode=c-shared main.go - name: Linux Before install run: sudo ./ci/${{ matrix.os_name }}_runner.sh before_install diff --git a/apisix/core/config_shdict.lua b/apisix/core/config_shdict.lua index eb9e5629f845..e99b04f7c21a 100644 --- a/apisix/core/config_shdict.lua +++ b/apisix/core/config_shdict.lua @@ -29,7 +29,7 @@ local io = io local package = package local new_tab = base.new_tab local ngx_timer_at = ngx.timer.at -local ffi = require 'ffi' +local ffi = require ("ffi") local C = ffi.C local router_config = ngx.shared["router-config"] @@ -40,7 +40,7 @@ end ffi.cdef[[ -extern void CreateMock(void* writeRoute); +extern void initial(void* writeRoute); ]] @@ -88,27 +88,21 @@ local function load_ameshagent(lib_name) if not ameshagent then tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' - .. lib_name - error("can not load amesh agent, tried paths: " .. - table.concat(tried_paths, '\r\n', 1, #tried_paths)) + .. lib_name + error("can not load Amesh library, tried paths: " .. + table.concat(tried_paths, '\r\n', 1, #tried_paths)) end local router_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(router_config[1]) local router_shd_cdata = ffi.cast("void*", router_zone) - ameshagent.CreateMock(router_shd_cdata) + ameshagent.initial(router_shd_cdata) end) end function _M.init_worker() - local lib_name = "libameshagent" - - local postfix = ".so" - if ffi.os == "OSX" then - postfix = ".dylib" - end - lib_name = lib_name .. postfix + local lib_name = "libamesh.so" if process.type() == "privileged agent" then load_ameshagent(lib_name) diff --git a/t/amesh-agent/config_shdict.t b/t/amesh-library/config_shdict.t similarity index 90% rename from t/amesh-agent/config_shdict.t rename to t/amesh-library/config_shdict.t index 2ace37f1a28e..5e55836cacc3 100644 --- a/t/amesh-agent/config_shdict.t +++ b/t/amesh-library/config_shdict.t @@ -37,7 +37,7 @@ add_block_preprocessor(sub { my $lua_deps_path = $block->lua_deps_path // <<_EOC_; lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; - lua_package_cpath "$apisix_home/?.so;$apisix_home/t/amesh-agent/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; + lua_package_cpath "$apisix_home/?.so;$apisix_home/t/amesh-library/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; _EOC_ $block->set_value("lua_deps_path", $lua_deps_path); @@ -47,7 +47,7 @@ run_tests; __DATA__ -=== TEST 1: load amesh agent so successfully +=== TEST 1: load Amesh library so successfully --- yaml_config apisix: node_listen: 1984 @@ -60,11 +60,11 @@ apisix: } } --- no_error_log eval -qr/can not load amesh agent/ +qr/can not load Amesh library/ -=== TEST 2: read data form shdict that wirted by amesh agent +=== TEST 2: read data form shdict that wirted by Amesh library --- yaml_config apisix: node_listen: 1984 @@ -73,7 +73,7 @@ apisix: --- config location /t { content_by_lua_block { - -- wait for amesh agent sync data + -- wait for Amesh library sync data ngx.sleep(1.5) local core = require("apisix.core") local value = ngx.shared["router-config"]:get("/apisix/routes/1") diff --git a/t/amesh-agent/main.go b/t/amesh-library/main.go similarity index 97% rename from t/amesh-agent/main.go rename to t/amesh-library/main.go index 7340f2814b5d..4323911220d4 100644 --- a/t/amesh-agent/main.go +++ b/t/amesh-library/main.go @@ -39,8 +39,8 @@ import ( func main() { } -//export CreateMock -func CreateMock(zone unsafe.Pointer) { +//export initial +func initial(zone unsafe.Pointer) { time.Sleep(time.Second) value := fmt.Sprintf(`{ "status": 1, From 3123d8174cbbc3ed7399c33bcb8c3a039a6249e9 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 21:54:46 +0800 Subject: [PATCH 07/28] replace amesh mock method --- apisix/core/config_shdict.lua | 4 ++-- t/amesh-library/config_shdict.t | 2 +- t/amesh-library/main.go | 16 +++++++++++++--- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/apisix/core/config_shdict.lua b/apisix/core/config_shdict.lua index e99b04f7c21a..6bec3c4e8025 100644 --- a/apisix/core/config_shdict.lua +++ b/apisix/core/config_shdict.lua @@ -78,7 +78,7 @@ local function load_shared_lib(lib_name) end -local function load_ameshagent(lib_name) +local function load_libamesh(lib_name) ngx_timer_at(0, function(premature) if premature then return @@ -105,7 +105,7 @@ function _M.init_worker() local lib_name = "libamesh.so" if process.type() == "privileged agent" then - load_ameshagent(lib_name) + load_libamesh(lib_name) end return true diff --git a/t/amesh-library/config_shdict.t b/t/amesh-library/config_shdict.t index 5e55836cacc3..9ad6a5bde625 100644 --- a/t/amesh-library/config_shdict.t +++ b/t/amesh-library/config_shdict.t @@ -74,7 +74,7 @@ apisix: location /t { content_by_lua_block { -- wait for Amesh library sync data - ngx.sleep(1.5) + ngx.sleep(3.5) local core = require("apisix.core") local value = ngx.shared["router-config"]:get("/apisix/routes/1") local route_conf, err = core.json.decode(value) diff --git a/t/amesh-library/main.go b/t/amesh-library/main.go index 4323911220d4..994a8c39a568 100644 --- a/t/amesh-library/main.go +++ b/t/amesh-library/main.go @@ -31,7 +31,9 @@ extern void ngx_http_lua_ffi_shdict_store(void *zone, int op, import "C" import ( + "context" "fmt" + "math/rand" "time" "unsafe" ) @@ -39,10 +41,16 @@ import ( func main() { } + //export initial func initial(zone unsafe.Pointer) { - time.Sleep(time.Second) - value := fmt.Sprintf(`{ + ctx := context.Background() + for { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second * time.Duration(rand.Intn(3))): + value := fmt.Sprintf(`{ "status": 1, "update_time": 1647250524, "create_time": 1646972532, @@ -65,7 +73,9 @@ func initial(zone unsafe.Pointer) { } }`) - writeShdict(zone, "/apisix/routes/1", value) + writeShdict(zone, "/apisix/routes/1", value) + } + } } func writeShdict(zone unsafe.Pointer, key, value string) { From ede18daaba4e6fcca875658ec15ce999f960c2c0 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Tue, 15 Mar 2022 22:22:34 +0800 Subject: [PATCH 08/28] revert amesh initial function --- t/amesh-library/config_shdict.t | 2 +- t/amesh-library/main.go | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/t/amesh-library/config_shdict.t b/t/amesh-library/config_shdict.t index 9ad6a5bde625..5e55836cacc3 100644 --- a/t/amesh-library/config_shdict.t +++ b/t/amesh-library/config_shdict.t @@ -74,7 +74,7 @@ apisix: location /t { content_by_lua_block { -- wait for Amesh library sync data - ngx.sleep(3.5) + ngx.sleep(1.5) local core = require("apisix.core") local value = ngx.shared["router-config"]:get("/apisix/routes/1") local route_conf, err = core.json.decode(value) diff --git a/t/amesh-library/main.go b/t/amesh-library/main.go index 994a8c39a568..690d38b97e96 100644 --- a/t/amesh-library/main.go +++ b/t/amesh-library/main.go @@ -31,9 +31,7 @@ extern void ngx_http_lua_ffi_shdict_store(void *zone, int op, import "C" import ( - "context" "fmt" - "math/rand" "time" "unsafe" ) @@ -44,13 +42,8 @@ func main() { //export initial func initial(zone unsafe.Pointer) { - ctx := context.Background() - for { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second * time.Duration(rand.Intn(3))): - value := fmt.Sprintf(`{ + time.Sleep(time.Second) + value := fmt.Sprintf(`{ "status": 1, "update_time": 1647250524, "create_time": 1646972532, @@ -73,9 +66,7 @@ func initial(zone unsafe.Pointer) { } }`) - writeShdict(zone, "/apisix/routes/1", value) - } - } + writeShdict(zone, "/apisix/routes/1", value) } func writeShdict(zone unsafe.Pointer, key, value string) { From 5b8726ffe13da30865cdea6882a4c8d4e7d26e3c Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 16 Mar 2022 00:28:36 +0800 Subject: [PATCH 09/28] fix test cases lib name --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 89da4d2eecca..96d0436a3197 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: test_dir: - t/plugin - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/discovery t/error_page t/misc - - t/node t/router t/script t/stream-node t/utils t/wasm t/amesh-agent + - t/node t/router t/script t/stream-node t/utils t/wasm t/amesh-library runs-on: ${{ matrix.platform }} timeout-minutes: 90 From d4c8c1945905771fc204734831fc7e7dfb5dc520 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 16 Mar 2022 11:08:56 +0800 Subject: [PATCH 10/28] fix CI error --- t/plugin/traffic-split.t | 58 +++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/t/plugin/traffic-split.t b/t/plugin/traffic-split.t index 423816f1f956..c2f3932d0fb9 100644 --- a/t/plugin/traffic-split.t +++ b/t/plugin/traffic-split.t @@ -676,40 +676,38 @@ qr/dns resolver domain: apiseven.com to \d+.\d+.\d+.\d+/ location /t { content_by_lua_block { local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "uri": "/server_port", - "plugins": { - "traffic-split": { - "rules": [ + local json = require("toolkit.json") + + local data = { + uri = "/server_port", + plugins = { + ["traffic-split"] = { + rules = {{ + weighted_upstreams = { { - "weighted_upstreams": [ - { - "upstream": { - "name": "upstream_A", - "type": "roundrobin", - "nodes": { - "127.0.0.1:1981":1 - } - }, - "weight": 2 - }, - { - "weight": 1 - } - ] + upstream = { + name = "upstream_A", + type = "roundrobin", + nodes = {["127.0.0.1:1981"] = 1} + }, + weight = 2 + }, + { + weight = 1 } - ] - } - }, - "upstream": { - "type": "roundrobin", - "nodes": { - "127.0.0.1:1980": 1 } + }} } - }]] + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + } + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) ) if code >= 300 then ngx.status = code From ed3fa36b015a577fd74f8a9e51f92d333ecbb078 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 16 Mar 2022 12:15:26 +0800 Subject: [PATCH 11/28] fix CI error --- t/plugin/traffic-split2.t | 59 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/t/plugin/traffic-split2.t b/t/plugin/traffic-split2.t index 853b98e84c42..d5919fcdac34 100644 --- a/t/plugin/traffic-split2.t +++ b/t/plugin/traffic-split2.t @@ -111,42 +111,39 @@ GET /server_port?name=jack&age=18 location /t { content_by_lua_block { local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [=[{ - "uri": "/uri", - "plugins": { - "traffic-split": { - "rules": [ + local json = require("toolkit.json") + + local data = { + uri = "/uri", + plugins = { + ["traffic-split"] = { + rules = {{ + match = { { + vars = { { "arg_name", "==", "jack" } } + } }, + weighted_upstreams = { { - "match": [ - { - "vars": [["arg_name", "==", "jack"]] - } - ], - "weighted_upstreams": [ - { - "upstream": { - "type": "roundrobin", - "pass_host": "pass", - "nodes": { - "127.0.0.1:1981":1 - } - } - } - ] + upstream = { + type = "roundrobin", + pass_host = "pass", + nodes = {["127.0.0.1:1981"] = 1} + } } - ] - } - }, - "upstream": { - "type": "roundrobin", - "nodes": { - "127.0.0.1:1980": 1 } + }} } - }]=] + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + } + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) ) + if code >= 300 then ngx.status = code end From 87e97c12d10b3170077c5ce66d35965bbf32bb3a Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 16 Mar 2022 14:07:17 +0800 Subject: [PATCH 12/28] fix CI error --- t/plugin/traffic-split2.t | 59 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/t/plugin/traffic-split2.t b/t/plugin/traffic-split2.t index d5919fcdac34..9499a8cd0818 100644 --- a/t/plugin/traffic-split2.t +++ b/t/plugin/traffic-split2.t @@ -231,42 +231,39 @@ x-real-ip: 127.0.0.1 location /t { content_by_lua_block { local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PATCH, - [=[{ - "uri": "/uri", - "plugins": { - "traffic-split": { - "rules": [ + local json = require("toolkit.json") + + local data = { + uri = "/uri", + plugins = { + ["traffic-split"] = { + rules = {{ + match = { { + vars = { { "arg_name", "==", "jack" } } + } }, + weighted_upstreams = { { - "match": [ - { - "vars": [["arg_name", "==", "jack"]] - } - ], - "weighted_upstreams": [ - { - "upstream": { - "type": "roundrobin", - "pass_host": "node", - "nodes": { - "localhost:1981":1 - } - } - } - ] + upstream = { + type = "roundrobin", + pass_host = "node", + nodes = {["127.0.0.1:1981"] = 1} + } } - ] - } - }, - "upstream": { - "type": "roundrobin", - "nodes": { - "127.0.0.1:1980": 1 } + }} } - }]=] + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + } + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) ) + if code >= 300 then ngx.status = code end From 519a8238492a11ca92687000e8c48a2b6654a148 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 16 Mar 2022 15:51:33 +0800 Subject: [PATCH 13/28] fix CI error --- t/plugin/traffic-split2.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/plugin/traffic-split2.t b/t/plugin/traffic-split2.t index 9499a8cd0818..49afb3582e5b 100644 --- a/t/plugin/traffic-split2.t +++ b/t/plugin/traffic-split2.t @@ -246,7 +246,7 @@ x-real-ip: 127.0.0.1 upstream = { type = "roundrobin", pass_host = "node", - nodes = {["127.0.0.1:1981"] = 1} + nodes = {["localhost:1981"] = 1} } } } From 3345a31189b348c372a9883f4c259f49939252df Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 00:10:36 +0800 Subject: [PATCH 14/28] revert test cases changed in traffic-split Signed-off-by: tzssangglass --- t/plugin/traffic-split.t | 58 ++++++++++--------- t/plugin/traffic-split2.t | 118 ++++++++++++++++++++------------------ 2 files changed, 92 insertions(+), 84 deletions(-) diff --git a/t/plugin/traffic-split.t b/t/plugin/traffic-split.t index c2f3932d0fb9..423816f1f956 100644 --- a/t/plugin/traffic-split.t +++ b/t/plugin/traffic-split.t @@ -676,38 +676,40 @@ qr/dns resolver domain: apiseven.com to \d+.\d+.\d+.\d+/ location /t { content_by_lua_block { local t = require("lib.test_admin").test - local json = require("toolkit.json") - - local data = { - uri = "/server_port", - plugins = { - ["traffic-split"] = { - rules = {{ - weighted_upstreams = { - { - upstream = { - name = "upstream_A", - type = "roundrobin", - nodes = {["127.0.0.1:1981"] = 1} - }, - weight = 2 - }, + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/server_port", + "plugins": { + "traffic-split": { + "rules": [ { - weight = 1 + "weighted_upstreams": [ + { + "upstream": { + "name": "upstream_A", + "type": "roundrobin", + "nodes": { + "127.0.0.1:1981":1 + } + }, + "weight": 2 + }, + { + "weight": 1 + } + ] } + ] + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 } - }} } - }, - upstream = { - type = "roundrobin", - nodes = {["127.0.0.1:1980"] = 1} - } - } - - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - json.encode(data) + }]] ) if code >= 300 then ngx.status = code diff --git a/t/plugin/traffic-split2.t b/t/plugin/traffic-split2.t index 49afb3582e5b..853b98e84c42 100644 --- a/t/plugin/traffic-split2.t +++ b/t/plugin/traffic-split2.t @@ -111,39 +111,42 @@ GET /server_port?name=jack&age=18 location /t { content_by_lua_block { local t = require("lib.test_admin").test - local json = require("toolkit.json") - - local data = { - uri = "/uri", - plugins = { - ["traffic-split"] = { - rules = {{ - match = { { - vars = { { "arg_name", "==", "jack" } } - } }, - weighted_upstreams = { + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "uri": "/uri", + "plugins": { + "traffic-split": { + "rules": [ { - upstream = { - type = "roundrobin", - pass_host = "pass", - nodes = {["127.0.0.1:1981"] = 1} - } + "match": [ + { + "vars": [["arg_name", "==", "jack"]] + } + ], + "weighted_upstreams": [ + { + "upstream": { + "type": "roundrobin", + "pass_host": "pass", + "nodes": { + "127.0.0.1:1981":1 + } + } + } + ] } + ] + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 } - }} } - }, - upstream = { - type = "roundrobin", - nodes = {["127.0.0.1:1980"] = 1} - } - } - - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - json.encode(data) + }]=] ) - if code >= 300 then ngx.status = code end @@ -231,39 +234,42 @@ x-real-ip: 127.0.0.1 location /t { content_by_lua_block { local t = require("lib.test_admin").test - local json = require("toolkit.json") - - local data = { - uri = "/uri", - plugins = { - ["traffic-split"] = { - rules = {{ - match = { { - vars = { { "arg_name", "==", "jack" } } - } }, - weighted_upstreams = { + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PATCH, + [=[{ + "uri": "/uri", + "plugins": { + "traffic-split": { + "rules": [ { - upstream = { - type = "roundrobin", - pass_host = "node", - nodes = {["localhost:1981"] = 1} - } + "match": [ + { + "vars": [["arg_name", "==", "jack"]] + } + ], + "weighted_upstreams": [ + { + "upstream": { + "type": "roundrobin", + "pass_host": "node", + "nodes": { + "localhost:1981":1 + } + } + } + ] } + ] + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 } - }} } - }, - upstream = { - type = "roundrobin", - nodes = {["127.0.0.1:1980"] = 1} - } - } - - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - json.encode(data) + }]=] ) - if code >= 300 then ngx.status = code end From 9a4cb1f2accaf43c425aba369b55036e30afb520 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 09:44:10 +0800 Subject: [PATCH 15/28] fix code review --- t/amesh-library/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/amesh-library/main.go b/t/amesh-library/main.go index 690d38b97e96..1463ecf91c9e 100644 --- a/t/amesh-library/main.go +++ b/t/amesh-library/main.go @@ -66,10 +66,10 @@ func initial(zone unsafe.Pointer) { } }`) - writeShdict(zone, "/apisix/routes/1", value) + write_route(zone, "/apisix/routes/1", value) } -func writeShdict(zone unsafe.Pointer, key, value string) { +func write_route(zone unsafe.Pointer, key, value string) { var keyCStr = C.CString(key) defer C.free(unsafe.Pointer(keyCStr)) var keyLen = C.size_t(len(key)) From 62190bc6c068c7d331b8faaf72894fe06f073bde Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 14:25:02 +0800 Subject: [PATCH 16/28] fix code review --- .github/workflows/build.yml | 2 +- apisix/cli/ngx_tpl.lua | 2 +- apisix/cli/ops.lua | 1 - apisix/cli/schema.lua | 2 +- .../{config_shdict.lua => config_xds.lua} | 35 ++++++------------- conf/config.yaml | 1 + .../{config_shdict.t => config_xds.t} | 22 ++++++++++-- 7 files changed, 35 insertions(+), 30 deletions(-) rename apisix/core/{config_shdict.lua => config_xds.lua} (74%) rename t/amesh-library/{config_shdict.t => config_xds.t} (85%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 96d0436a3197..1cc24a189d0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: test_dir: - t/plugin - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/discovery t/error_page t/misc - - t/node t/router t/script t/stream-node t/utils t/wasm t/amesh-library + - t/amesh-library t/node t/router t/script t/stream-node t/utils t/wasm runs-on: ${{ matrix.platform }} timeout-minutes: 90 diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 81ad4eb1f5f3..05efd45cf455 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -235,7 +235,7 @@ http { lua_shared_dict ext-plugin {* http.lua_shared_dict["ext-plugin"] *}; # cache for ext-plugin {% end %} - {% if config_center == "shdict" then %} + {% if config_center == "xds" then %} lua_shared_dict router-config 10m; {% end %} diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 38759608be62..7208e5c6d6eb 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -560,7 +560,6 @@ Please modify "admin_key" in conf/config.yaml . end sys_conf["wasm"] = yaml_conf.wasm - sys_conf["config_center"] = yaml_conf.apisix.config_center local wrn = sys_conf["worker_rlimit_nofile"] local wc = sys_conf["event"]["worker_connections"] diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua index dae703c25faa..8c7a873214c1 100644 --- a/apisix/cli/schema.lua +++ b/apisix/cli/schema.lua @@ -28,7 +28,7 @@ local config_schema = { apisix = { properties = { config_center = { - enum = {"etcd", "yaml", "shdict"}, + enum = {"etcd", "yaml", "xds"}, }, lua_module_hook = { pattern = "^[a-zA-Z._-]+$", diff --git a/apisix/core/config_shdict.lua b/apisix/core/config_xds.lua similarity index 74% rename from apisix/core/config_shdict.lua rename to apisix/core/config_xds.lua index 6bec3c4e8025..dd71feed6402 100644 --- a/apisix/core/config_shdict.lua +++ b/apisix/core/config_xds.lua @@ -17,7 +17,7 @@ --- Get configuration form ngx.shared.DICT. -- --- @module core.config_shdict +-- @module core.config_xds local base = require("resty.core.base") local config_local = require("apisix.core.config_local") @@ -40,7 +40,7 @@ end ffi.cdef[[ -extern void initial(void* writeRoute); +extern void initial(void* router_zone_ptr); ]] @@ -79,24 +79,17 @@ end local function load_libamesh(lib_name) - ngx_timer_at(0, function(premature) - if premature then - return - end - - local ameshagent, tried_paths = load_shared_lib(lib_name) + local ameshagent, tried_paths = load_shared_lib(lib_name) - if not ameshagent then - tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' - .. lib_name - error("can not load Amesh library, tried paths: " .. - table.concat(tried_paths, '\r\n', 1, #tried_paths)) - end + if not ameshagent then + tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' .. lib_name + error("can not load Amesh library, tried paths: " .. + table.concat(tried_paths, '\r\n', 1, #tried_paths)) + end - local router_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(router_config[1]) - local router_shd_cdata = ffi.cast("void*", router_zone) - ameshagent.initial(router_shd_cdata) - end) + local router_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(router_config[1]) + local router_shd_cdata = ffi.cast("void*", router_zone) + ameshagent.initial(router_shd_cdata) end @@ -112,10 +105,4 @@ function _M.init_worker() end -function _M.new(key, opts) - -- mock for test - return { true } -end - - return _M diff --git a/conf/config.yaml b/conf/config.yaml index 421ac0912aa6..76bb16279dd7 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -41,6 +41,7 @@ # This will find environment variable `ETCD_HOST` first, and if it's not exist it will use `localhost` as default value. # apisix: + config_center: shdict admin_key: - name: admin key: edd1c9f034335f136f87ad84b625c8f1 # using fixed API token has security risk, please update it when you deploy to production environment diff --git a/t/amesh-library/config_shdict.t b/t/amesh-library/config_xds.t similarity index 85% rename from t/amesh-library/config_shdict.t rename to t/amesh-library/config_xds.t index 5e55836cacc3..da02205b7ecb 100644 --- a/t/amesh-library/config_shdict.t +++ b/t/amesh-library/config_xds.t @@ -41,6 +41,24 @@ add_block_preprocessor(sub { _EOC_ $block->set_value("lua_deps_path", $lua_deps_path); + + my $extra_init_by_lua = <<_EOC_; + -- + local config_xds = require("apisix.core.config_xds") + + local inject = function(mod, name) + local old_f = mod[name] + mod[name] = function (...) + ngx.log(ngx.WARN, "config_xds run ", name) + return { true } + end + end + + inject(config_xds, "new") + +_EOC_ + + $block->set_value("extra_init_by_lua", $extra_init_by_lua); }); run_tests; @@ -51,7 +69,7 @@ __DATA__ --- yaml_config apisix: node_listen: 1984 - config_center: shdict + config_center: xds enable_admin: false --- config location /t { @@ -68,7 +86,7 @@ qr/can not load Amesh library/ --- yaml_config apisix: node_listen: 1984 - config_center: shdict + config_center: xds enable_admin: false --- config location /t { From 82fbcdcc08a5d47d0055eb3d36253012856242e9 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 14:27:07 +0800 Subject: [PATCH 17/28] revert config.yaml --- conf/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/config.yaml b/conf/config.yaml index 76bb16279dd7..421ac0912aa6 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -41,7 +41,6 @@ # This will find environment variable `ETCD_HOST` first, and if it's not exist it will use `localhost` as default value. # apisix: - config_center: shdict admin_key: - name: admin key: edd1c9f034335f136f87ad84b625c8f1 # using fixed API token has security risk, please update it when you deploy to production environment From b0fa53425d490317453f9d80fd7a06d8ffcdf200 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 17:33:59 +0800 Subject: [PATCH 18/28] resolve code review Signed-off-by: tzssangglass --- .github/workflows/build.yml | 6 +++--- apisix/core/config_xds.lua | 15 ++++++--------- apisix/init.lua | 2 +- t/amesh-library/config_xds.t | 2 +- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1cc24a189d0c..489f36aad933 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: test_dir: - t/plugin - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/discovery t/error_page t/misc - - t/amesh-library t/node t/router t/script t/stream-node t/utils t/wasm + - t/xds-library t/node t/router t/script t/stream-node t/utils t/wasm runs-on: ${{ matrix.platform }} timeout-minutes: 90 @@ -90,9 +90,9 @@ jobs: sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p - - name: Build Amesh library + - name: Build xDS library run: | - cd t/amesh-library + cd t/xds-library go build -o libamesh.so -buildmode=c-shared main.go - name: Linux Before install diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index dd71feed6402..40f656b74c2e 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -24,14 +24,16 @@ local config_local = require("apisix.core.config_local") local table = table local error = error local is_http = ngx.config.subsystem == "http" -local string = string local io = io +local io_open = io.open +local io_close = io.close local package = package local new_tab = base.new_tab -local ngx_timer_at = ngx.timer.at local ffi = require ("ffi") local C = ffi.C local router_config = ngx.shared["router-config"] +local ngx_re_match = ngx.re.match +local ngx_re_gmatch = ngx.re.gmatch local process if is_http then @@ -52,17 +54,12 @@ local _M = { -- todo: refactor this function in chash.lua and radixtree.lua local function load_shared_lib(lib_name) - local string_gmatch = string.gmatch - local string_match = string.match - local io_open = io.open - local io_close = io.close - local cpath = package.cpath local tried_paths = new_tab(32, 0) local i = 1 - for k, _ in string_gmatch(cpath, "[^;]+") do - local fpath = string_match(k, "(.*/)") + for k, _ in ngx_re_gmatch(cpath, "[^;]+") do + local fpath = ngx_re_match(k, "(.*/)") fpath = fpath .. lib_name local f = io_open(fpath) diff --git a/apisix/init.lua b/apisix/init.lua index 17bfe38ed80b..6e059b839892 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -111,7 +111,7 @@ function _M.http_init_worker() require("apisix.balancer").init_worker() load_balancer = require("apisix.balancer") - if core.config == require("apisix.core.config_shdict") then + if core.config == require("apisix.core.config_xds") then core.config.init_worker() end diff --git a/t/amesh-library/config_xds.t b/t/amesh-library/config_xds.t index da02205b7ecb..5721514d9f3e 100644 --- a/t/amesh-library/config_xds.t +++ b/t/amesh-library/config_xds.t @@ -37,7 +37,7 @@ add_block_preprocessor(sub { my $lua_deps_path = $block->lua_deps_path // <<_EOC_; lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; - lua_package_cpath "$apisix_home/?.so;$apisix_home/t/amesh-library/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; + lua_package_cpath "$apisix_home/?.so;$apisix_home/t/xds-library/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; _EOC_ $block->set_value("lua_deps_path", $lua_deps_path); From 39cca64da109b99bf150d96598e88d91ca785644 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 17:35:51 +0800 Subject: [PATCH 19/28] rename Signed-off-by: tzssangglass --- .github/workflows/build.yml | 2 +- apisix/core/config_xds.lua | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 489f36aad933..f2ec9e5a2572 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,7 +93,7 @@ jobs: - name: Build xDS library run: | cd t/xds-library - go build -o libamesh.so -buildmode=c-shared main.go + go build -o libxds.so -buildmode=c-shared main.go - name: Linux Before install run: sudo ./ci/${{ matrix.os_name }}_runner.sh before_install diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index 40f656b74c2e..0e4c0bc66084 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -75,10 +75,10 @@ local function load_shared_lib(lib_name) end -local function load_libamesh(lib_name) - local ameshagent, tried_paths = load_shared_lib(lib_name) +local function load_libxds(lib_name) + local xdsagent, tried_paths = load_shared_lib(lib_name) - if not ameshagent then + if not xdsagent then tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' .. lib_name error("can not load Amesh library, tried paths: " .. table.concat(tried_paths, '\r\n', 1, #tried_paths)) @@ -86,16 +86,16 @@ local function load_libamesh(lib_name) local router_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(router_config[1]) local router_shd_cdata = ffi.cast("void*", router_zone) - ameshagent.initial(router_shd_cdata) + xdsagent.initial(router_shd_cdata) end function _M.init_worker() - local lib_name = "libamesh.so" + local lib_name = "libxds.so" if process.type() == "privileged agent" then - load_libamesh(lib_name) + load_libxds(lib_name) end return true From add82f7187f7593108db0db553665114fdd798e4 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 17:38:33 +0800 Subject: [PATCH 20/28] rename Signed-off-by: tzssangglass --- t/{amesh-library => xds-library}/config_xds.t | 0 t/{amesh-library => xds-library}/main.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename t/{amesh-library => xds-library}/config_xds.t (100%) rename t/{amesh-library => xds-library}/main.go (100%) diff --git a/t/amesh-library/config_xds.t b/t/xds-library/config_xds.t similarity index 100% rename from t/amesh-library/config_xds.t rename to t/xds-library/config_xds.t diff --git a/t/amesh-library/main.go b/t/xds-library/main.go similarity index 100% rename from t/amesh-library/main.go rename to t/xds-library/main.go From 74653952dab080d0bf6095861723970d2e23ea10 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 18:27:15 +0800 Subject: [PATCH 21/28] resolve code review --- .github/workflows/build.yml | 2 +- apisix/core/config_xds.lua | 28 ++++++++++++++++++++-------- t/xds-library/config_xds.t | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f2ec9e5a2572..5349651ee303 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: test_dir: - t/plugin - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/discovery t/error_page t/misc - - t/xds-library t/node t/router t/script t/stream-node t/utils t/wasm + - t/node t/router t/script t/stream-node t/utils t/wasm t/xds-library runs-on: ${{ matrix.platform }} timeout-minutes: 90 diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index 0e4c0bc66084..14b4fe4155aa 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -56,19 +56,31 @@ local _M = { local function load_shared_lib(lib_name) local cpath = package.cpath local tried_paths = new_tab(32, 0) - local i = 1 - for k, _ in ngx_re_gmatch(cpath, "[^;]+") do - local fpath = ngx_re_match(k, "(.*/)") - fpath = fpath .. lib_name + local iter, err = ngx_re_gmatch(cpath, "[^;]+", "jo") + if not iter then + error("failed to gmatch: " .. err) + end + + while true do + local it = iter() + local fpath + fpath, err = ngx_re_match(it[0], "(.*/)", "jo") + if err then + error("failed to match: " .. err) + end + local spath = fpath[0] .. lib_name - local f = io_open(fpath) + local f = io_open(spath) if f ~= nil then io_close(f) - return ffi.load(fpath) + return ffi.load(spath) + end + tried_paths[i] = spath + + if not it then + break end - tried_paths[i] = fpath - i = i + 1 end return nil, tried_paths diff --git a/t/xds-library/config_xds.t b/t/xds-library/config_xds.t index 5721514d9f3e..b2963254153d 100644 --- a/t/xds-library/config_xds.t +++ b/t/xds-library/config_xds.t @@ -78,7 +78,7 @@ apisix: } } --- no_error_log eval -qr/can not load Amesh library/ +qr/can not load xDS library/ From 3123324cf0a1378ae19c812e0fd2bfc1325f0c13 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 17 Mar 2022 21:07:59 +0800 Subject: [PATCH 22/28] fix CI --- apisix/core/config_xds.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index 14b4fe4155aa..d24777a5779f 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -56,6 +56,7 @@ local _M = { local function load_shared_lib(lib_name) local cpath = package.cpath local tried_paths = new_tab(32, 0) + local i = 1 local iter, err = ngx_re_gmatch(cpath, "[^;]+", "jo") if not iter then @@ -77,6 +78,7 @@ local function load_shared_lib(lib_name) return ffi.load(spath) end tried_paths[i] = spath + i = i + 1 if not it then break From 8a2a67d3bff2b2903cf9566753a984bdc503e25d Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Fri, 18 Mar 2022 00:27:27 +0800 Subject: [PATCH 23/28] resolve code review --- apisix/core/config_xds.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index d24777a5779f..28e4995212d7 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -35,6 +35,9 @@ local router_config = ngx.shared["router-config"] local ngx_re_match = ngx.re.match local ngx_re_gmatch = ngx.re.gmatch +local xds_lib_name = "libxds.so" + + local process if is_http then process = require("ngx.process") @@ -106,10 +109,8 @@ end function _M.init_worker() - local lib_name = "libxds.so" - if process.type() == "privileged agent" then - load_libxds(lib_name) + load_libxds(xds_lib_name) end return true From f1fee3bfc7e0a6185a68de84c86fe33308e83404 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Fri, 18 Mar 2022 12:23:55 +0800 Subject: [PATCH 24/28] resolve code review --- apisix/init.lua | 13 ++++++------- t/xds-library/config_xds.t | 21 +++++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/apisix/init.lua b/apisix/init.lua index 6e059b839892..d89364d000bb 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -110,11 +110,6 @@ function _M.http_init_worker() end require("apisix.balancer").init_worker() load_balancer = require("apisix.balancer") - - if core.config == require("apisix.core.config_xds") then - core.config.init_worker() - end - require("apisix.admin.init").init_worker() require("apisix.timers").init_worker() @@ -127,8 +122,12 @@ function _M.http_init_worker() plugin_config.init_worker() require("apisix.consumer").init_worker() - if core.config == require("apisix.core.config_yaml") then - core.config.init_worker() + if core.config.init_worker then + local ok, err = core.config.init_worker() + if not ok then + core.log.error("failed to init worker process of ", core.config.type, + " config center, err: ", err) + end end apisix_upstream.init_worker() diff --git a/t/xds-library/config_xds.t b/t/xds-library/config_xds.t index b2963254153d..ce68b5e04398 100644 --- a/t/xds-library/config_xds.t +++ b/t/xds-library/config_xds.t @@ -59,6 +59,17 @@ _EOC_ _EOC_ $block->set_value("extra_init_by_lua", $extra_init_by_lua); + + if (!$block->yaml_config) { + my $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + config_center: xds + enable_admin: false +_EOC_ + + $block->set_value("yaml_config", $yaml_config); + } }); run_tests; @@ -66,11 +77,6 @@ run_tests; __DATA__ === TEST 1: load Amesh library so successfully ---- yaml_config -apisix: - node_listen: 1984 - config_center: xds - enable_admin: false --- config location /t { content_by_lua_block { @@ -83,11 +89,6 @@ qr/can not load xDS library/ === TEST 2: read data form shdict that wirted by Amesh library ---- yaml_config -apisix: - node_listen: 1984 - config_center: xds - enable_admin: false --- config location /t { content_by_lua_block { From a9c9ba01aabf1aa64a0110e7c70b17288022083e Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Fri, 18 Mar 2022 15:51:00 +0800 Subject: [PATCH 25/28] fix CI error --- apisix/core/config_yaml.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apisix/core/config_yaml.lua b/apisix/core/config_yaml.lua index 89753f0f1e0e..24a5ff57aa6f 100644 --- a/apisix/core/config_yaml.lua +++ b/apisix/core/config_yaml.lua @@ -389,6 +389,8 @@ end function _M.init_worker() -- sync data in each non-master process ngx.timer.every(1, read_apisix_yaml) + + return true end From 9b31b3457d2dbb4b43ce2677922542b63c8652ac Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Fri, 18 Mar 2022 22:22:04 +0800 Subject: [PATCH 26/28] rename test cases names --- t/xds-library/config_xds.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/xds-library/config_xds.t b/t/xds-library/config_xds.t index ce68b5e04398..a50aceb4d1e4 100644 --- a/t/xds-library/config_xds.t +++ b/t/xds-library/config_xds.t @@ -76,7 +76,7 @@ run_tests; __DATA__ -=== TEST 1: load Amesh library so successfully +=== TEST 1: load xDS library successfully --- config location /t { content_by_lua_block { @@ -88,7 +88,7 @@ qr/can not load xDS library/ -=== TEST 2: read data form shdict that wirted by Amesh library +=== TEST 2: read data form shdict that wirted by xDS library --- config location /t { content_by_lua_block { From 513c2b3c824c343d9438eded344c9c424f938955 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 21 Mar 2022 09:42:57 +0800 Subject: [PATCH 27/28] resolve code review --- apisix/cli/ngx_tpl.lua | 2 +- apisix/core/config_xds.lua | 4 ++-- t/APISIX.pm | 2 +- t/xds-library/config_xds.t | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index a924b6f61dc4..49e1baa478ed 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -240,7 +240,7 @@ http { {% end %} {% if config_center == "xds" then %} - lua_shared_dict router-config 10m; + lua_shared_dict xds-route-config 10m; {% end %} # for custom shared dict diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index 28e4995212d7..a2f3aa4371bc 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -31,7 +31,7 @@ local package = package local new_tab = base.new_tab local ffi = require ("ffi") local C = ffi.C -local router_config = ngx.shared["router-config"] +local router_config = ngx.shared["xds-route-config"] local ngx_re_match = ngx.re.match local ngx_re_gmatch = ngx.re.gmatch @@ -97,7 +97,7 @@ local function load_libxds(lib_name) if not xdsagent then tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' .. lib_name - error("can not load Amesh library, tried paths: " .. + error("can not load xds library, tried paths: " .. table.concat(tried_paths, '\r\n', 1, #tried_paths)) end diff --git a/t/APISIX.pm b/t/APISIX.pm index a3500e1deba1..7c23804cdbda 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -508,7 +508,7 @@ _EOC_ lua_shared_dict etcd-cluster-health-check 10m; # etcd health check lua_shared_dict ext-plugin 1m; lua_shared_dict kubernetes 1m; - lua_shared_dict router-config 1m; + lua_shared_dict xds-route-config 1m; proxy_ssl_name \$upstream_host; proxy_ssl_server_name on; diff --git a/t/xds-library/config_xds.t b/t/xds-library/config_xds.t index a50aceb4d1e4..dabdbc141ab7 100644 --- a/t/xds-library/config_xds.t +++ b/t/xds-library/config_xds.t @@ -92,10 +92,10 @@ qr/can not load xDS library/ --- config location /t { content_by_lua_block { - -- wait for Amesh library sync data + -- wait for xds library sync data ngx.sleep(1.5) local core = require("apisix.core") - local value = ngx.shared["router-config"]:get("/apisix/routes/1") + local value = ngx.shared["xds-route-config"]:get("/apisix/routes/1") local route_conf, err = core.json.decode(value) local json_encode = require("toolkit.json").encode ngx.say(json_encode(route_conf)) From 57c88d1dd240afbb1ce366510e5d6097bd99ace5 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 21 Mar 2022 17:13:17 +0800 Subject: [PATCH 28/28] resolve code review --- apisix/core/config_xds.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apisix/core/config_xds.lua b/apisix/core/config_xds.lua index a2f3aa4371bc..7c0c9f4a6950 100644 --- a/apisix/core/config_xds.lua +++ b/apisix/core/config_xds.lua @@ -31,7 +31,7 @@ local package = package local new_tab = base.new_tab local ffi = require ("ffi") local C = ffi.C -local router_config = ngx.shared["xds-route-config"] +local route_config = ngx.shared["xds-route-config"] local ngx_re_match = ngx.re.match local ngx_re_gmatch = ngx.re.gmatch @@ -45,7 +45,7 @@ end ffi.cdef[[ -extern void initial(void* router_zone_ptr); +extern void initial(void* route_zone_ptr); ]] @@ -101,9 +101,9 @@ local function load_libxds(lib_name) table.concat(tried_paths, '\r\n', 1, #tried_paths)) end - local router_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(router_config[1]) - local router_shd_cdata = ffi.cast("void*", router_zone) - xdsagent.initial(router_shd_cdata) + local route_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(route_config[1]) + local route_shd_cdata = ffi.cast("void*", route_zone) + xdsagent.initial(route_shd_cdata) end