feat: allow customizing lua_package_path & lua_package_cpath

.travis/apisix_cli_test/test_main.sh

@@ -23,6 +23,20 @@
 
 . ./.travis/apisix_cli_test/common.sh
 
+# validate extra_lua_path
+echo '
+apisix:
+    extra_lua_path: ";"
+' > conf/config.yaml
+
+out=$(make init 2>&1 || true)
+if ! echo "$out" | grep 'invalid extra_lua_path'; then
+    echo "failed: can't detect invalid extra_lua_path"
+    exit 1
+fi
+
+echo "passed: detect invalid extra_lua_path"
+
 git checkout conf/config.yaml
 
 # check 'Server: APISIX' is not in nginx.conf. We already added it in Lua code.
@@ -1013,3 +1027,27 @@ echo "passed: show password error successfully"
 etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 auth disable
 etcdctl --endpoints=127.0.0.1:2379 role delete root
 etcdctl --endpoints=127.0.0.1:2379 user delete root
+
+# support 3rd-party plugin
+echo '
+apisix:
+    extra_lua_path: "\$prefix/example/?.lua"
+    extra_lua_cpath: "\$prefix/example/?.lua"
+plugins:
+    - 3rd-party
+stream_plugins:
+    - 3rd-party
+' > conf/config.yaml
+
+rm logs/error.log
+make init
+make run
+
+sleep 0.5
+make stop
+
+if grep "failed to load plugin [3rd-party]" logs/error.log > /dev/null; then
+    echo "failed: 3rd-party plugin can not be loaded"
+    exit 1
+fi
+echo "passed: 3rd-party plugin can be loaded"

apisix/cli/ngx_tpl.lua

@@ -56,9 +56,9 @@ env {*name*};
 
 {% if stream_proxy then %}
 stream {
-    lua_package_path  "$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;]=]
+    lua_package_path  "{*extra_lua_path*}$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;]=]
                       .. [=[{*apisix_lua_home*}/?.lua;{*apisix_lua_home*}/?/init.lua;;{*lua_path*};";
-    lua_package_cpath "$prefix/deps/lib64/lua/5.1/?.so;]=]
+    lua_package_cpath "{*extra_lua_cpath*}$prefix/deps/lib64/lua/5.1/?.so;]=]
                       .. [=[$prefix/deps/lib/lua/5.1/?.so;;]=]
                       .. [=[{*lua_cpath*};";
     lua_socket_log_errors off;
@@ -117,9 +117,11 @@ stream {
 {% end %}
 
 http {
-    lua_package_path  "$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;]=]
+    # put extra_lua_path in front of the builtin path
+    # so user can override the source code
+    lua_package_path  "{*extra_lua_path*}$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;]=]
                        .. [=[{*apisix_lua_home*}/?.lua;{*apisix_lua_home*}/?/init.lua;;{*lua_path*};";
-    lua_package_cpath "$prefix/deps/lib64/lua/5.1/?.so;]=]
+    lua_package_cpath "{*extra_lua_cpath*}$prefix/deps/lib64/lua/5.1/?.so;]=]
                       .. [=[$prefix/deps/lib/lua/5.1/?.so;;]=]
                       .. [=[{*lua_cpath*};";
 

apisix/cli/ops.lua

@@ -36,6 +36,7 @@ local getenv = os.getenv
 local max = math.max
 local floor = math.floor
 local str_find = string.find
+local str_byte = string.byte
 local str_sub = string.sub
 
 
@@ -121,6 +122,25 @@ local function version()
 end
 
 
+local function get_lua_path(conf)
+    -- we use "" as the placeholder to enforce the type to be string
+    if conf and conf ~= "" then
+        if #conf < 2 then
+            -- the shortest valid path is ';;'
+            util.die("invalid extra_lua_path/extra_lua_cpath: \"", conf, "\"\n")
+        end
+
+        local path = conf
+        if path:byte(-1) ~= str_byte(';') then
+            path = path .. ';'
+        end
+        return path
+    end
+
+    return ""
+end
+
+
 local function init(env)
     if env.is_root_path then
         print('Warning! Running apisix under /root is only suitable for '
@@ -357,6 +377,10 @@ Please modify "admin_key" in conf/config.yaml .
         end
     end
 
+    -- fix up lua path
+    sys_conf["extra_lua_path"] = get_lua_path(yaml_conf.apisix.extra_lua_path)
+    sys_conf["extra_lua_cpath"] = get_lua_path(yaml_conf.apisix.extra_lua_cpath)
+
     local conf_render = template.compile(ngx_tpl)
     local ngxconf = conf_render(sys_conf)
 

conf/config-default.yaml

@@ -43,6 +43,10 @@ apisix:
   enable_server_tokens: true       # Whether the APISIX version number should be shown in Server header.
                                    # It's enabled by default.
 
+  # configurations to load third party code and/or override the builtin one.
+  extra_lua_path: ""               # extend lua_package_path to load third party code
+  extra_lua_cpath: ""              # extend lua_package_cpath to load third party code
+
   proxy_cache:                     # Proxy Caching configuration
     cache_ttl: 10s                 # The default caching time if the upstream does not specify the cache time
     zones:                         # The parameters of a cache

doc/plugin-develop.md

@@ -20,6 +20,7 @@
 
 # table of contents
 
+- [**where to put your plugin**](#where-to-put-your-plugin)
 - [**check dependencies**](#check-dependencies)
 - [**name and config**](#name-and-config)
 - [**schema and check**](#schema-and-check)
@@ -29,10 +30,39 @@
 - [**register public API**](#register-public-api)
 - [**register control API**](#register-control-api)
 
+## where to put your plugins
+
+There are two ways to add new features based on APISIX.
+
+1. modify the source of APISIX and redistribute it (not so recommended)
+1. setup the `extra_lua_path` and `extra_lua_cpath` in `conf/config.yaml` to load your own code. Your own code will be loaded instead of the builtin one with the same name, so you can use this way to override the builtin behavior if needed.
+
+For example, you can create a directory structure like this:
+
+```
+├── example
+│   └── apisix
+│       ├── plugins
+│       │   └── 3rd-party.lua
+│       └── stream
+│           └── plugins
+│               └── 3rd-party.lua
+```
+
+Then add this configuration into your `conf/config.yaml`:
+
+```yaml
+apisix:
+    ...
+    extra_lua_path: "/path/to/example/?.lua"
+```
+
+Now using `require "apisix.plugins.3rd-party"` will load your plugin, just like `require "apisix.plugins.jwt-auth"` will load the `jwt-auth` plugin.
+
 ## check dependencies
 
 if you have dependencies on external libraries, check the dependent items. if your plugin needs to use shared memory, it
- needs to declare in __bin/apisix__, for example :
+ needs to declare in **apisix/cli/ngx_tpl.lua**, for example :
 
 ```nginx
     lua_shared_dict plugin-limit-req     10m;
@@ -100,7 +130,23 @@ plugins:                          # plugin list
 
 Note : the order of the plugins is not related to the order of execution.
 
-If your plugin has a new code directory of its own, you will need to modify the `Makefile` to create directory, such as:
+To enable your plugin, copy this plugin list into `conf/config.yaml`, and add your plugin name. For instance:
+
+```yaml
+apisix:
+  admin_key:
+    - name: "admin"
+      # yamllint disable rule:comments-indentation
+      key: edd1c9f034335f136f87ad84b625c8f1 # using fixed API token has security risk, please update it when you deploy to production environment
+      # yamllint enable rule:comments-indentation
+      role: admin
+
+plugins: # copied from config-default.yaml
+  ...
+  - your-plugin
+```
+
+If your plugin has a new code directory of its own, and you need to redistribute it with the APISIX source code, you will need to modify the `Makefile` to create directory, such as:
 
 ```
 $(INSTALL) -d $(INST_LUADIR)/apisix/plugins/skywalking

example/apisix/plugins/3rd-party.lua

@@ -0,0 +1,51 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+
+
+local schema = {
+    type = "object",
+    properties = {
+        body = {
+            description = "body to replace response.",
+            type = "string"
+        },
+    },
+    required = {"body"},
+}
+
+local plugin_name = "3rd-party"
+
+local _M = {
+    version = 0.1,
+    priority = 12,
+    name = plugin_name,
+    schema = schema,
+}
+
+
+function _M.check_schema(conf)
+    return core.schema.check(schema, conf)
+end
+
+
+function _M.access(conf, ctx)
+    return 200, conf.body
+end
+
+
+return _M

example/apisix/stream/plugins/3rd-party.lua

@@ -0,0 +1,51 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+
+
+local schema = {
+    type = "object",
+    properties = {
+        body = {
+            description = "body to replace response.",
+            type = "string"
+        },
+    },
+    required = {"body"},
+}
+
+local plugin_name = "3rd-party"
+
+local _M = {
+    version = 0.1,
+    priority = 12,
+    name = plugin_name,
+    schema = schema,
+}
+
+
+function _M.check_schema(conf)
+    return core.schema.check(schema, conf)
+end
+
+
+function _M.access(conf, ctx)
+    return 200, conf.body
+end
+
+
+return _M