From b2b5b18418df2754bae0c825c5b465aea0bbeeb1 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 8 Sep 2016 12:03:07 +0200 Subject: [PATCH 01/12] docs(gh-templ) include startup logs info --- .github/ISSUE_TEMPLATE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index ff5ea79689c..7a6beab08e6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -12,6 +12,7 @@ SUMMARY_GOES_HERE ### Additional Details & Logs - Kong version (`$ kong version`) -- Kong configuration (registered APIs/Plugins & configuration file) +- Kong debug-level startup logs (`$ kong start --vv`) - Kong error logs (`/logs/error.log`) +- Kong configuration (registered APIs/Plugins & configuration file) - Operating System From 23c4f5600a38c7ff8f3053d11dcf2766cdd92d2f Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Mon, 12 Sep 2016 21:34:23 -0700 Subject: [PATCH 02/12] fix(cli) prevent xpcall handler to crash serf_signals.stop() would eventually crash and prevent execution of the following code, thus we would not report the actual crashing reason to the user (silent kong start failure) --- kong/cmd/start.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kong/cmd/start.lua b/kong/cmd/start.lua index d3fa6241d44..b5bb1880470 100644 --- a/kong/cmd/start.lua +++ b/kong/cmd/start.lua @@ -14,8 +14,8 @@ local function execute(args) local dao = DAOFactory(conf) local err xpcall(function() - assert(dao:run_migrations()) assert(prefix_handler.prepare_prefix(conf, args.nginx_conf)) + assert(dao:run_migrations()) if conf.dnsmasq then assert(dnsmasq_signals.start(conf)) end @@ -24,10 +24,10 @@ local function execute(args) log("Kong started") end, function(e) log.verbose("could not start Kong, stopping services") - nginx_signals.stop(conf) - serf_signals.stop(conf, dao) + pcall(nginx_signals.stop(conf)) + pcall(serf_signals.stop(conf, dao)) if conf.dnsmasq then - dnsmasq_signals.stop(conf) + pcall(dnsmasq_signals.stop(conf)) end err = e -- cannot throw from this function log.verbose("stopped services") From 46f938090b5729a7deb965452ed32a2488eb876a Mon Sep 17 00:00:00 2001 From: Morton Fox Date: Tue, 13 Sep 2016 00:31:58 -0400 Subject: [PATCH 03/12] docs(readme) fix the second luacheck link Parentheses must be escaped otherwise Github's Markdown implementation will treat that part as the URL. Signed-off-by: Thibault Charbonnier --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a92785e9f6..ee436edf0db 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ usage of the tests suites and the Makefile. Finally, a very useful tool in Lua development (as with many other dynamic languages) is performing static linting of your code. You can use [luacheck] -(installed with `make dev`) for this: +\(installed with `make dev`\) for this: ``` $ make lint From 558ec5065c81acbc1549d7963346ddf85fa6ce1d Mon Sep 17 00:00:00 2001 From: Morton Fox Date: Tue, 13 Sep 2016 00:31:58 -0400 Subject: [PATCH 04/12] docs(readme) fix the second luacheck link Parentheses must be escaped otherwise Github's Markdown implementation will treat that part as the URL. Signed-off-by: Thibault Charbonnier --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a92785e9f6..ee436edf0db 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ usage of the tests suites and the Makefile. Finally, a very useful tool in Lua development (as with many other dynamic languages) is performing static linting of your code. You can use [luacheck] -(installed with `make dev`) for this: +\(installed with `make dev`\) for this: ``` $ make lint From cc1c446963bb6cee5d25bdf74d0aa24b81513d7e Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 13 Sep 2016 14:37:01 +0200 Subject: [PATCH 05/12] added some debug stuff, failing all over... using postgres it won't start, complaining no options table was provided, yet the debug line shows it does. using cassandra the tcp override debug lines don't show --> cassandra bypasses those overrrides. How?? --- bin/busted | 10 ++-------- bin/kong | 8 +------- kong/core/globalpatches.lua | 6 +++--- kong/dao/postgres_db.lua | 1 + 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/bin/busted b/bin/busted index ecc2bbe633a..c457dcb6b0c 100755 --- a/bin/busted +++ b/bin/busted @@ -1,15 +1,9 @@ #!/usr/bin/env resty --- force LuaSocket usage to resolve `/etc/hosts` until --- supported by resty-cli. --- See https://github.com/Mashape/kong/issues/1523 -for _, namespace in ipairs({"cassandra", "pgmoon-mashape"}) do - local socket = require(namespace .. ".socket") - socket.force_luasocket(ngx.get_phase(), true) -end - package.path = "?/init.lua;"..package.path +require "kong.core.globalpatches" + if ngx ~= nil then ngx.exit = function()end end diff --git a/bin/kong b/bin/kong index 39f8628cdc8..375ad1cedea 100755 --- a/bin/kong +++ b/bin/kong @@ -1,11 +1,5 @@ #!/usr/bin/env resty --- force LuaSocket usage to resolve `/etc/hosts` until --- supported by resty-cli. --- See https://github.com/Mashape/kong/issues/1523 -for _, namespace in ipairs({"cassandra", "pgmoon-mashape"}) do - local socket = require(namespace .. ".socket") - socket.force_luasocket(ngx.get_phase(), true) -end +require "kong.core.globalpatches" require("kong.cmd.init")(arg) diff --git a/kong/core/globalpatches.lua b/kong/core/globalpatches.lua index 845d8c3a03f..a074b7f809c 100644 --- a/kong/core/globalpatches.lua +++ b/kong/core/globalpatches.lua @@ -17,9 +17,7 @@ local seed -- unique seed for Nginx workers). -- luacheck: globals math _G.math.randomseed = function() - if ngx.get_phase() ~= "init_worker" then - ngx.log(ngx.ERR, "math.randomseed() must be called in init_worker") - elseif not seed then + if not seed then seed = ngx.time() + ngx.worker.pid() ngx.log(ngx.DEBUG, "random seed: ", seed, " for worker n", ngx.worker.id(), " (pid: ", ngx.worker.pid(), ")") @@ -42,9 +40,11 @@ local toip local old_tcp = ngx.socket.tcp -- STEP 4: patch globals _G.ngx.socket.tcp = function(...) +print("=======================> Here we go! <=======================") local sock = old_tcp(...) local old_connect = sock.connect sock.connect = function(s, host, port, sock_opts) +print("=======================> connecting <========================") local target_ip, target_port = toip(host, port) if not target_ip then diff --git a/kong/dao/postgres_db.lua b/kong/dao/postgres_db.lua index ec4642d1252..0b543056f57 100644 --- a/kong/dao/postgres_db.lua +++ b/kong/dao/postgres_db.lua @@ -138,6 +138,7 @@ function PostgresDB:query(query, schema) PostgresDB.super.query(self, query) local conn_opts = self:_get_conn_options() +print(require("pl.pretty").write(conn_opts)) local pg = pgmoon.new(conn_opts) local ok, err = pg:connect() if not ok then From 3fc057f938442279dfd6072d2d33e5b3ae6144dd Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 13 Sep 2016 16:44:04 +0200 Subject: [PATCH 06/12] wtf --- bin/busted | 4 +--- bin/kong | 2 ++ kong-0.9.1-0.rockspec | 4 ++-- kong/conf_loader.lua | 4 ++++ kong/core/globalpatches.lua | 6 +++--- kong/dao/postgres_db.lua | 3 +-- kong/tools/dns.lua | 5 ++--- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bin/busted b/bin/busted index c457dcb6b0c..c809baa4e34 100755 --- a/bin/busted +++ b/bin/busted @@ -2,10 +2,8 @@ package.path = "?/init.lua;"..package.path -require "kong.core.globalpatches" - if ngx ~= nil then - ngx.exit = function()end + ngx.exit = function() end end -- Busted command-line runner diff --git a/bin/kong b/bin/kong index 375ad1cedea..5430db7de28 100755 --- a/bin/kong +++ b/bin/kong @@ -1,5 +1,7 @@ #!/usr/bin/env resty +-- require global patches, to enable dns resolution by the internal client +-- initializing the dns client is done in `./kong/conf_loader.lua` require "kong.core.globalpatches" require("kong.cmd.init")(arg) diff --git a/kong-0.9.1-0.rockspec b/kong-0.9.1-0.rockspec index 5afc045545e..2104f9042e4 100644 --- a/kong-0.9.1-0.rockspec +++ b/kong-0.9.1-0.rockspec @@ -21,14 +21,14 @@ dependencies = { "version == 0.2", "lapis == 1.5.1", "lua-cassandra == 0.5.3", - "pgmoon-mashape == 2.0.0", + "pgmoon == 1.6.0", "luatz == 0.3", "lua_system_constants == 0.1.1", "lua-resty-iputils == 0.2.1", "luacrypto == 0.3.2", "luasyslog == 1.0.0", "lua_pack == 1.0.4", - "dns == 0.1", + "dns == 0.1.0", } build = { type = "builtin", diff --git a/kong/conf_loader.lua b/kong/conf_loader.lua index c84f0ee6d8e..30545f5f9f1 100644 --- a/kong/conf_loader.lua +++ b/kong/conf_loader.lua @@ -372,6 +372,10 @@ local function load(path, custom_conf) log.verbose("prefix in use: %s", conf.prefix) + -- initialize the dns client, so the globally patched tcp.connect method + -- will work from here onwards. + assert(require("kong.tools.dns")(conf)) + return setmetatable(conf, nil) -- remove Map mt end diff --git a/kong/core/globalpatches.lua b/kong/core/globalpatches.lua index a074b7f809c..1ada58aacf5 100644 --- a/kong/core/globalpatches.lua +++ b/kong/core/globalpatches.lua @@ -39,16 +39,16 @@ local toip -- STEP 3: store original unpatched versions local old_tcp = ngx.socket.tcp -- STEP 4: patch globals +print(" ======================> Patching global socket.tcp...") _G.ngx.socket.tcp = function(...) -print("=======================> Here we go! <=======================") local sock = old_tcp(...) local old_connect = sock.connect sock.connect = function(s, host, port, sock_opts) -print("=======================> connecting <========================") +print(" ======================> Patching a tcp socket here...") local target_ip, target_port = toip(host, port) if not target_ip then - return nil, target_port + return nil, "[toip() name lookup failed]:"..tostring(target_port) else -- need to do the extra check here: https://github.com/openresty/lua-nginx-module/issues/860 if not sock_opts then diff --git a/kong/dao/postgres_db.lua b/kong/dao/postgres_db.lua index 0b543056f57..4ea4777e02c 100644 --- a/kong/dao/postgres_db.lua +++ b/kong/dao/postgres_db.lua @@ -1,4 +1,4 @@ -local pgmoon = require "pgmoon-mashape" +local pgmoon = require "pgmoon" local BaseDB = require "kong.dao.base_db" local Errors = require "kong.dao.errors" local utils = require "kong.tools.utils" @@ -138,7 +138,6 @@ function PostgresDB:query(query, schema) PostgresDB.super.query(self, query) local conn_opts = self:_get_conn_options() -print(require("pl.pretty").write(conn_opts)) local pg = pgmoon.new(conn_opts) local ok, err = pg:connect() if not ok then diff --git a/kong/tools/dns.lua b/kong/tools/dns.lua index ac679ec6f02..2c4334467e9 100644 --- a/kong/tools/dns.lua +++ b/kong/tools/dns.lua @@ -1,12 +1,11 @@ local dns_client --- Load and setup the DNS client according to the provided configuration. +-- Will clear the cache if called multiple times. -- @param conf (table) Kong configuration -- @return the initialized `dns.client` module, or nil+error if it was already initialized local setup_client = function(conf) - if dns_client then - return nil, "DNS client already initialized" - else + if not dns_client then dns_client = require "dns.client" end From 3144e77d013c344471e26cf58d3c7262b951a11a Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Fri, 9 Sep 2016 17:30:25 -0700 Subject: [PATCH 07/12] fix(globalpatches) randomseed throws error when seeded in wrong context --- kong/core/globalpatches.lua | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kong/core/globalpatches.lua b/kong/core/globalpatches.lua index 433f9290529..c69d3aae3d6 100644 --- a/kong/core/globalpatches.lua +++ b/kong/core/globalpatches.lua @@ -17,18 +17,19 @@ local seed -- unique seed for Nginx workers). -- luacheck: globals math _G.math.randomseed = function() - if ngx.get_phase() ~= "init_worker" then - ngx.log(ngx.ERR, "math.randomseed() must be called in init_worker") - elseif not seed then + if not seed then + if ngx.get_phase() ~= "init_worker" then + error("math.randomseed() must be called in init_worker", 2) + end + seed = ngx.time() + ngx.worker.pid() ngx.log(ngx.DEBUG, "random seed: ", seed, " for worker n", ngx.worker.id(), " (pid: ", ngx.worker.pid(), ")") randomseed(seed) else ngx.log(ngx.DEBUG, "attempt to seed random number generator, but ", - "already seeded") + "already seeded with ", seed) end return seed end - From 19ba9905aa77c2a5973d6807bbe91bd5c07b956f Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 14 Sep 2016 03:22:03 +0200 Subject: [PATCH 08/12] added patches to make the cli work with co-sockets instead of luasocket --- kong/core/globalpatches.lua | 152 ++++++++++++++++++++++++------------ kong/dao/cassandra_db.lua | 3 - 2 files changed, 103 insertions(+), 52 deletions(-) diff --git a/kong/core/globalpatches.lua b/kong/core/globalpatches.lua index 1ada58aacf5..90d479caa31 100644 --- a/kong/core/globalpatches.lua +++ b/kong/core/globalpatches.lua @@ -1,64 +1,118 @@ local meta = require "kong.meta" -local randomseed = math.randomseed _G._KONG = { _NAME = meta._NAME, _VERSION = meta._VERSION } -local seed - ---- Seeds the random generator, use with care. --- The uuid.seed() method will create a unique seed per worker --- process, using a combination of both time and the worker's pid. --- We only allow it to be called once to prevent third-party modules --- from overriding our correct seed (many modules make a wrong usage --- of `math.randomseed()` by calling it multiple times or do not use --- unique seed for Nginx workers). --- luacheck: globals math -_G.math.randomseed = function() - if not seed then - seed = ngx.time() + ngx.worker.pid() - ngx.log(ngx.DEBUG, "random seed: ", seed, " for worker n", ngx.worker.id(), - " (pid: ", ngx.worker.pid(), ")") - randomseed(seed) - else - ngx.log(ngx.DEBUG, "attempt to seed random number generator, but ", - "already seeded") - end - return seed -end +do -- randomseeding patch + + local randomseed = math.randomseed + local seed ---- Patch the TCP connect method such that all connections will be resolved --- first by the internal DNS resolver. --- STEP 1: load code that should not be using the patched versions -require "resty.dns.resolver" -- will cache TCP and UDP functions --- STEP 2: forward declaration of locals to hold stuff loaded AFTER patching -local toip --- STEP 3: store original unpatched versions -local old_tcp = ngx.socket.tcp --- STEP 4: patch globals -print(" ======================> Patching global socket.tcp...") -_G.ngx.socket.tcp = function(...) - local sock = old_tcp(...) - local old_connect = sock.connect - sock.connect = function(s, host, port, sock_opts) -print(" ======================> Patching a tcp socket here...") - local target_ip, target_port = toip(host, port) - - if not target_ip then - return nil, "[toip() name lookup failed]:"..tostring(target_port) + --- Seeds the random generator, use with care. + -- The uuid.seed() method will create a unique seed per worker + -- process, using a combination of both time and the worker's pid. + -- We only allow it to be called once to prevent third-party modules + -- from overriding our correct seed (many modules make a wrong usage + -- of `math.randomseed()` by calling it multiple times or do not use + -- unique seed for Nginx workers). + -- luacheck: globals math + _G.math.randomseed = function() + if not seed then + seed = ngx.time() + ngx.worker.pid() + ngx.log(ngx.DEBUG, "random seed: ", seed, " for worker n", ngx.worker.id(), + " (pid: ", ngx.worker.pid(), ")") + randomseed(seed) else - -- need to do the extra check here: https://github.com/openresty/lua-nginx-module/issues/860 - if not sock_opts then - return old_connect(s, target_ip, target_port) + ngx.log(ngx.DEBUG, "attempt to seed random number generator, but ", + "already seeded") + end + + return seed + end +end + + + +do -- cosockets connect patch for dns resolution + + --- Patch the TCP connect method such that all connections will be resolved + -- first by the internal DNS resolver. + -- STEP 1: load code that should not be using the patched versions + require "resty.dns.resolver" -- will cache TCP and UDP functions + -- STEP 2: forward declaration of locals to hold stuff loaded AFTER patching + local toip + -- STEP 3: store original unpatched versions + local old_tcp = ngx.socket.tcp + -- STEP 4: patch globals + _G.ngx.socket.tcp = function(...) + local sock = old_tcp(...) + local old_connect = sock.connect + sock.connect = function(s, host, port, sock_opts) + local target_ip, target_port = toip(host, port) + + if not target_ip then + return nil, "[toip() name lookup failed]:"..tostring(target_port) else - return old_connect(s, target_ip, target_port, sock_opts) + -- need to do the extra check here: https://github.com/openresty/lua-nginx-module/issues/860 + if not sock_opts then + return old_connect(s, target_ip, target_port) + else + return old_connect(s, target_ip, target_port, sock_opts) + end end end + return sock + end + -- STEP 5: load code that should be using the patched versions, if any (because of dependency chain) + toip = require("dns.client").toip -- this will load utils and penlight modules for example +end + + + +do -- patch for LuaSocket tcp sockets, block usage in cli. + + if arg then -- resty has an `arg` global, `openresty` does not + local socket = require "socket" + socket.tcp = function(...) + error("should not be using this") + end end - return sock end --- STEP 5: load code that should be using the patched versions, if any (because of dependency chain) -toip = require("dns.client").toip -- this will load utils and penlight modules for example + + + +do -- Cassandra cache-shm patch + + --- Patch cassandra driver. + -- The cache module depends on an `shm` which isn't available on the `resty` cli. + -- in non-nginx Lua it uses a stub. So for the cli make it think it's non-nginx. + if arg then -- resty has an `arg` global, `openresty` does not + local old_ngx = _G.ngx + _G.ngx = nil + require "cassandra.cache" + _G.ngx = old_ngx + end +end + + + +do -- cassandra resty-lock patch + + --- stub for resty.lock module which isn't available in the `resty` cli because it + -- requires an `shm`. + package.loaded["resty.lock"] = { + new = function() + return { + lock = function(self, key) + return 0 -- cli is single threaded, so a lock always succeeds + end, + unlock = function(self, key) + return 1 -- same as above, always succeeds + end + } + end, + } +end diff --git a/kong/dao/cassandra_db.lua b/kong/dao/cassandra_db.lua index 4c83dae4060..43051753e2e 100644 --- a/kong/dao/cassandra_db.lua +++ b/kong/dao/cassandra_db.lua @@ -4,10 +4,7 @@ local BaseDB = require "kong.dao.base_db" local utils = require "kong.tools.utils" local uuid = utils.uuid -local ngx_stub = _G.ngx -_G.ngx = nil local cassandra = require "cassandra" -_G.ngx = ngx_stub local CassandraDB = BaseDB:extend() From 3ed9468f791968cbf4dd5d4240756aa5fdc028b0 Mon Sep 17 00:00:00 2001 From: thefosk Date: Wed, 14 Sep 2016 10:37:12 -0700 Subject: [PATCH 09/12] fix(oauth2) removing Kong from error message --- kong/plugins/oauth2/access.lua | 6 +++--- spec/03-plugins/99-oauth2/03-access_spec.lua | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kong/plugins/oauth2/access.lua b/kong/plugins/oauth2/access.lua index 086c793c8ee..60a0a7e5455 100644 --- a/kong/plugins/oauth2/access.lua +++ b/kong/plugins/oauth2/access.lua @@ -132,7 +132,7 @@ local function authorize(conf) response_params = {[ERROR] = "access_denied", error_description = err or "You must use HTTPS"} else if conf.provision_key ~= parameters.provision_key then - response_params = {[ERROR] = "invalid_provision_key", error_description = "Invalid Kong provision_key"} + response_params = {[ERROR] = "invalid_provision_key", error_description = "Invalid provision_key"} elseif not parameters.authenticated_userid or utils.strip(parameters.authenticated_userid) == "" then response_params = {[ERROR] = "invalid_authenticated_userid", error_description = "Missing authenticated_userid parameter"} else @@ -306,7 +306,7 @@ local function issue_token(conf) elseif grant_type == GRANT_CLIENT_CREDENTIALS then -- Only check the provision_key if the authenticated_userid is being set if parameters.authenticated_userid and conf.provision_key ~= parameters.provision_key then - response_params = {[ERROR] = "invalid_provision_key", error_description = "Invalid Kong provision_key"} + response_params = {[ERROR] = "invalid_provision_key", error_description = "Invalid provision_key"} else -- Check scopes local ok, scopes = retrieve_scopes(parameters, conf) @@ -319,7 +319,7 @@ local function issue_token(conf) elseif grant_type == GRANT_PASSWORD then -- Check that it comes from the right client if conf.provision_key ~= parameters.provision_key then - response_params = {[ERROR] = "invalid_provision_key", error_description = "Invalid Kong provision_key"} + response_params = {[ERROR] = "invalid_provision_key", error_description = "Invalid provision_key"} elseif not parameters.authenticated_userid or utils.strip(parameters.authenticated_userid) == "" then response_params = {[ERROR] = "invalid_authenticated_userid", error_description = "Missing authenticated_userid parameter"} else diff --git a/spec/03-plugins/99-oauth2/03-access_spec.lua b/spec/03-plugins/99-oauth2/03-access_spec.lua index ec5fb1da638..bf503170017 100644 --- a/spec/03-plugins/99-oauth2/03-access_spec.lua +++ b/spec/03-plugins/99-oauth2/03-access_spec.lua @@ -203,7 +203,7 @@ describe("#ci Plugin: oauth2 (access)", function() } }) local body = assert.res_status(400, res) - assert.equal([[{"error_description":"Invalid Kong provision_key","error":"invalid_provision_key"}]], body) + assert.equal([[{"error_description":"Invalid provision_key","error":"invalid_provision_key"}]], body) assert.are.equal("no-store", res.headers["cache-control"]) assert.are.equal("no-cache", res.headers["pragma"]) end) @@ -474,7 +474,7 @@ describe("#ci Plugin: oauth2 (access)", function() } }) local body = assert.res_status(400, res) - assert.equal([[{"error_description":"Invalid Kong provision_key","error":"invalid_provision_key"}]], body) + assert.equal([[{"error_description":"Invalid provision_key","error":"invalid_provision_key"}]], body) end) it("returns success with a path", function() local res = assert(proxy_ssl_client:send { @@ -816,7 +816,7 @@ describe("#ci Plugin: oauth2 (access)", function() } }) local body = assert.res_status(400, res) - assert.equal([[{"error_description":"Invalid Kong provision_key","error":"invalid_provision_key"}]], body) + assert.equal([[{"error_description":"Invalid provision_key","error":"invalid_provision_key"}]], body) end) it("fails when setting authenticated_userid and invalid provision_key", function() local res = assert(proxy_ssl_client:send { @@ -836,7 +836,7 @@ describe("#ci Plugin: oauth2 (access)", function() } }) local body = assert.res_status(400, res) - assert.equal([[{"error_description":"Invalid Kong provision_key","error":"invalid_provision_key"}]], body) + assert.equal([[{"error_description":"Invalid provision_key","error":"invalid_provision_key"}]], body) end) it("returns success", function() local res = assert(proxy_ssl_client:send { @@ -1097,7 +1097,7 @@ describe("#ci Plugin: oauth2 (access)", function() } }) local body = assert.res_status(400, res) - assert.equal([[{"error_description":"Invalid Kong provision_key","error":"invalid_provision_key"}]], body) + assert.equal([[{"error_description":"Invalid provision_key","error":"invalid_provision_key"}]], body) end) it("fails when no provision key is being sent", function() local res = assert(proxy_ssl_client:send { @@ -1115,7 +1115,7 @@ describe("#ci Plugin: oauth2 (access)", function() } }) local body = assert.res_status(400, res) - assert.equal([[{"error_description":"Invalid Kong provision_key","error":"invalid_provision_key"}]], body) + assert.equal([[{"error_description":"Invalid provision_key","error":"invalid_provision_key"}]], body) end) it("fails when no authenticated user id is being sent", function() local res = assert(proxy_ssl_client:send { From 4ff30682c46937093b066d09887ca2b818845bdd Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Wed, 14 Sep 2016 13:06:09 -0700 Subject: [PATCH 10/12] chore(ci) better Travis-CI setup_env.sh This allows for caching multiple versions of our dependencies at once. --- .ci/run_tests.sh | 2 +- .ci/setup_env.sh | 97 +++++++++++++++++++++++++++++------------------- .travis.yml | 13 +++++-- 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh index 1d0df964bcf..64869d2fecc 100755 --- a/.ci/run_tests.sh +++ b/.ci/run_tests.sh @@ -1,7 +1,7 @@ set -e export BUSTED_ARGS="-o gtest -v --exclude-tags=ci" -export TEST_CMD="KONG_SERF_PATH=$SERF_INSTALL/serf bin/busted $BUSTED_ARGS" +export TEST_CMD="KONG_SERF_PATH=$SERF_PATH bin/busted $BUSTED_ARGS" if [ "$TEST_SUITE" == "lint" ]; then make lint diff --git a/.ci/setup_env.sh b/.ci/setup_env.sh index 72765802390..7be06febc17 100755 --- a/.ci/setup_env.sh +++ b/.ci/setup_env.sh @@ -1,31 +1,61 @@ set -e -export OPENRESTY_INSTALL=$CACHE_DIR/openresty -export LUAROCKS_INSTALL=$CACHE_DIR/luarocks -export SERF_INSTALL=$CACHE_DIR/serf +#--------- +# Download +#--------- +OPENSSL_DOWNLOAD=$DOWNLOAD_CACHE/openssl-$OPENSSL +OPENRESTY_DOWNLOAD=$DOWNLOAD_CACHE/openresty-$OPENRESTY +LUAROCKS_DOWNLOAD=$DOWNLOAD_CACHE/luarocks-$LUAROCKS +SERF_DOWNLOAD=$DOWNLOAD_CACHE/serf-$SERF -mkdir -p $CACHE_DIR +mkdir -p $OPENSSL_DOWNLOAD $OPENRESTY_DOWNLOAD $LUAROCKS_DOWNLOAD $SERF_DOWNLOAD -if [ ! "$(ls -A $CACHE_DIR)" ]; then - # Not in cache +if [ ! "$(ls -A $OPENSSL_DOWNLOAD)" ]; then + pushd $DOWNLOAD_CACHE + curl -L http://www.openssl.org/source/openssl-$OPENSSL.tar.gz | tar xz + popd +fi + +if [ ! "$(ls -A $OPENRESTY_DOWNLOAD)" ]; then + pushd $DOWNLOAD_CACHE + curl -L https://openresty.org/download/openresty-$OPENRESTY.tar.gz | tar xz + popd +fi + +if [ ! "$(ls -A $LUAROCKS_DOWNLOAD)" ]; then + git clone https://github.com/keplerproject/luarocks.git $LUAROCKS_DOWNLOAD +fi + +if [ ! "$(ls -A $SERF_DOWNLOAD)" ]; then + pushd $SERF_DOWNLOAD + wget https://releases.hashicorp.com/serf/${SERF}/serf_${SERF}_linux_amd64.zip + unzip serf_${SERF}_linux_amd64.zip + popd +fi + +#-------- +# Install +#-------- +OPENSSL_INSTALL=$INSTALL_CACHE/openssl-$OPENSSL +OPENRESTY_INSTALL=$INSTALL_CACHE/openresty-$OPENRESTY +LUAROCKS_INSTALL=$INSTALL_CACHE/luarocks-$LUAROCKS +SERF_INSTALL=$INSTALL_CACHE/serf-$SERF - # --------------- - # Install OpenSSL - # --------------- - OPENSSL_BASE=openssl-$OPENSSL - curl -L http://www.openssl.org/source/$OPENSSL_BASE.tar.gz | tar xz +mkdir -p $OPENSSL_INSTALL $OPENRESTY_INSTALL $LUAROCKS_INSTALL $SERF_INSTALL - # ----------------- - # Install OpenResty - # ----------------- - OPENRESTY_BASE=openresty-$OPENRESTY - mkdir -p $OPENRESTY_INSTALL - curl -L https://openresty.org/download/$OPENRESTY_BASE.tar.gz | tar xz +if [ ! "$(ls -A $OPENSSL_INSTALL)" ]; then + pushd $OPENSSL_DOWNLOAD + ./config shared --prefix=$OPENSSL_INSTALL + make + make install + popd +fi - pushd $OPENRESTY_BASE +if [ ! "$(ls -A $OPENRESTY_INSTALL)" ]; then + pushd $OPENRESTY_DOWNLOAD ./configure \ --prefix=$OPENRESTY_INSTALL \ - --with-openssl=../$OPENSSL_BASE \ + --with-openssl=$OPENSSL_DOWNLOAD \ --with-ipv6 \ --with-pcre-jit \ --with-http_ssl_module \ @@ -34,17 +64,10 @@ if [ ! "$(ls -A $CACHE_DIR)" ]; then make make install popd +fi - rm -rf $OPENRESTY_BASE - - # ---------------- - # Install Luarocks - # ---------------- - LUAROCKS_BASE=luarocks-$LUAROCKS - mkdir -p $LUAROCKS_INSTALL - git clone https://github.com/keplerproject/luarocks.git $LUAROCKS_BASE - - pushd $LUAROCKS_BASE +if [ ! "$(ls -A $LUAROCKS_INSTALL)" ]; then + pushd $LUAROCKS_DOWNLOAD git checkout v$LUAROCKS ./configure \ --prefix=$LUAROCKS_INSTALL \ @@ -54,19 +77,15 @@ if [ ! "$(ls -A $CACHE_DIR)" ]; then make build make install popd +fi - rm -rf $LUAROCKS_BASE - - # ------------ - # Install Serf - # ------------ - mkdir -p $SERF_INSTALL - pushd $SERF_INSTALL - wget https://releases.hashicorp.com/serf/${SERF}/serf_${SERF}_linux_amd64.zip - unzip serf_${SERF}_linux_amd64.zip - popd +if [ ! "$(ls -A $SERF_INSTALL)" ]; then + ln -s $SERF_DOWNLOAD/serf $SERF_INSTALL/serf fi +export OPENSSL_DIR=$OPENSSL_INSTALL # for LuaSec install +export SERF_PATH=$SERF_INSTALL/serf # for our test instance (not in default bin/sh $PATH) + export PATH=$PATH:$OPENRESTY_INSTALL/nginx/sbin:$OPENRESTY_INSTALL/bin:$LUAROCKS_INSTALL/bin:$SERF_INSTALL eval `luarocks path` diff --git a/.travis.yml b/.travis.yml index ae3a214ff8c..2c57795dea6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,10 @@ sudo: false +language: c + +compiler: + - gcc + notifications: email: false @@ -15,11 +20,12 @@ addons: env: global: - SERF=0.7.0 - - LUAROCKS=2.3.0 + - LUAROCKS=2.4.0 - OPENSSL=1.0.2h - CASSANDRA=2.2.7 - OPENRESTY=1.9.15.1 - - $CACHE_DIR=$HOME/cache + - DOWNLOAD_CACHE=$HOME/download-cache + - INSTALL_CACHE=$HOME/install-cache matrix: - TEST_SUITE=lint - TEST_SUITE=unit @@ -39,5 +45,6 @@ cache: apt: true pip: true directories: - - $CACHE_DIR + - $DOWNLOAD_CACHE + - $INSTALL_CACHE - $HOME/.ccm/repository From e08972519ce69359d96884b70c5222d13b60e166 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 15 Sep 2016 22:58:04 +0200 Subject: [PATCH 11/12] fix Kong starting errors, due to yield across c-boundary error --- bin/busted | 2 ++ kong/cmd/start.lua | 8 ++++---- kong/tools/dns.lua | 7 +++---- spec/02-integration/04-core/02-hooks_spec.lua | 2 +- spec/helpers.lua | 15 +++++++++++++++ 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/bin/busted b/bin/busted index c809baa4e34..d955c785be1 100755 --- a/bin/busted +++ b/bin/busted @@ -2,6 +2,8 @@ package.path = "?/init.lua;"..package.path +-- require "kong.core.globalpatches" + if ngx ~= nil then ngx.exit = function() end end diff --git a/kong/cmd/start.lua b/kong/cmd/start.lua index 687fd78a6e0..fe22178b262 100644 --- a/kong/cmd/start.lua +++ b/kong/cmd/start.lua @@ -19,14 +19,14 @@ local function execute(args) assert(nginx_signals.start(conf)) log("Kong started") end, function(e) - log.verbose("could not start Kong, stopping services") - pcall(nginx_signals.stop(conf)) - pcall(serf_signals.stop(conf, dao)) err = e -- cannot throw from this function - log.verbose("stopped services") end) if err then + log.verbose("could not start Kong, stopping services") + pcall(nginx_signals.stop(conf)) + pcall(serf_signals.stop(conf, dao)) + log.verbose("stopped services") error(err) -- report to main error handler end end diff --git a/kong/tools/dns.lua b/kong/tools/dns.lua index 2c4334467e9..c3362605556 100644 --- a/kong/tools/dns.lua +++ b/kong/tools/dns.lua @@ -11,12 +11,11 @@ local setup_client = function(conf) conf = conf or {} local hosts = conf.dns_hostsfile -- filename - local servers = conf.dns_resolver -- array with ipv4[:port] entries - if #servers == 0 then servers = nil end -- drop an empty table to switch to defaults from `resolv.conf` + local servers = {} -- servers must be reformatted as name/port sub-arrays - if servers then - for i, server in ipairs(servers) do + if conf.dns_resolver then + for i, server in ipairs(conf.dns_resolver) do local ip, port = server:match("^([^:]+)%:*(%d*)$") servers[i] = { ip, tonumber(port) or 53 } -- inserting port if omitted end diff --git a/spec/02-integration/04-core/02-hooks_spec.lua b/spec/02-integration/04-core/02-hooks_spec.lua index 444b1707724..96a7f28ae2d 100644 --- a/spec/02-integration/04-core/02-hooks_spec.lua +++ b/spec/02-integration/04-core/02-hooks_spec.lua @@ -657,7 +657,7 @@ describe("Core Hooks", function() os.execute(string.format("kill `cat %s` >/dev/null 2>&1", PID_FILE)) end - it("should syncronize nodes on members events", function() + it("should synchronize nodes on members events", function() start_serf() -- Tell Kong to join the new Serf diff --git a/spec/helpers.lua b/spec/helpers.lua index 084b9045704..0ae249b61ad 100644 --- a/spec/helpers.lua +++ b/spec/helpers.lua @@ -34,6 +34,21 @@ local dao = DAOFactory(conf) ----------------- local resty_http_proxy_mt = {} +local pack = function(...) return { n = select("#", ...), ... } end +local unpack = function(t) return unpack(t, 1, t.n) end + +--- Prints all returned parameters. +-- Simple debugging aid. +-- @usage -- modify +-- local a,b = some_func(c,d) +-- -- into +-- local a,b = intercept(some_func(c,d)) +local function intercept(...) + local args = pack(...) + print(require("pl.pretty").write(args)) + return unpack(args) +end + -- Case insensitive lookup function, returns the value and the original key. Or -- if not found nil and the search key -- @usage -- sample usage From 04d14893bf3a5e1b584fa2c01f9b11012fc9b526 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 16 Sep 2016 02:24:37 +0200 Subject: [PATCH 12/12] fixes the cassandra tests --- bin/busted | 4 ++-- kong/core/globalpatches.lua | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/busted b/bin/busted index d955c785be1..ee69d53046c 100755 --- a/bin/busted +++ b/bin/busted @@ -1,8 +1,8 @@ -#!/usr/bin/env resty +#!/usr/bin/env resty -c 65535 package.path = "?/init.lua;"..package.path --- require "kong.core.globalpatches" +require "kong.core.globalpatches" if ngx ~= nil then ngx.exit = function() end diff --git a/kong/core/globalpatches.lua b/kong/core/globalpatches.lua index 64a4906daf9..66f0da49ff6 100644 --- a/kong/core/globalpatches.lua +++ b/kong/core/globalpatches.lua @@ -75,9 +75,9 @@ do -- cosockets connect patch for dns resolution _G.ngx.socket.tcp = function(...) local sock = old_tcp(...) local old_connect = sock.connect + sock.connect = function(s, host, port, sock_opts) local target_ip, target_port = toip(host, port) - if not target_ip then return nil, "[toip() name lookup failed]:"..tostring(target_port) else