From 845bf5986065a10ea48a1405b874a320836697aa Mon Sep 17 00:00:00 2001 From: thefosk Date: Thu, 30 Apr 2015 22:59:34 -0700 Subject: [PATCH 1/2] dnsmasq integration for better DNS resolution --- .travis.yml | 3 +- .travis/setup_dnsmasq.sh | 4 ++ kong.yml | 3 +- kong/cli/quit.lua | 2 +- kong/cli/reload.lua | 2 +- kong/cli/restart.lua | 2 +- kong/cli/stop.lua | 2 +- kong/cli/utils/signal.lua | 71 ++++++++++++++++++++++++-- kong/constants.lua | 3 +- kong/tools/faker.lua | 3 ++ kong/tools/io.lua | 42 +++++++++++---- spec/integration/proxy/dns_spec.lua | 59 +++++++++++++++++++++ spec/integration/proxy/realip_spec.lua | 1 - spec/spec_helpers.lua | 4 +- spec/unit/statics_spec.lua | 3 +- spec/unit/tools/faker_spec.lua | 2 +- spec/unit/tools/syslog_spec.lua | 2 + 17 files changed, 182 insertions(+), 26 deletions(-) create mode 100644 .travis/setup_dnsmasq.sh create mode 100644 spec/integration/proxy/dns_spec.lua diff --git a/.travis.yml b/.travis.yml index 6ca2ed50d74..cf7a7c3fd09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ before_install: - bash .travis/setup_openresty.sh - export PATH=$PATH:/usr/local/openresty/nginx/sbin - bash .travis/setup_cassandra.sh + - bash .travis/setup_dnsmasq.sh install: - sudo make install @@ -20,4 +21,4 @@ install: script: "busted --coverage spec/" -after_success: "luacov-coveralls -i kong" +after_success: "luacov-coveralls -i kong" \ No newline at end of file diff --git a/.travis/setup_dnsmasq.sh b/.travis/setup_dnsmasq.sh new file mode 100644 index 00000000000..75a6e6b7ea8 --- /dev/null +++ b/.travis/setup_dnsmasq.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +sudo apt-get update && sudo apt-get install dnsmasq sudo +echo -e "user=root" | sudo tee /etc/dnsmasq.conf \ No newline at end of file diff --git a/kong.yml b/kong.yml index 3000cc92974..6e5711ef323 100644 --- a/kong.yml +++ b/kong.yml @@ -13,6 +13,7 @@ nginx_working_dir: /usr/local/kong/ proxy_port: 8000 admin_api_port: 8001 +dnsmasq_port: 8053 # Specify the DAO to use database: cassandra @@ -49,7 +50,7 @@ nginx: | } http { - resolver 8.8.8.8; + resolver {{dns_resolver}}; charset UTF-8; access_log logs/access.log; diff --git a/kong/cli/quit.lua b/kong/cli/quit.lua index ee620409e38..9393e80d58b 100644 --- a/kong/cli/quit.lua +++ b/kong/cli/quit.lua @@ -19,7 +19,7 @@ if not running then end -- Send 'quit' signal (graceful shutdown) -if signal.send_signal(args.config, "quit") then +if signal.send_signal(args.config, signal.QUIT) then cutils.logger:success("Stopped") else cutils.logger:error_exit("Could not gracefully stop Kong") diff --git a/kong/cli/reload.lua b/kong/cli/reload.lua index 8fa510af511..359a59e5701 100644 --- a/kong/cli/reload.lua +++ b/kong/cli/reload.lua @@ -20,7 +20,7 @@ end signal.prepare_kong(args.config) -if signal.send_signal(args.config, "reload") then +if signal.send_signal(args.config, signal.RELOAD) then cutils.logger:success("Reloaded") else cutils.logger:error_exit("Could not reload Kong") diff --git a/kong/cli/restart.lua b/kong/cli/restart.lua index 578668018f7..c347cb3c638 100644 --- a/kong/cli/restart.lua +++ b/kong/cli/restart.lua @@ -16,7 +16,7 @@ Options: ]], constants.CLI.GLOBAL_KONG_CONF)) if signal.is_running(args.config) then - if not signal.send_signal(args.config, "stop") then + if not signal.send_signal(args.config, signal.STOP) then cutils.logger:error_exit("Could not stop Kong") end end diff --git a/kong/cli/stop.lua b/kong/cli/stop.lua index 552b184df7c..2efe38b712b 100755 --- a/kong/cli/stop.lua +++ b/kong/cli/stop.lua @@ -19,7 +19,7 @@ if not running then end -- Send 'stop' signal (fast shutdown) -if signal.send_signal(args.config, "stop") then +if signal.send_signal(args.config, signal.STOP) then cutils.logger:success("Stopped") else cutils.logger:error_exit("Could not stop Kong") diff --git a/kong/cli/utils/signal.lua b/kong/cli/utils/signal.lua index bb739b09b6b..28bdc832434 100644 --- a/kong/cli/utils/signal.lua +++ b/kong/cli/utils/signal.lua @@ -6,6 +6,7 @@ local IO = require "kong.tools.io" local cutils = require "kong.cli.utils" local constants = require "kong.constants" local syslog = require "kong.tools.syslog" +local stringy = require "stringy" -- Cache config path, parsed config and DAO factory local kong_config_path @@ -87,7 +88,8 @@ local function prepare_nginx_working_dir(args_config) local nginx_config = kong_config.nginx local nginx_inject = { proxy_port = kong_config.proxy_port, - admin_api_port = kong_config.admin_api_port + admin_api_port = kong_config.admin_api_port, + dns_resolver = "127.0.0.1:"..kong_config.dnsmasq_port } -- Auto-tune @@ -153,12 +155,55 @@ local function prepare_database(args_config) end end +local function stop_dnsmasq(kong_config) + local file_pid = kong_config.nginx_working_dir.."/"..constants.CLI.DNSMASQ_PID + if IO.file_exists(file_pid) then + local res, code = IO.os_execute("cat "..file_pid) + if code == 0 then + local _, kill_code = IO.kill_process_by_pid(res) + if kill_code and kill_code == 0 then + cutils.logger:info("dnsmasq stopped") + end + else + cutils.logger:error_exit(res) + end + end +end + +local function start_dnsmasq(kong_config) + local cmd = IO.cmd_exists("dnsmasq") and "dnsmasq" or + (IO.cmd_exists("/usr/local/sbin/dnsmasq") and "/usr/local/sbin/dnsmasq" or nil) -- On OS X dnsmasq is at /usr/local/sbin/ + if not cmd then + cutils.logger:error_exit("Can't find dnsmasq") + end + + -- Start the dnsmasq + local file_pid = kong_config.nginx_working_dir..(stringy.endswith(kong_config.nginx_working_dir, "/") and "" or "/")..constants.CLI.DNSMASQ_PID + local res, code = IO.os_execute(cmd.." -p "..kong_config.dnsmasq_port.." --pid-file="..file_pid) + if code ~= 0 then + cutils.logger:error_exit(res) + else + cutils.logger:info("dnsmasq started") + end +end + -- -- PUBLIC -- local _M = {} +-- Constants +local START = "start" +local RESTART = "restart" +local RELOAD = "reload" +local STOP = "stop" +local QUIT = "quit" + +_M.RELOAD = RELOAD +_M.STOP = STOP +_M.QUIT = QUIT + function _M.prepare_kong(args_config) local kong_config = get_kong_config(args_config) local dao_config = kong_config.databases_available[kong_config.database].properties @@ -169,10 +214,12 @@ function _M.prepare_kong(args_config) -- Print important informations cutils.logger:info(string.format([[Proxy port.........%s Admin API port.....%s + dnsmasq port.......%s Database...........%s %s ]], kong_config.proxy_port, kong_config.admin_api_port, + kong_config.dnsmasq_port, kong_config.database, tostring(dao_config))) @@ -205,19 +252,35 @@ function _M.send_signal(args_config, signal) constants.CLI.NGINX_PID, signal ~= nil and "-s "..signal or "") - if not signal then signal = "start" end - if signal == "start" or signal == "restart" or signal == "reload" then + if not signal then signal = START end + + if signal == START then + stop_dnsmasq(kong_config) + start_dnsmasq(kong_config) + end + + if signal == STOP then + stop_dnsmasq(kong_config) + end + + -- Check ulimit value + if signal == START or signal == RESTART or signal == RELOAD then local res, code = IO.os_execute("ulimit -n") if code == 0 and tonumber(res) < 4096 then cutils.logger:warn("ulimit is currently set to \""..res.."\". For better performance set it to at least \"4096\" using \"ulimit -n\"") end end + -- Check settings for anonymous reports if kong_config.send_anonymous_reports then syslog.log({signal=signal}) end - return os.execute(cmd) == 0 + local success = os.execute(cmd) == 0 + if signal == START and not success then + stop_dnsmasq(kong_config) + end + return success end -- Test if Kong is already running by detecting a pid file. diff --git a/kong/constants.lua b/kong/constants.lua index 1c80f6c6d59..58477ca031e 100644 --- a/kong/constants.lua +++ b/kong/constants.lua @@ -8,7 +8,8 @@ return { CLI = { GLOBAL_KONG_CONF = "/etc/kong/kong.yml", NGINX_CONFIG = "nginx.conf", - NGINX_PID = "kong.pid" + NGINX_PID = "kong.pid", + DNSMASQ_PID = "dnsmasq.pid", }, DATABASE_NULL_ID = "00000000-0000-0000-0000-000000000000", DATABASE_ERROR_TYPES = { diff --git a/kong/tools/faker.lua b/kong/tools/faker.lua index f72b6e03292..986ec1ca0b2 100644 --- a/kong/tools/faker.lua +++ b/kong/tools/faker.lua @@ -33,6 +33,9 @@ Faker.FIXTURES = { { name = "API TESTS 6", public_dns = "cors1.com", target_url = "http://mockbin.com" }, { name = "API TESTS 7", public_dns = "cors2.com", target_url = "http://mockbin.com" }, + { name = "API TESTS 8 (dns)", public_dns = "dns1.com", target_url = "http://127.0.0.1:7771" }, + { name = "API TESTS 9 (dns)", public_dns = "dns2.com", target_url = "http://localhost:7771" }, + -- DEVELOPMENT APIs. Please do not use those in tests { name = "API DEV 1", public_dns = "dev.com", target_url = "http://mockbin.com" }, }, diff --git a/kong/tools/io.lua b/kong/tools/io.lua index f5bf22acaaf..dba1d6557ad 100644 --- a/kong/tools/io.lua +++ b/kong/tools/io.lua @@ -1,11 +1,22 @@ local constants = require "kong.constants" local path = require("path").new("/") local yaml = require "yaml" +local stringy = require "stringy" local _M = {} _M.path = path +function _M.file_exists(name) + local f, err = io.open(name, "r") + if f ~= nil then + io.close(f) + return true + else + return false, err + end +end + function _M.os_execute(command) local n = os.tmpname() -- get a temporary file name to store output local exit_code = os.execute("/bin/bash -c '"..command.." > "..n.." 2>&1'") @@ -15,6 +26,20 @@ function _M.os_execute(command) return string.gsub(result, "[%\r%\n]", ""), exit_code / 256 end +function _M.cmd_exists(cmd) + local _, code = _M.os_execute("hash "..cmd) + return code == 0 +end + +-- Kills a process by PID and waits until it's terminated +-- +-- @param {string} the pid to kill +function _M.kill_process_by_pid(pid, signal) + local res, code = _M.os_execute("kill "..(signal and "-"..tostring(signal).." " or "")..pid) + _M.os_execute("wait "..pid) + return res, code +end + function _M.read_file(path) local contents = nil local file = io.open(path, "rb") @@ -36,16 +61,6 @@ function _M.write_to_file(path, value) return true end -function _M.file_exists(name) - local f = io.open(name, "r") - if f ~= nil then - io.close(f) - return true - else - return false - end -end - function _M.retrieve_files(dir, options) local fs = require "luarocks.fs" local pattern = options.file_pattern @@ -91,6 +106,13 @@ function _M.load_configuration_and_dao(configuration_path) -- Alias the DAO configuration we are using for this instance for easy access configuration.dao_config = dao_config + -- Load absolute path for the nginx working directory + if not stringy.startswith(configuration.nginx_working_dir, "/") then + -- It's a relative path, convert it to absolute + local fs = require "luarocks.fs" + configuration.nginx_working_dir = fs.current_dir().."/"..configuration.nginx_working_dir + end + -- Instanciate the DAO Factory along with the configuration local DaoFactory = require("kong.dao."..configuration.database..".factory") local dao_factory = DaoFactory(dao_config.properties) diff --git a/spec/integration/proxy/dns_spec.lua b/spec/integration/proxy/dns_spec.lua new file mode 100644 index 00000000000..e5fb9c20395 --- /dev/null +++ b/spec/integration/proxy/dns_spec.lua @@ -0,0 +1,59 @@ +local spec_helper = require "spec.spec_helpers" +local http_client = require "kong.tools.http_client" +local Threads = require "llthreads2.ex" + +local STUB_GET_URL = spec_helper.STUB_GET_URL + +local function start_tcp_server() + local thread = Threads.new({ + function() + local socket = require "socket" + local server = assert(socket.bind("*", 7771)) + local client = server:accept() + local line, err = client:receive() + local message = "{\"ok\": true}" + if not err then client:send("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: "..string.len(message).."\r\n\r\n"..message) end + client:close() + return line + end; + }) + + thread:start() + return thread; +end + +describe("DNS", function() + + setup(function() + spec_helper.prepare_db() + spec_helper.start_kong() + end) + + teardown(function() + spec_helper.stop_kong() + spec_helper.reset_db() + end) + + describe("DNS", function() + + it("should work when calling local IP", function() + local thread = start_tcp_server() -- Starting the mock TCP server + + local response, status = http_client.get(spec_helper.STUB_GET_URL, nil, { host = "dns1.com" }) + assert.are.equal(200, status) + + thread:join() -- Wait til it exists + end) + + it("should work when calling local hostname", function() + local thread = start_tcp_server() -- Starting the mock TCP server + + local response, status = http_client.get(spec_helper.STUB_GET_URL, nil, { host = "dns2.com" }) + assert.are.equal(200, status) + + thread:join() -- Wait til it exists + end) + + end) + +end) diff --git a/spec/integration/proxy/realip_spec.lua b/spec/integration/proxy/realip_spec.lua index abc06e7321a..967ec75380a 100644 --- a/spec/integration/proxy/realip_spec.lua +++ b/spec/integration/proxy/realip_spec.lua @@ -1,6 +1,5 @@ local spec_helper = require "spec.spec_helpers" local http_client = require "kong.tools.http_client" -local Threads = require "llthreads2.ex" local cjson = require "cjson" local yaml = require "yaml" local IO = require "kong.tools.io" diff --git a/spec/spec_helpers.lua b/spec/spec_helpers.lua index b0c9d221568..de1c21d60ad 100644 --- a/spec/spec_helpers.lua +++ b/spec/spec_helpers.lua @@ -58,7 +58,7 @@ function _M.start_kong(conf_file, skip_wait) end if not skip_wait then - os.execute("while ! [ -f "..env.configuration.pid_file.." ]; do sleep 1; done") + os.execute("while ! [ -f "..env.configuration.pid_file.." ]; do sleep 0.5; done") end return result, exit_code @@ -72,7 +72,7 @@ function _M.stop_kong(conf_file) error("spec_helper cannot stop kong: "..result) end - os.execute("while [ -f "..env.configuration.pid_file.." ]; do sleep 1; done") + os.execute("while [ -f "..env.configuration.pid_file.." ]; do sleep 0.5; done") return result, exit_code end diff --git a/spec/unit/statics_spec.lua b/spec/unit/statics_spec.lua index b640cfe271b..413fc886a8f 100644 --- a/spec/unit/statics_spec.lua +++ b/spec/unit/statics_spec.lua @@ -50,6 +50,7 @@ nginx_working_dir: /usr/local/kong/ proxy_port: 8000 admin_api_port: 8001 +dnsmasq_port: 8053 # Specify the DAO to use database: cassandra @@ -86,7 +87,7 @@ nginx: | } http { - resolver 8.8.8.8; + resolver {{dns_resolver}}; charset UTF-8; access_log logs/access.log; diff --git a/spec/unit/tools/faker_spec.lua b/spec/unit/tools/faker_spec.lua index e68a7fe501b..82726abbe74 100644 --- a/spec/unit/tools/faker_spec.lua +++ b/spec/unit/tools/faker_spec.lua @@ -78,7 +78,7 @@ describe("Faker #tools", function() it("should be possible to add some random entities complementing the default hard-coded ones", function() faker:seed(2000) assert.spy(faker.insert_from_table).was.called(2) - assert.spy(insert_spy).was.called(8025) -- 3*2000 + base entities + assert.spy(insert_spy).was.called(8027) -- 3*2000 + base entities end) it("should create relations between entities_to_insert and inserted entities", function() diff --git a/spec/unit/tools/syslog_spec.lua b/spec/unit/tools/syslog_spec.lua index 5d199a349ef..c916fcb799c 100644 --- a/spec/unit/tools/syslog_spec.lua +++ b/spec/unit/tools/syslog_spec.lua @@ -66,6 +66,8 @@ describe("Syslog", function() assert.truthy(has_hostname) assert.truthy(has_cores) assert.truthy(has_hello) + + thread:join() -- wait til it exists end) end) \ No newline at end of file From 8219762788f35420caf0c35e12a5f9647eb3c77d Mon Sep 17 00:00:00 2001 From: thefosk Date: Fri, 8 May 2015 18:31:53 -0700 Subject: [PATCH 2/2] IO tests --- spec/unit/tools/io_spec.lua | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 spec/unit/tools/io_spec.lua diff --git a/spec/unit/tools/io_spec.lua b/spec/unit/tools/io_spec.lua new file mode 100644 index 00000000000..3bbca4941e1 --- /dev/null +++ b/spec/unit/tools/io_spec.lua @@ -0,0 +1,37 @@ +local IO = require "kong.tools.io" + +local TEST_FILE = "/tmp/test_file" + +describe("IO", function() + + before_each(function() + os.remove(TEST_FILE) + end) + + it("should detect existing commands", function() + assert.truthy(IO.cmd_exists("hash")) + assert.falsy(IO.cmd_exists("hashasdasd")) + end) + + it("should write and read from files", function() + assert.truthy(IO.write_to_file(TEST_FILE, "this is a test")) + assert.are.same("this is a test", IO.read_file(TEST_FILE)) + end) + + it("should detect existing files", function() + assert.falsy(IO.file_exists(TEST_FILE)) + IO.write_to_file(TEST_FILE, "Test") + assert.truthy(IO.cmd_exists(TEST_FILE)) + end) + + it("should execute an OS command", function() + local res, code = IO.os_execute("echo \"Hello\"") + assert.are.same(0, code) + assert.truthy("Hello", res) + + local res, code = IO.os_execute("asdasda \"Hello\"") + assert.are.same(127, code) + assert.are.same("/bin/bash: asdasda: command not found", res) + end) + +end)