diff --git a/kong-0.3.0-1.rockspec b/kong-0.3.0-1.rockspec index cfaafd1fb28..f11e5820e5a 100644 --- a/kong-0.3.0-1.rockspec +++ b/kong-0.3.0-1.rockspec @@ -71,6 +71,10 @@ build = { ["kong.resolver.header_filter"] = "kong/resolver/header_filter.lua", ["kong.resolver.certificate"] = "kong/resolver/certificate.lua", + ["kong.reports.handler"] = "kong/reports/handler.lua", + ["kong.reports.init_worker"] = "kong/reports/init_worker.lua", + ["kong.reports.log"] = "kong/reports/log.lua", + ["kong.dao.error"] = "kong/dao/error.lua", ["kong.dao.schemas_validation"] = "kong/dao/schemas_validation.lua", ["kong.dao.schemas.apis"] = "kong/dao/schemas/apis.lua", diff --git a/kong.yml b/kong.yml index 5dd56c76bc7..8b76220562b 100644 --- a/kong.yml +++ b/kong.yml @@ -106,6 +106,7 @@ nginx: | lua_code_cache on; lua_max_running_timers 4096; lua_max_pending_timers 16384; + lua_shared_dict locks 100k; lua_shared_dict cache {{memory_cache_size}}m; lua_socket_log_errors off; @@ -118,6 +119,8 @@ nginx: | end '; + init_worker_by_lua 'kong.exec_plugins_init_worker()'; + server { server_name _; listen {{proxy_port}}; diff --git a/kong/api/crud_helpers.lua b/kong/api/crud_helpers.lua index 40577b4dbed..75b01ce8e76 100644 --- a/kong/api/crud_helpers.lua +++ b/kong/api/crud_helpers.lua @@ -1,6 +1,7 @@ local responses = require "kong.tools.responses" local validations = require "kong.dao.schemas_validation" local app_helpers = require "lapis.application" +local utils = require "kong.tools.utils" local _M = {} @@ -93,11 +94,12 @@ function _M.put(params, dao_collection) end end -function _M.post(params, dao_collection) +function _M.post(params, dao_collection, success) local data, err = dao_collection:insert(params) if err then return app_helpers.yield_error(err) else + if success then success(utils.table_copy(data)) end return responses.send_HTTP_CREATED(data) end end diff --git a/kong/api/routes/plugins_configurations.lua b/kong/api/routes/plugins_configurations.lua index 418c5c898c9..8401d6105f2 100644 --- a/kong/api/routes/plugins_configurations.lua +++ b/kong/api/routes/plugins_configurations.lua @@ -1,4 +1,6 @@ local crud = require "kong.api.crud_helpers" +local syslog = require "kong.tools.syslog" +local constants = require "kong.constants" return { ["/plugins_configurations"] = { @@ -11,7 +13,12 @@ return { end, POST = function(self, dao_factory) - crud.post(self.params, dao_factory.plugins_configurations) + crud.post(self.params, dao_factory.plugins_configurations, function(data) + if configuration.send_anonymous_reports then + data.signal = constants.SYSLOG.API + syslog.log(syslog.format_entity(data)) + end + end) end }, diff --git a/kong/constants.lua b/kong/constants.lua index 073cfafb563..880b02ef97f 100644 --- a/kong/constants.lua +++ b/kong/constants.lua @@ -6,7 +6,8 @@ return { ROCK_VERSION = VERSION.."-1", SYSLOG = { ADDRESS = "kong-hf.mashape.com", - PORT = 61828 + PORT = 61828, + API = "api" }, CLI = { GLOBAL_KONG_CONF = "/etc/kong/kong.yml", diff --git a/kong/kong.lua b/kong/kong.lua index 3730e0d04d5..d213f0dbb91 100644 --- a/kong/kong.lua +++ b/kong/kong.lua @@ -112,6 +112,14 @@ local function init_plugins() handler = require("kong.resolver.handler")() }) + if configuration.send_anonymous_reports then + table.insert(result, { + reports = true, + name = "reports", + handler = require("kong.reports.handler")() + }) + end + -- Add the plugins in a sorted order for _, v in utils.sort_table_iter(unsorted_plugins, utils.sort.descending) do -- In descending order if v then @@ -150,6 +158,13 @@ function _M.init() plugins = init_plugins() end +function _M.exec_plugins_init_worker() + -- Calling the Init handler + for _, plugin in ipairs(plugins) do + plugin.handler:init_worker() + end +end + function _M.exec_plugins_certificate() ngx.ctx.plugin_conf = {} @@ -259,8 +274,8 @@ function _M.exec_plugins_log() for _, plugin in ipairs(plugins) do local conf = ngx.ctx.plugin_conf[plugin.name] - if conf then - plugin.handler:log(conf.value) + if conf or plugin.reports then + plugin.handler:log(conf and conf.value or nil) end end end diff --git a/kong/plugins/base_plugin.lua b/kong/plugins/base_plugin.lua index b41b3461428..2f3b5113bd9 100644 --- a/kong/plugins/base_plugin.lua +++ b/kong/plugins/base_plugin.lua @@ -5,6 +5,10 @@ function BasePlugin:new(name) self._name = name end +function BasePlugin:init_worker() + ngx.log(ngx.DEBUG, " executing plugin \""..self._name.."\": init_worker") +end + function BasePlugin:certificate() ngx.log(ngx.DEBUG, " executing plugin \""..self._name.."\": certificate") end diff --git a/kong/reports/handler.lua b/kong/reports/handler.lua new file mode 100644 index 00000000000..e931ca30daf --- /dev/null +++ b/kong/reports/handler.lua @@ -0,0 +1,21 @@ +local BasePlugin = require "kong.plugins.base_plugin" +local init_worker = require "kong.reports.init_worker" +local log = require "kong.reports.log" + +local ReportsHandler = BasePlugin:extend() + +function ReportsHandler:new() + ReportsHandler.super.new(self, "reports") +end + +function ReportsHandler:init_worker() + ReportsHandler.super.init_worker(self) + init_worker.execute() +end + +function ReportsHandler:log() + ReportsHandler.super.log(self) + log.execute() +end + +return ReportsHandler diff --git a/kong/reports/init_worker.lua b/kong/reports/init_worker.lua new file mode 100644 index 00000000000..10989d0f74f --- /dev/null +++ b/kong/reports/init_worker.lua @@ -0,0 +1,34 @@ +local syslog = require "kong.tools.syslog" +local lock = require "resty.lock" +local cache = require "kong.tools.database_cache" + +local INTERVAL = 3600 + +local _M = {} + +local function create_timer(at, cb) + local ok, err = ngx.timer.at(at, cb) + if not ok then + ngx.log(ngx.ERR, "[reports] failed to create timer: ", err) + end +end + +local function send_ping(premature) + local lock = lock:new("locks", { + exptime = INTERVAL - 0.001 + }) + local elapsed = lock:lock("ping") + if elapsed and elapsed == 0 then + local reqs = cache.get(cache.requests_key()) + if not reqs then reqs = 0 end + syslog.log({signal = "ping", requests=reqs}) + end + create_timer(INTERVAL, send_ping) +end + +function _M.execute() + cache.rawset(cache.requests_key(), 0, 0) -- Initializing the counter + create_timer(0, send_ping) +end + +return _M diff --git a/kong/reports/log.lua b/kong/reports/log.lua new file mode 100644 index 00000000000..284525bdc26 --- /dev/null +++ b/kong/reports/log.lua @@ -0,0 +1,9 @@ +local cache = require "kong.tools.database_cache" + +local _M = {} + +function _M.execute(conf) + cache.incr(cache.requests_key(), 1) +end + +return _M \ No newline at end of file diff --git a/kong/tools/database_cache.lua b/kong/tools/database_cache.lua index 66a6073c266..0a7d98d145d 100644 --- a/kong/tools/database_cache.lua +++ b/kong/tools/database_cache.lua @@ -6,30 +6,39 @@ local CACHE_KEYS = { PLUGINS_CONFIGURATIONS = "plugins_configurations", BASICAUTH_CREDENTIAL = "basicauth_credentials", KEYAUTH_CREDENTIAL = "keyauth_credentials", - SSL = "ssl" + SSL = "ssl", + REQUESTS = "requests" } local _M = {} +function _M.rawset(key, value, exptime) + local cache = ngx.shared.cache + return cache:set(key, value, exptime or 0) +end + function _M.set(key, value, exptime) if exptime == nil then exptime = configuration and configuration.database_cache_expiration or 0 end - local cache = ngx.shared.cache if value then value = cjson.encode(value) ngx.log(ngx.DEBUG, " saving cache key \""..key.."\": "..value) end - return cache:set(key, value, exptime) + return _M.rawset(key, value, exptime) end -function _M.get(key) +function _M.rawget(key) ngx.log(ngx.DEBUG, " Try to get cache key \""..key.."\"") - local cache = ngx.shared.cache - local value, flags = cache:get(key) + return cache:get(key) + +end + +function _M.get(key) + local value, flags = _M.rawget(key) if value then ngx.log(ngx.DEBUG, " Found cache value for key \""..key.."\": "..value) value = cjson.decode(value) @@ -37,11 +46,20 @@ function _M.get(key) return value, flags end +function _M.incr(key, value) + local cache = ngx.shared.cache + return cache:incr(key, value) +end + function _M.delete(key) local cache = ngx.shared.cache cache:delete(key) end +function _M.requests_key() + return CACHE_KEYS.REQUESTS +end + function _M.api_key(host) return CACHE_KEYS.APIS.."/"..host end diff --git a/kong/tools/syslog.lua b/kong/tools/syslog.lua index e822870a94a..50077153f04 100644 --- a/kong/tools/syslog.lua +++ b/kong/tools/syslog.lua @@ -1,5 +1,6 @@ local constants = require "kong.constants" local IO = require "kong.tools.io" +local stringy = require "stringy" local _M = {} @@ -40,7 +41,17 @@ function _M.log(args) end -- Send - IO.os_execute("nc -w1 -u "..constants.SYSLOG.ADDRESS.." "..tostring(constants.SYSLOG.PORT).." <<< \"<14>"..info.."\"") + IO.os_execute("nc -w1 -u "..constants.SYSLOG.ADDRESS.." "..tostring(constants.SYSLOG.PORT).." <<< \"<14>"..info.."\" &") +end + +function _M.format_entity(t) + t.created_at = nil + for k, _ in pairs(t) do + if stringy.endswith(k, "id") then + t[k] = nil + end + end + return t end return _M \ No newline at end of file diff --git a/kong/tools/utils.lua b/kong/tools/utils.lua index 33b77ffed35..92ba17f9a52 100644 --- a/kong/tools/utils.lua +++ b/kong/tools/utils.lua @@ -58,6 +58,21 @@ function _M.is_array(t) return true end +function _M.table_copy(orig) + local orig_type = type(orig) + local copy + if orig_type == 'table' then + copy = {} + for orig_key, orig_value in next, orig, nil do + copy[_M.table_copy(orig_key)] = _M.table_copy(orig_value) + end + setmetatable(copy, _M.table_copy(getmetatable(orig))) + else -- number, string, boolean, etc + copy = orig + end + return copy +end + -- Add an error message to a key/value table. -- Can accept a nil argument, and if is nil, will initialize the table. -- @param `errors` (Optional) Can be nil. Table to attach the error to. If nil, the table will be created. diff --git a/spec/integration/proxy/database_cache_spec.lua b/spec/integration/proxy/database_cache_spec.lua index ac7195d12c2..2930f1e612b 100644 --- a/spec/integration/proxy/database_cache_spec.lua +++ b/spec/integration/proxy/database_cache_spec.lua @@ -1,5 +1,6 @@ local spec_helper = require "spec.spec_helpers" local http_client = require "kong.tools.http_client" +local cache = require "kong.tools.database_cache" local env = spec_helper.get_env() diff --git a/spec/unit/statics_spec.lua b/spec/unit/statics_spec.lua index 0f698d6f600..b3beb560f05 100644 --- a/spec/unit/statics_spec.lua +++ b/spec/unit/statics_spec.lua @@ -146,6 +146,7 @@ nginx: | lua_code_cache on; lua_max_running_timers 4096; lua_max_pending_timers 16384; + lua_shared_dict locks 100k; lua_shared_dict cache {{memory_cache_size}}m; lua_socket_log_errors off; @@ -158,6 +159,8 @@ nginx: | end '; + init_worker_by_lua 'kong.exec_plugins_init_worker()'; + server { server_name _; listen {{proxy_port}}; diff --git a/spec/unit/stub_coverage_spec.lua b/spec/unit/stub_coverage_spec.lua index 8a45c1feeb2..7fa86a6bb65 100644 --- a/spec/unit/stub_coverage_spec.lua +++ b/spec/unit/stub_coverage_spec.lua @@ -6,7 +6,7 @@ local IO = require "kong.tools.io" -- Stub DAO for lapis controllers _G.dao = {} -local lua_sources = IO.retrieve_files("./kong", { exclude_dir_patterns = {"cli", "vendor", "filelog"}, file_pattern = ".lua" }) +local lua_sources = IO.retrieve_files("./kong", { exclude_dir_patterns = {"cli", "vendor", "filelog", "reports"}, file_pattern = ".lua" }) for _, source_link in ipairs(lua_sources) do dofile(source_link) diff --git a/spec/unit/tools/database_cache_spec.lua b/spec/unit/tools/database_cache_spec.lua index fadf579b308..2fad5ec8528 100644 --- a/spec/unit/tools/database_cache_spec.lua +++ b/spec/unit/tools/database_cache_spec.lua @@ -19,4 +19,8 @@ describe("Database cache", function() assert.are.equal("basicauth_credentials/username", cache.basicauth_credential_key("username")) end) + it("should return a valid requests cache key", function() + assert.are.equal("requests", cache.requests_key()) + end) + end)