From 6289affc8f4574147c4f6181dcdcdfcc1fa19c1b Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Tue, 21 Apr 2015 18:51:44 +0200 Subject: [PATCH] feat(APIAnalytics) ALF serializer skeleton + tests - almost complete ALF serialization - basic data push to apianalytics - entries can be queued in the serializer, and flushed anytime - serializer can output itself in JSON, being a valid ALF object - more tests, with fixtures - introduce serializers for logging plugins Only a temporary solution before each serializer to be a module of its own (maybe?) and each logging plugin to be independent from Kong, and require serializers as dependencies. Following the discussion in #86 --- kong-0.3.0-1.rockspec | 7 +- kong.yml | 1 + kong/kong.lua | 47 ++---- kong/plugins/apianalytics/handler.lua | 58 ++++++++ kong/plugins/apianalytics/schema.lua | 1 + kong/plugins/filelog/handler.lua | 7 +- kong/plugins/filelog/log.lua | 10 +- kong/plugins/log_serializers/alf.lua | 136 ++++++++++++++++++ kong/plugins/log_serializers/basic.lua | 24 ++++ kong/plugins/tcplog/handler.lua | 7 +- kong/plugins/tcplog/log.lua | 4 +- kong/plugins/udplog/handler.lua | 7 +- kong/plugins/udplog/log.lua | 4 +- kong/tools/http_client.lua | 9 +- .../apianalytics/alf_serializer_spec.lua | 53 +++++++ .../apianalytics/fixtures/requests.lua | 91 ++++++++++++ spec/unit/statics_spec.lua | 1 + 17 files changed, 412 insertions(+), 55 deletions(-) create mode 100644 kong/plugins/apianalytics/handler.lua create mode 100644 kong/plugins/apianalytics/schema.lua create mode 100644 kong/plugins/log_serializers/alf.lua create mode 100644 kong/plugins/log_serializers/basic.lua create mode 100644 spec/plugins/apianalytics/alf_serializer_spec.lua create mode 100644 spec/plugins/apianalytics/fixtures/requests.lua diff --git a/kong-0.3.0-1.rockspec b/kong-0.3.0-1.rockspec index bb1bb9cbf26..d5b9618d191 100644 --- a/kong-0.3.0-1.rockspec +++ b/kong-0.3.0-1.rockspec @@ -98,6 +98,9 @@ build = { ["kong.plugins.keyauth.schema"] = "kong/plugins/keyauth/schema.lua", ["kong.plugins.keyauth.api"] = "kong/plugins/keyauth/api.lua", + ["kong.plugins.log_serializers.basic"] = "kong/plugins/log_serializers/basic.lua", + ["kong.plugins.log_serializers.alf"] = "kong/plugins/log_serializers/alf.lua", + ["kong.plugins.tcplog.handler"] = "kong/plugins/tcplog/handler.lua", ["kong.plugins.tcplog.log"] = "kong/plugins/tcplog/log.lua", ["kong.plugins.tcplog.schema"] = "kong/plugins/tcplog/schema.lua", @@ -111,10 +114,12 @@ build = { ["kong.plugins.httplog.schema"] = "kong/plugins/httplog/schema.lua", ["kong.plugins.filelog.handler"] = "kong/plugins/filelog/handler.lua", - ["kong.plugins.filelog.log"] = "kong/plugins/filelog/log.lua", ["kong.plugins.filelog.schema"] = "kong/plugins/filelog/schema.lua", ["kong.plugins.filelog.fd_util"] = "kong/plugins/filelog/fd_util.lua", + ["kong.plugins.apianalytics.handler"] = "kong/plugins/apianalytics/handler.lua", + ["kong.plugins.apianalytics.schema"] = "kong/plugins/apianalytics/schema.lua", + ["kong.plugins.ratelimiting.handler"] = "kong/plugins/ratelimiting/handler.lua", ["kong.plugins.ratelimiting.access"] = "kong/plugins/ratelimiting/access.lua", ["kong.plugins.ratelimiting.schema"] = "kong/plugins/ratelimiting/schema.lua", diff --git a/kong.yml b/kong.yml index 2af0d253ca9..a47969fbc76 100644 --- a/kong.yml +++ b/kong.yml @@ -12,6 +12,7 @@ plugins_available: - request_transformer - response_transformer - requestsizelimiting + - analytics ## The Kong working directory ## (Make sure you have read and write permissions) diff --git a/kong/kong.lua b/kong/kong.lua index 3730e0d04d5..cb2d5b60887 100644 --- a/kong/kong.lua +++ b/kong/kong.lua @@ -186,9 +186,9 @@ function _M.exec_plugins_access() end end - local conf = ngx.ctx.plugin_conf[plugin.name] - if not ngx.ctx.stop_phases and (plugin.resolver or conf) then - plugin.handler:access(conf and conf.value or nil) + local plugin_conf = ngx.ctx.plugin_conf[plugin.name] + if not ngx.ctx.stop_phases and (plugin.resolver or plugin_conf) then + plugin.handler:access(plugin_conf and plugin_conf.value or nil) end end @@ -211,9 +211,9 @@ function _M.exec_plugins_header_filter() ngx.header["Via"] = constants.NAME.."/"..constants.VERSION for _, plugin in ipairs(plugins) do - local conf = ngx.ctx.plugin_conf[plugin.name] - if conf then - plugin.handler:header_filter(conf.value) + local plugin_conf = ngx.ctx.plugin_conf[plugin.name] + if plugin_conf then + plugin.handler:header_filter(plugin_conf.value) end end end @@ -223,9 +223,9 @@ end function _M.exec_plugins_body_filter() if not ngx.ctx.stop_phases then for _, plugin in ipairs(plugins) do - local conf = ngx.ctx.plugin_conf[plugin.name] - if conf then - plugin.handler:body_filter(conf.value) + local plugin_conf = ngx.ctx.plugin_conf[plugin.name] + if plugin_conf then + plugin.handler:body_filter(plugin_conf.value) end end end @@ -234,33 +234,10 @@ end -- Calls log() on every loaded plugin function _M.exec_plugins_log() if not ngx.ctx.stop_phases then - -- Creating the log variable that will be serialized - local message = { - request = { - uri = ngx.var.request_uri, - request_uri = ngx.var.scheme.."://"..ngx.var.host..":"..ngx.var.server_port..ngx.var.request_uri, - querystring = ngx.req.get_uri_args(), -- parameters, as a table - method = ngx.req.get_method(), -- http method - headers = ngx.req.get_headers(), - size = ngx.var.request_length - }, - response = { - status = ngx.status, - headers = ngx.resp.get_headers(), - size = ngx.var.bytes_sent - }, - authenticated_entity = ngx.ctx.authenticated_entity, - api = ngx.ctx.api, - client_ip = ngx.var.remote_addr, - started_at = ngx.req.start_time() * 1000 - } - - ngx.ctx.log_message = message - for _, plugin in ipairs(plugins) do - local conf = ngx.ctx.plugin_conf[plugin.name] - if conf then - plugin.handler:log(conf.value) + local plugin_conf = ngx.ctx.plugin_conf[plugin.name] + if plugin_conf then + plugin.handler:log(plugin_conf.value) end end end diff --git a/kong/plugins/apianalytics/handler.lua b/kong/plugins/apianalytics/handler.lua new file mode 100644 index 00000000000..33b282e96c9 --- /dev/null +++ b/kong/plugins/apianalytics/handler.lua @@ -0,0 +1,58 @@ +local http = require "socket.http" +local ltn12 = require "ltn12" +local BasePlugin = require "kong.plugins.base_plugin" +local ALFSerializer = require "kong.plugins.log_serializers.alf" + +local http_client = require "kong.tools.http_client" + +--local SERVER_URL = "http://socket.apianalytics.com/" +local SERVER_URL = "http://localhost:58000/alf_1.0.0" + +local APIAnalyticsHandler = BasePlugin:extend() + +function APIAnalyticsHandler:new() + APIAnalyticsHandler.super.new(self, "apianalytics") +end + +function APIAnalyticsHandler:access(conf) + APIAnalyticsHandler.super.access(self) + + ngx.req.read_body() + ngx.ctx.req_body = ngx.req.get_body_data() + ngx.ctx.res_body = "" +end + +function APIAnalyticsHandler:body_filter(conf) + APIAnalyticsHandler.super.body_filter(self) + + -- concatenate response chunks for response.content.text + local chunk = ngx.arg[1] + ngx.ctx.res_body = ngx.ctx.res_body..chunk +end + +function APIAnalyticsHandler:log(conf) + APIAnalyticsHandler.super.log(self) + + ALFSerializer:add_entry(ngx) + + -- if queue is full + local message = ALFSerializer:to_json_string("54d2b98ee0d5076065fd6f93") + print("MESSAGE: "..message) + + -- TODO: use the cosocket API + local response, status, headers = http_client.post(SERVER_URL, message, + { + ["content-length"] = string.len(message), + ["content-type"] = "application/json" + }) + + print("STATUS: "..status) + if status ~= 200 then + ngx.log(ngx.ERR, "Could not send entry to "..SERVER_URL) + print("RESPONSE IS: "..response) + end + + -- todo: flush +end + +return APIAnalyticsHandler diff --git a/kong/plugins/apianalytics/schema.lua b/kong/plugins/apianalytics/schema.lua new file mode 100644 index 00000000000..a564707544f --- /dev/null +++ b/kong/plugins/apianalytics/schema.lua @@ -0,0 +1 @@ +return {} diff --git a/kong/plugins/filelog/handler.lua b/kong/plugins/filelog/handler.lua index 70ce61cd6d9..cdf3c74f311 100644 --- a/kong/plugins/filelog/handler.lua +++ b/kong/plugins/filelog/handler.lua @@ -1,7 +1,8 @@ -- Copyright (C) Mashape, Inc. +local basic_serializer = require "kong.plugins.log_serializers.basic" local BasePlugin = require "kong.plugins.base_plugin" -local log = require "kong.plugins.filelog.log" +local cjson = require "cjson" local FileLogHandler = BasePlugin:extend() @@ -11,7 +12,9 @@ end function FileLogHandler:log(conf) FileLogHandler.super.log(self) - log.execute(conf) + + local message = basic_serializer.serialize(ngx) + ngx.log(ngx.INFO, cjson.encode(message)) end return FileLogHandler diff --git a/kong/plugins/filelog/log.lua b/kong/plugins/filelog/log.lua index 55464c580cb..577abfa36ff 100644 --- a/kong/plugins/filelog/log.lua +++ b/kong/plugins/filelog/log.lua @@ -1,7 +1,9 @@ -- Copyright (C) Mashape, Inc. -local cjson = require "cjson" + local ffi = require "ffi" +local cjson = require "cjson" local fd_util = require "kong.plugins.filelog.fd_util" +local basic_serializer = require "kong.plugins.log_serializers.basic" ffi.cdef[[ typedef struct { @@ -24,7 +26,7 @@ int fprintf(FILE *stream, const char *format, ...); -- @param `conf` Configuration table, holds http endpoint details -- @param `message` Message to be logged local function log(premature, conf, message) - local message = cjson.encode(message).."\n" + message = cjson.encode(message).."\n" local f = fd_util.get_fd() if not f then @@ -39,7 +41,9 @@ end local _M = {} function _M.execute(conf) - local ok, err = ngx.timer.at(0, log, conf, ngx.ctx.log_message) + local message = basic_serializer.serialize(ngx) + + local ok, err = ngx.timer.at(0, log, conf, message) if not ok then ngx.log(ngx.ERR, "[filelog] failed to create timer: ", err) end diff --git a/kong/plugins/log_serializers/alf.lua b/kong/plugins/log_serializers/alf.lua new file mode 100644 index 00000000000..389cb2500d2 --- /dev/null +++ b/kong/plugins/log_serializers/alf.lua @@ -0,0 +1,136 @@ +-- ALF serializer module. +-- ALF is the format supported by API Analytics (http://apianalytics.com) +-- +-- - ALF specifications: https://github.com/Mashape/api-log-format +-- - Nginx lua module documentation: http://wiki.nginx.org/HttpLuaModule +-- - ngx_http_core_module: http://wiki.nginx.org/HttpCoreModule#.24http_HEADER + +local EMPTY_ARRAY_PLACEHOLDER = "__empty_array_placeholder__" + +local alf_mt = {} +alf_mt.__index = alf_mt + +local ALF = { + version = "1.0.0", + serviceToken = "", + har = { + log = { + version = "1.2", + creator = { + name = "kong-api-analytics-plugin", + version = "0.1" + }, + entries = {} + } + } +} + +-- Transform a key/value lua table into an array of elements with `name`, `value` +-- Since Lua won't recognize {} as an empty array but an empty object, we need to force it +-- to be an array, hence we will do "[__empty_array_placeholder__]". +-- Then once the ALF will be stringified, we will remove the placeholder so the only left element will be "[]". +-- @param hash key/value dictionary to serialize +-- @return an array, or nil +local function dic_to_array(hash) + local arr = {} + for k, v in pairs(hash) do + table.insert(arr, { name = k, value = v }) + end + + if #arr > 0 then + return arr + else + return {EMPTY_ARRAY_PLACEHOLDER} + end +end + +-- Serialize into one ALF entry +-- For performance reasons, it tries to use the NGINX Lua API +-- instead of ngx_http_core_module when possible. +function alf_mt:serialize_entry(ngx) + -- Extracted data + local req_headers = ngx.req.get_headers() + local res_headers = ngx.resp.get_headers() + + local req_body = ngx.ctx.req_body + local res_body = ngx.ctx.res_body + + -- ALF format + local alf_req_mimeType = req_headers["Content-Type"] and req_headers["Content-Type"] or "application/octet-stream" + local alf_res_mimeType = res_headers["Content-Type"] and res_headers["Content-Type"] or "application/octet-stream" + local alf_req_bodySize = req_headers["Content-Length"] and req_headers["Content-Length"] or 0 + local alf_res_bodySize = res_headers["Content-Length"] and res_headers["Content-Length"] or 0 + + return { + startedDateTime = os.date("!%Y-%m-%dT%TZ", ngx.req.start_time()), + clientIPAddress = ngx.var.remote_addr, + time = 3, + -- REQUEST + request = { + method = ngx.req.get_method(), + url = ngx.var.scheme.."://"..ngx.var.host..ngx.var.uri, + httpVersion = "HTTP/"..ngx.req.http_version(), + queryString = dic_to_array(ngx.req.get_uri_args()), + headers = dic_to_array(req_headers), + headersSize = 10, + cookies = {EMPTY_ARRAY_PLACEHOLDER}, + bodySize = tonumber(alf_req_bodySize), + content = { + size = tonumber(ngx.var.request_length), + mimeType = alf_req_mimeType, + text = req_body and req_body or "" + } + }, + -- RESPONSE + response = { + status = ngx.status, + statusText = "", + httpVersion = "", + headers = dic_to_array(res_headers), + headersSize = 10, + cookies = {EMPTY_ARRAY_PLACEHOLDER}, + bodySize = tonumber(alf_res_bodySize), + redirectURL = "", + content = { + size = tonumber(ngx.var.bytes_sent), + mimeType = alf_res_mimeType, + text = res_body and res_body or "" + } + }, + cache = {}, + -- TIMINGS + timings = { + send = 1, + wait = 1, + receive = 1, + blocked = 0, + connect = 0, + dns = 0, + ssl = 0 + } + } -- end of entry +end + +function alf_mt:add_entry(ngx) + table.insert(self.har.log.entries, self:serialize_entry(ngx)) +end + +function alf_mt:to_json_string(token) + if not token then + error("API Analytics serviceToken required", 2) + end + + local cjson = require "cjson" + + -- inject token + self.serviceToken = token + + local str = cjson.encode(self) + return str:gsub("\""..EMPTY_ARRAY_PLACEHOLDER.."\"", ""):gsub("\\/", "/") +end + +function alf_mt:flush_entries() + self.har.log.entries = {} +end + +return setmetatable(ALF, alf_mt) diff --git a/kong/plugins/log_serializers/basic.lua b/kong/plugins/log_serializers/basic.lua new file mode 100644 index 00000000000..d4c84036061 --- /dev/null +++ b/kong/plugins/log_serializers/basic.lua @@ -0,0 +1,24 @@ +local _M = {} + +function _M.serialize(ngx) + return { + uri = ngx.var.request_uri, + request_uri = ngx.var.scheme.."://"..ngx.var.host..":"..ngx.var.server_port..ngx.var.request_uri, + querystring = ngx.req.get_uri_args(), -- parameters, as a table + method = ngx.req.get_method(), + headers = ngx.req.get_headers(), + size = ngx.var.request_length + }, + response = { + status = ngx.status, + headers = ngx.resp.get_headers(), + size = ngx.var.bytes_sent + }, + authenticated_entity = ngx.ctx.authenticated_entity, + api = ngx.ctx.api, + client_ip = ngx.var.remote_addr, + started_at = ngx.req.start_time() * 1000 + } +end + +return _M diff --git a/kong/plugins/tcplog/handler.lua b/kong/plugins/tcplog/handler.lua index febfedbd5c7..17f1f47bc38 100644 --- a/kong/plugins/tcplog/handler.lua +++ b/kong/plugins/tcplog/handler.lua @@ -1,5 +1,6 @@ -local BasePlugin = require "kong.plugins.base_plugin" local log = require "kong.plugins.tcplog.log" +local BasePlugin = require "kong.plugins.base_plugin" +local basic_serializer = require "kong.plugins.log_serializers.basic" local TcpLogHandler = BasePlugin:extend() @@ -9,7 +10,9 @@ end function TcpLogHandler:log(conf) TcpLogHandler.super.log(self) - log.execute(conf) + + local message = basic_serializer.serialize(ngx) + log.execute(conf, message) end return TcpLogHandler diff --git a/kong/plugins/tcplog/log.lua b/kong/plugins/tcplog/log.lua index 39112e2da00..1aa19b1bf5d 100644 --- a/kong/plugins/tcplog/log.lua +++ b/kong/plugins/tcplog/log.lua @@ -30,8 +30,8 @@ local function log(premature, conf, message) end end -function _M.execute(conf) - local ok, err = ngx.timer.at(0, log, conf, ngx.ctx.log_message) +function _M.execute(conf, message) + local ok, err = ngx.timer.at(0, log, conf, message) if not ok then ngx.log(ngx.ERR, "[tcplog] failed to create timer: ", err) end diff --git a/kong/plugins/udplog/handler.lua b/kong/plugins/udplog/handler.lua index be1fd5bad75..c285c0abb13 100644 --- a/kong/plugins/udplog/handler.lua +++ b/kong/plugins/udplog/handler.lua @@ -1,7 +1,8 @@ -- Copyright (C) Mashape, Inc. -local BasePlugin = require "kong.plugins.base_plugin" +local basic_serializer = require "kong.plugins.log_serializers.basic" local log = require "kong.plugins.udplog.log" +local BasePlugin = require "kong.plugins.base_plugin" local UdpLogHandler = BasePlugin:extend() @@ -11,7 +12,9 @@ end function UdpLogHandler:log(conf) UdpLogHandler.super.log(self) - log.execute(conf) + + local message = basic_serializer.serialize(ngx) + log.execute(conf, message) end return UdpLogHandler diff --git a/kong/plugins/udplog/log.lua b/kong/plugins/udplog/log.lua index 651417473c1..94503c7939f 100644 --- a/kong/plugins/udplog/log.lua +++ b/kong/plugins/udplog/log.lua @@ -28,8 +28,8 @@ local function log(premature, conf, message) end end -function _M.execute(conf) - local ok, err = ngx.timer.at(0, log, conf, ngx.ctx.log_message) +function _M.execute(conf, message) + local ok, err = ngx.timer.at(0, log, conf, message) if not ok then ngx.log(ngx.ERR, "failed to create timer: ", err) end diff --git a/kong/tools/http_client.lua b/kong/tools/http_client.lua index 936279e0d63..89228166a95 100644 --- a/kong/tools/http_client.lua +++ b/kong/tools/http_client.lua @@ -26,13 +26,10 @@ local function http_call(options) options.protocol = "tlsv1" options.mode = "client" options.options = "all" - - local _, code, headers = https.request(options) - return resp[1], code, headers - else - local _, code, headers = http.request(options) - return resp[1], code, headers end + + local _, code, headers = http.request(options) + return resp[1], code, headers end local function with_body(method) diff --git a/spec/plugins/apianalytics/alf_serializer_spec.lua b/spec/plugins/apianalytics/alf_serializer_spec.lua new file mode 100644 index 00000000000..dceaa2b3202 --- /dev/null +++ b/spec/plugins/apianalytics/alf_serializer_spec.lua @@ -0,0 +1,53 @@ +local ALFSerializer = require "kong.plugins.log_serializers.alf" +local fixtures = require "spec.plugins.apianalytics.fixtures.requests" + +describe("ALF serializer", function() + + describe("#serialize_entry()", function() + + it("should serialize an ngx GET request/response", function() + local entry = ALFSerializer:serialize_entry(fixtures.GET.NGX_STUB) + assert.are.same(fixtures.GET.ENTRY, entry) + end) + + end) + + describe("#add_entry()", function() + + it("should add the entry to the serializer entries property", function() + ALFSerializer:add_entry(fixtures.GET.NGX_STUB) + assert.are.same(1, table.getn(ALFSerializer.har.log.entries)) + assert.are.same(fixtures.GET.ENTRY, ALFSerializer.har.log.entries[1]) + + ALFSerializer:add_entry(fixtures.GET.NGX_STUB) + assert.are.same(2, table.getn(ALFSerializer.har.log.entries)) + assert.are.same(fixtures.GET.ENTRY, ALFSerializer.har.log.entries[2]) + end) + + end) + + describe("#to_json_string()", function() + + it("should throw an error if no token was given", function() + assert.has_error(function() ALFSerializer:to_json_string() end, + "API Analytics serviceToken required") + end) + + it("should return a JSON string", function() + local json_str = ALFSerializer:to_json_string("stub_service_token") + assert.are.same("string", type(json_str)) + end) + + end) + + describe("#flush_entries()", function() + + it("should remove any existing entry", function() + ALFSerializer:flush_entries() + assert.are.same(0, table.getn(ALFSerializer.har.log.entries)) + end) + + end) + + -- TODO: tests empty queryString (empty array) both JSON + Lua formats +end) diff --git a/spec/plugins/apianalytics/fixtures/requests.lua b/spec/plugins/apianalytics/fixtures/requests.lua new file mode 100644 index 00000000000..0e8b0d06db1 --- /dev/null +++ b/spec/plugins/apianalytics/fixtures/requests.lua @@ -0,0 +1,91 @@ +return { + ["GET"] = { + ["NGX_STUB"] = { + req = { + get_method = function() return "GET" end, + http_version = function() return 1.1 end, + get_headers = function() return {["Accept"]="/*/",["Host"]="mockbin.com"} end, + get_uri_args = function() return {["hello"]="world",["foo"]="bar"} end, + start_time = function() return 1429723321.026 end + }, + resp = { + get_headers = function() return {["Connection"]="close",["Content-Type"]="application/json",["Content-Length"]="934"} end + }, + status = 200, + var = { + scheme = "http", + host = "mockbin.com", + uri = "/request", + request_length = 123, + bytes_sent = 934, + remote_addr = "127.0.0.1" + }, + ctx = { + req_body = "request body", + res_body = "response body" + } + }, + ["ENTRY"] = { + clientIPAddress = "127.0.0.1", + request = { + bodySize = 0, + content = { + mimeType = "application/octet-stream", + size = 123, + text = "" + }, + headers = { { + name = "Accept", + value = "/*/" + }, { + name = "Host", + value = "mockbin.com" + } }, + headersSize = 10, + httpVersion = "HTTP/1.1", + method = "GET", + queryString = { { + name = "foo", + value = "bar" + }, { + name = "hello", + value = "world" + } }, + url = "http://mockbin.com/request" + }, + response = { + bodySize = 934, + content = { + mimeType = "application/json", + size = 934, + text = "" + }, + headers = { { + name = "Content-Length", + value = "934" + }, { + name = "Content-Type", + value = "application/json" + }, { + name = "Connection", + value = "close" + } }, + headersSize = 10, + httpVersion = "", + status = 200, + statusText = "" + }, + startedDateTime = "2015-04-22T17:22:01Z", + time = 3, + timings = { + blocked = 0, + connect = 0, + dns = 0, + receive = 1, + send = 1, + ssl = 0, + wait = 1 + } + } + } +} \ No newline at end of file diff --git a/spec/unit/statics_spec.lua b/spec/unit/statics_spec.lua index 1919a506482..aef0b189fe7 100644 --- a/spec/unit/statics_spec.lua +++ b/spec/unit/statics_spec.lua @@ -52,6 +52,7 @@ plugins_available: - request_transformer - response_transformer - requestsizelimiting + - analytics ## The Kong working directory ## (Make sure you have read and write permissions)