diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index dc2901e4947b..4d61ef13d3ab 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -61,7 +61,7 @@ local attr_schema = { trace_id_source = { type = "string", enum = {"x-request-id", "random"}, - description = "alternate use x-request-id as trace id", + description = "the source of trace id", default = "random", }, resource = { @@ -307,11 +307,14 @@ function _M.access(conf, api_ctx) end -function _M.body_filter(conf, ctx) +function _M.body_filter(conf, api_ctx) if ngx.arg[2] then - local upstream_status = core.response.get_upstream_status(ctx) + local upstream_status = core.response.get_upstream_status(api_ctx) + local ctx = context:current(); + ctx:detach() + -- get span from current context - local span = context:current():span() + local span = ctx:span() if upstream_status and upstream_status >= 500 then span:set_status(span_status.error, "upstream response status: " .. upstream_status) @@ -322,4 +325,19 @@ function _M.body_filter(conf, ctx) end +function _M.log(conf, api_ctx) + local ctx = context:current(); + if ctx then + -- get span from current context + local span = ctx:span() + if upstream_status and upstream_status >= 500 then + span:set_status(span_status.error, + "upstream response status: " .. upstream_status) + end + + span:finish() + end +end + + return _M diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md index 9183e276a395..4f7d599cc381 100644 --- a/docs/en/latest/plugins/opentelemetry.md +++ b/docs/en/latest/plugins/opentelemetry.md @@ -47,8 +47,8 @@ Just support reporting in `HTTP` with `Content-Type=application/x-protobuf`, the | sampler.options.root.name | string | optional | always_off | ["always_on", "always_off", "trace_id_ratio"] | sampling strategy | sampler.options.root.options | object | optional | {fraction = 0} | | sampling strategy parameters | sampler.options.root.options.fraction | number | optional | 0 | [0, 1] | trace_id_ratio fraction -| additional_attributes | array[string] | optional | | | append to trace span attributes -| additional_attributes[0] | string | required | | | key of ctx.var +| additional_attributes | array[string] | optional | | | attributes (variable and its value) which will be appended to the trace span +| additional_attributes[0] | string | required | | | APISIX or Nginx variable, like `http_header` or `route_id` ## How To Enable @@ -94,7 +94,7 @@ We can set the collecting by specifying the configuration in `conf/config.yaml`. | Name | Type | Default | Description | | ------------ | ------ | -------- | ----------------------------------------------------- | -| trace_id_source | enum | random | alternate use x-request-id as trace id, valid value is `random` or `x-request-id`, if use `x-request-id`, please make sure it match regex pattern `[0-9a-f]{32}` | +| trace_id_source | enum | random | the source of trace id, the valid value is `random` or `x-request-id`. If `x-request-id` is set, the value of `x-request-id` request header will be used as trace id. Please make sure it match regex pattern `[0-9a-f]{32}` | | resource | object | | additional [resource](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/sdk.md) append to trace | | collector | object | {address = "127.0.0.1:4317", request_timeout = 3} | otlp collector | | collector.address | string | 127.0.0.1:4317 | collector address | diff --git a/docs/zh/latest/plugins/opentelemetry.md b/docs/zh/latest/plugins/opentelemetry.md index d0e3e98b6f67..fbf784e3095e 100644 --- a/docs/zh/latest/plugins/opentelemetry.md +++ b/docs/zh/latest/plugins/opentelemetry.md @@ -47,9 +47,8 @@ title: opentelemetry | sampler.options.root.name | string | 可选 | always_off | ["always_on", "always_off", "trace_id_ratio"] | 采样算法 | sampler.options.root.options | object | 可选 | {fraction = 0} | | 采样算法参数 | sampler.options.root.options.fraction | number | 可选 | 0 | [0, 1] | trace_id_ratio 采样算法的百分比 -| tags | array[object] | 可选 | | | 追加到 trace span 的属性 -| tags.position | string | 必须 | | ["http", "arg", "cookie"] | 变量的位置 -| tags.name | string | 必须 | | | 变量的名称 +| additional_attributes | array[string] | optional | | | 追加到 trace span 的额外属性（变量名为 key，变量值为 value） +| additional_attributes[0] | string | required | | | APISIX or Nginx 变量，例如 `http_header` or `route_id` ## 如何启用 diff --git a/t/plugin/opentelemetry.t b/t/plugin/opentelemetry.t index 0f54a12692a9..83a9dce8e9d7 100644 --- a/t/plugin/opentelemetry.t +++ b/t/plugin/opentelemetry.t @@ -20,7 +20,8 @@ use t::APISIX 'no_plan'; add_block_preprocessor(sub { my ($block) = @_; - my $extra_yaml_config = <<_EOC_; + if (!$block->extra_yaml_config) { + my $extra_yaml_config = <<_EOC_; plugins: - opentelemetry plugin_attr: @@ -29,24 +30,27 @@ plugin_attr: max_export_batch_size: 1 inactive_timeout: 0.5 _EOC_ + $block->set_value("extra_yaml_config", $extra_yaml_config); + } - $block->set_value("extra_yaml_config", $extra_yaml_config); - my $extra_init_by_lua = <<_EOC_; - -- mock exporter http client - local client = require("opentelemetry.trace.exporter.http_client") - client.do_request = function() - ngx.log(ngx.INFO, "opentelemetry export span") - end + if (!$block->extra_init_by_lua) { + my $extra_init_by_lua = <<_EOC_; +-- mock exporter http client +local client = require("opentelemetry.trace.exporter.http_client") +client.do_request = function() + ngx.log(ngx.INFO, "opentelemetry export span") +end _EOC_ - $block->set_value("extra_init_by_lua", $extra_init_by_lua); + $block->set_value("extra_init_by_lua", $extra_init_by_lua); + } if (!$block->request) { $block->set_value("request", "GET /t"); } - if (!$block->response_body) { + if (!defined $block->response_body) { $block->set_value("response_body", "passed\n"); } @@ -88,28 +92,6 @@ __DATA__ "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "always_on" - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -154,25 +136,6 @@ opentelemetry export span "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -218,28 +181,6 @@ qr/opentelemetry export span/ "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "trace_id_ratio" - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -288,31 +229,6 @@ qr/opentelemetry export span/ "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "trace_id_ratio", - "options": { - "fraction": 1.0 - } - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -360,28 +276,6 @@ opentelemetry export span "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "parent_base" - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -432,33 +326,6 @@ qr/opentelemetry export span/ "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "parent_base", - "options": { - "root": { - "name": "always_on" - } - } - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -511,33 +378,6 @@ opentelemetry export span "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "parent_base", - "options": { - "root": { - "name": "trace_id_ratio" - } - } - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -606,36 +446,6 @@ opentelemetry export span "type": "roundrobin" }, "uri": "/opentracing" - }]], - [[{ - "node": { - "value": { - "plugins": { - "opentelemetry": { - "sampler": { - "name": "parent_base", - "options": { - "root": { - "name": "trace_id_ratio", - "options": { - "fraction": 1.0 - } - } - } - } - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }, - "key": "/apisix/routes/1" - }, - "action": "set" }]] ) @@ -671,3 +481,225 @@ opentracing --- grep_error_log eval qr/opentelemetry export span/ --- grep_error_log_out + + + +=== TEST 19: set additional_attributes +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/services/1', + ngx.HTTP_PUT, + [[{ + "name": "service_name", + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "name": "route_name", + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + }, + "additional_attributes": [ + "http_user_agent", + "arg_foo", + "cookie_token", + "remote_addr" + ] + } + }, + "uri": "/opentracing", + "service_id": "1" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 20: trigger opentelemetry, test trace_id_source=x-request-id, custom resource, additional_attributes +--- extra_yaml_config +plugins: + - opentelemetry +plugin_attr: + opentelemetry: + trace_id_source: x-request-id + resource: + service.name: test + test_key: test_val + batch_span_processor: + max_export_batch_size: 1 + inactive_timeout: 0.5 +--- extra_init_by_lua + local core = require("apisix.core") + local otlp = require("opentelemetry.trace.exporter.otlp") + otlp.export_spans = function(self, spans) + if (#spans ~= 1) then + ngx.log(ngx.ERR, "unexpected spans length: ", #spans) + return + end + + local span = spans[1] + if span:context().trace_id ~= "01010101010101010101010101010101" then + ngx.log(ngx.ERR, "unexpected trace id: ", span:context().trace_id) + return + end + + if span.name ~= "/opentracing?foo=bar&a=b" then + ngx.log(ngx.ERR, "expect span name: /opentracing?foo=bar&a=b, but got ", span.name) + return + end + + local expected_resource_attrs = { + test_key = "test_val", + } + expected_resource_attrs["service.name"] = "test" + expected_resource_attrs["telemetry.sdk.language"] = "lua" + expected_resource_attrs["telemetry.sdk.name"] = "opentelemetry-lua" + expected_resource_attrs["telemetry.sdk.version"] = "0.1.1" + expected_resource_attrs["hostname"] = core.utils.gethostname() + local actual_resource_attrs = span.tracer.provider.resource:attributes() + if #actual_resource_attrs ~= 6 then + ngx.log(ngx.ERR, "expect len(actual_resource) = 6, but got ", #actual_resource_attrs) + return + end + for _, attr in ipairs(actual_resource_attrs) do + local expected_val = expected_resource_attrs[attr.key] + if not expected_val then + ngx.log(ngx.ERR, "unexpected resource attr key: ", attr.key) + return + end + if attr.value.string_value ~= expected_val then + ngx.log(ngx.ERR, "unexpected resource attr val: ", attr.value.string_value) + return + end + end + + local expected_attributes = { + service = "service_name", + route = "route_name", + http_user_agent = "test_nginx", + arg_foo = "bar", + cookie_token = "auth_token", + remote_addr = "127.0.0.1", + } + if #span.attributes ~= 6 then + ngx.log(ngx.ERR, "expect len(span.attributes) = 6, but got ", #span.attributes) + return + end + for _, attr in ipairs(span.attributes) do + local expected_val = expected_attributes[attr.key] + if not expected_val then + ngx.log(ngx.ERR, "unexpected attr key: ", attr.key) + return + end + if attr.value.string_value ~= expected_val then + ngx.log(ngx.ERR, "unexpected attr val: ", attr.value.string_value) + return + end + end + + ngx.log(ngx.INFO, "opentelemetry export span") + end +--- request +GET /opentracing?foo=bar&a=b +--- more_headers +X-Request-Id: 01010101010101010101010101010101 +User-Agent: test_nginx +Cookie: token=auth_token; +--- response_body +opentracing +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span + + + +=== TEST 21: create route for /specific_status +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "name": "route_name", + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + } + } + }, + "uri": "/specific_status", + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 22: 500 status, test span.status +--- extra_init_by_lua + local otlp = require("opentelemetry.trace.exporter.otlp") + otlp.export_spans = function(self, spans) + if (#spans ~= 1) then + ngx.log(ngx.ERR, "unexpected spans length: ", #spans) + return + end + + local span = spans[1] + if span.status.code ~= 2 then + ngx.log(ngx.ERR, "unexpected status.code: ", span.status.code) + end + if span.status.message ~= "upstream response status: 500" then + ngx.log(ngx.ERR, "unexpected status.message: ", span.status.message) + end + + ngx.log(ngx.INFO, "opentelemetry export span") + end +--- request +GET /specific_status +--- more_headers +X-Test-Upstream-Status: 500 +--- error_code: 500 +--- response_body +upstream status: 500 +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span