From 1d269ee99fbb26f3e71e2634a980f98e7157fd44 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Tue, 30 Aug 2022 13:41:37 +0000 Subject: [PATCH 1/7] feat: Add ability to inject headers to otel traces --- apisix/plugins/opentelemetry.lua | 43 +++++++++++-- docs/en/latest/plugins/opentelemetry.md | 2 + t/plugin/opentelemetry2.t | 85 +++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 6 deletions(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index ea05b0a8025b..3babf80ca030 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -169,6 +169,13 @@ local schema = { type = "string", minLength = 1, } + }, + additional_header_attributes = { + type = "array", + items = { + type = "string", + minLength = 1, + } } } } @@ -273,6 +280,26 @@ local function create_tracer_obj(conf) end +local function inject_attributes(attributes, wanted_attributes, source) + for _, key in ipairs(wanted_attributes) do + local is_key_a_match = #key >= 2 and string.sub(key, -1, -1) == "*" + local prefix = string.sub(key, 0, -2) + local prefix_size = #prefix + local val = source[key] + if val then + core.table.insert(attributes, attr.string(key, val)) + end + if is_key_a_match then + for possible_key, value in pairs(source) do + if string.sub(possible_key, 0, prefix_size) == prefix then + core.table.insert(attributes, attr.string(possible_key, value)) + end + end + end + end +end + + function _M.rewrite(conf, api_ctx) local tracer, err = core.lrucache.plugin_ctx(lrucache, api_ctx, nil, create_tracer_obj, conf) if not tracer then @@ -286,13 +313,17 @@ function _M.rewrite(conf, api_ctx) attr.string("service", api_ctx.service_name), attr.string("route", api_ctx.route_name), } + if conf.additional_attributes then - for _, key in ipairs(conf.additional_attributes) do - local val = api_ctx.var[key] - if val then - core.table.insert(attributes, attr.string(key, val)) - end - end + inject_attributes(attributes, conf.additional_attributes, api_ctx.var) + end + + if conf.additional_header_attributes then + inject_attributes( + attributes, + conf.additional_header_attributes, + core.request.headers(api_ctx) + ) end local ctx = tracer:start(upstream_context, api_ctx.var.request_uri, { diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md index 257e018629c9..658f65a17ad9 100644 --- a/docs/en/latest/plugins/opentelemetry.md +++ b/docs/en/latest/plugins/opentelemetry.md @@ -46,6 +46,8 @@ The Plugin only supports binary-encoded [OLTP over HTTP](https://opentelemetry.i | sampler.options.root.options.fraction | number | False | 0 | [0, 1] | Root sampling probability for `trace_id_ratio`. | | additional_attributes | array[string] | False | | | Variables and its values which will be appended to the trace span. | | additional_attributes[0] | string | True | | | APISIX or Nginx variables. For example, `http_header` or `route_id`. | +| additional_header_attributes | array[string] | False | | | Variables and its values which will be appended to the trace span. | +| additional_header_attributes[0] | string | True | | | Request headers. For example, `x-my-header"` or `x-my-headers-*` to include all headers with the prefix `x-my-headers-`. | ### Configuring the collector diff --git a/t/plugin/opentelemetry2.t b/t/plugin/opentelemetry2.t index f173d125ba91..9b5be77776e7 100644 --- a/t/plugin/opentelemetry2.t +++ b/t/plugin/opentelemetry2.t @@ -142,3 +142,88 @@ plugin body_filter phase opentelemetry context current opentelemetry context current opentelemetry export span + + + +=== TEST 3: set additional_attributes with match +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "name": "route_name", + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + }, + "additional_header_attributes": [ + "x-my-header-*" + ] + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/attributes" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 4: opentelemetry expands headers +--- extra_init_by_lua + local otlp = require("opentelemetry.trace.exporter.otlp") + otlp.export_spans = function(self, spans) + if (#spans ~= 1) then + ngx.log(ngx.ERR, "unexpected spans length: ", #spans) + return + end + + local attributes_names = {} + local attributes = {} + local span = spans[1] + for _, attribute in ipairs(span.attributes) do + if attribute.key == "hostname" then + -- remove any randomness + goto skip + end + table.insert(attributes_names, attribute.key) + attributes[attribute.key] = attribute.value.string_value or "" + ::skip:: + end + table.sort(attributes_names) + for _, attribute in ipairs(attributes_names) do + ngx.log(ngx.INFO, "attribute " .. attribute .. ": " .. attributes[attribute]) + end + + ngx.log(ngx.INFO, "opentelemetry export span") + end +--- request +GET /attributes +--- more_headers +x-my-header-name: william +x-my-header-nick: bill +--- wait: 1 +--- error_code: 404 +--- grep_error_log eval +qr/attribute .+?:.[^,]*/ +--- grep_error_log_out +attribute route: route_name +attribute service: +attribute x-my-header-name: william +attribute x-my-header-nick: bill From 429c71884c28382fac1c826a7714894cc58a2fba Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Wed, 31 Aug 2022 10:41:07 +0000 Subject: [PATCH 2/7] chore: Reuse string.has_prefix form apisix.core --- apisix/plugins/opentelemetry.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index 3babf80ca030..6c9fa3d4ba6f 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -284,14 +284,13 @@ local function inject_attributes(attributes, wanted_attributes, source) for _, key in ipairs(wanted_attributes) do local is_key_a_match = #key >= 2 and string.sub(key, -1, -1) == "*" local prefix = string.sub(key, 0, -2) - local prefix_size = #prefix local val = source[key] if val then core.table.insert(attributes, attr.string(key, val)) end if is_key_a_match then for possible_key, value in pairs(source) do - if string.sub(possible_key, 0, prefix_size) == prefix then + if core.string.has_prefix(possible_key, prefix) then core.table.insert(attributes, attr.string(possible_key, value)) end end From 07c13ba916642f6144b6e8729050154b6714709d Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Wed, 31 Aug 2022 13:59:20 +0200 Subject: [PATCH 3/7] Apply suggestions from code review Co-authored-by: tzssangglass --- apisix/plugins/opentelemetry.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index 6c9fa3d4ba6f..02efa728510f 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -282,8 +282,8 @@ end local function inject_attributes(attributes, wanted_attributes, source) for _, key in ipairs(wanted_attributes) do - local is_key_a_match = #key >= 2 and string.sub(key, -1, -1) == "*" - local prefix = string.sub(key, 0, -2) + local is_key_a_match = #key >= 2 and key:sub(-1, -1) == "*" + local prefix = key:sub(0, -2) local val = source[key] if val then core.table.insert(attributes, attr.string(key, val)) From 45c6d4a14eab42a013859dfdca4f56d39a1b28c7 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Wed, 31 Aug 2022 13:21:58 +0000 Subject: [PATCH 4/7] chore: Change test output check to avoid editorconfig breaking --- t/plugin/opentelemetry2.t | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/t/plugin/opentelemetry2.t b/t/plugin/opentelemetry2.t index 9b5be77776e7..d0ceeecbbc80 100644 --- a/t/plugin/opentelemetry2.t +++ b/t/plugin/opentelemetry2.t @@ -208,7 +208,7 @@ passed end table.sort(attributes_names) for _, attribute in ipairs(attributes_names) do - ngx.log(ngx.INFO, "attribute " .. attribute .. ": " .. attributes[attribute]) + ngx.log(ngx.INFO, "attribute " .. attribute .. ": \"" .. attributes[attribute] .. "\"") end ngx.log(ngx.INFO, "opentelemetry export span") @@ -223,7 +223,7 @@ x-my-header-nick: bill --- grep_error_log eval qr/attribute .+?:.[^,]*/ --- grep_error_log_out -attribute route: route_name -attribute service: -attribute x-my-header-name: william -attribute x-my-header-nick: bill +attribute route: "route_name" +attribute service: "" +attribute x-my-header-name: "william" +attribute x-my-header-nick: "bill" From 1d91fd31268d48565e53a9e22881c3630067332b Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Fri, 2 Sep 2022 12:39:22 +0000 Subject: [PATCH 5/7] Apply code review suggestions --- apisix/plugins/opentelemetry.lua | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index 02efa728510f..aa54ba5aee77 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -280,20 +280,22 @@ local function create_tracer_obj(conf) end -local function inject_attributes(attributes, wanted_attributes, source) +local function inject_attributes(attributes, wanted_attributes, source, with_prefix) for _, key in ipairs(wanted_attributes) do - local is_key_a_match = #key >= 2 and key:sub(-1, -1) == "*" - local prefix = key:sub(0, -2) - local val = source[key] - if val then - core.table.insert(attributes, attr.string(key, val)) - end + local is_key_a_match = #key >= 2 and key:sub(-1, -1) == "*" and with_prefix + if is_key_a_match then + local prefix = key:sub(0, -2) for possible_key, value in pairs(source) do if core.string.has_prefix(possible_key, prefix) then core.table.insert(attributes, attr.string(possible_key, value)) end end + else + local val = source[key] + if val then + core.table.insert(attributes, attr.string(key, val)) + end end end end @@ -314,14 +316,15 @@ function _M.rewrite(conf, api_ctx) } if conf.additional_attributes then - inject_attributes(attributes, conf.additional_attributes, api_ctx.var) + inject_attributes(attributes, conf.additional_attributes, api_ctx.var, false) end if conf.additional_header_attributes then inject_attributes( attributes, conf.additional_header_attributes, - core.request.headers(api_ctx) + core.request.headers(api_ctx), + true ) end From 8266b2a442511f7e74932d3b189eff8e1ef1452b Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Mon, 5 Sep 2022 08:01:58 +0000 Subject: [PATCH 6/7] chore: Extract asterisk logic to a constant and use key:byte --- apisix/plugins/opentelemetry.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index aa54ba5aee77..6c8d687e6bd9 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -53,6 +53,7 @@ local lrucache = core.lrucache.new({ type = 'plugin', count = 128, ttl = 24 * 60 * 60, }) +local asterisk = string.byte("*", 1) local attr_schema = { type = "object", @@ -282,7 +283,7 @@ end local function inject_attributes(attributes, wanted_attributes, source, with_prefix) for _, key in ipairs(wanted_attributes) do - local is_key_a_match = #key >= 2 and key:sub(-1, -1) == "*" and with_prefix + local is_key_a_match = #key >= 2 and key:byte(-1) == asterisk and with_prefix if is_key_a_match then local prefix = key:sub(0, -2) From a19a27bf12599c3e8755324bac104708a801b428 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Mon, 5 Sep 2022 08:05:31 +0000 Subject: [PATCH 7/7] chore: Rename additional_header_attributes to additional_header_prefix_attributes --- apisix/plugins/opentelemetry.lua | 6 +++--- docs/en/latest/plugins/opentelemetry.md | 4 ++-- t/plugin/opentelemetry2.t | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index 6c8d687e6bd9..c3aa57421683 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -171,7 +171,7 @@ local schema = { minLength = 1, } }, - additional_header_attributes = { + additional_header_prefix_attributes = { type = "array", items = { type = "string", @@ -320,10 +320,10 @@ function _M.rewrite(conf, api_ctx) inject_attributes(attributes, conf.additional_attributes, api_ctx.var, false) end - if conf.additional_header_attributes then + if conf.additional_header_prefix_attributes then inject_attributes( attributes, - conf.additional_header_attributes, + conf.additional_header_prefix_attributes, core.request.headers(api_ctx), true ) diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md index 658f65a17ad9..10d6826b2dcf 100644 --- a/docs/en/latest/plugins/opentelemetry.md +++ b/docs/en/latest/plugins/opentelemetry.md @@ -46,8 +46,8 @@ The Plugin only supports binary-encoded [OLTP over HTTP](https://opentelemetry.i | sampler.options.root.options.fraction | number | False | 0 | [0, 1] | Root sampling probability for `trace_id_ratio`. | | additional_attributes | array[string] | False | | | Variables and its values which will be appended to the trace span. | | additional_attributes[0] | string | True | | | APISIX or Nginx variables. For example, `http_header` or `route_id`. | -| additional_header_attributes | array[string] | False | | | Variables and its values which will be appended to the trace span. | -| additional_header_attributes[0] | string | True | | | Request headers. For example, `x-my-header"` or `x-my-headers-*` to include all headers with the prefix `x-my-headers-`. | +| additional_header_prefix_attributes | array[string] | False | | | Headers or headers prefixes to be appended to the trace span's attributes. | +| additional_header_prefix_attributes[0]| string | True | | | Request headers. For example, `x-my-header"` or `x-my-headers-*` to include all headers with the prefix `x-my-headers-`. | ### Configuring the collector diff --git a/t/plugin/opentelemetry2.t b/t/plugin/opentelemetry2.t index d0ceeecbbc80..2495d8ef2adf 100644 --- a/t/plugin/opentelemetry2.t +++ b/t/plugin/opentelemetry2.t @@ -159,7 +159,7 @@ opentelemetry export span "sampler": { "name": "always_on" }, - "additional_header_attributes": [ + "additional_header_prefix_attributes": [ "x-my-header-*" ] }