diff --git a/CHANGELOG.md b/CHANGELOG.md index 20cdb6cb5f..0963d694d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1116](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1116)) - fixed typo in `system.network.io` metric configuration ([#1135](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1135)) - +- Fix keys() in class ASGIGetter so it returns the HTTP header keys instead of a list of available request data. + ([#1172](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1172)) +- Use resp.text instead of resp.body for Falcon 3 to avoid a deprecation warning. + ([#1172](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1172)) +- Make ASGIGetter.get() compare all keys in a case insensitive manner. + ([#1172](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1172)) ### Added - `opentelemetry-instrumentation-aiohttp-client` Add support for optional custom trace_configs argument. @@ -55,6 +60,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1110](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1110)) - Integrated sqlcommenter plugin into opentelemetry-instrumentation-django ([#896](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/896)) +- Add support for regular expression matching of HTTP headers. + ([#1172](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1172)) +- Add support for sanitizing HTTP header values. + ([#1172](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1172)) ## [1.12.0rc1-0.31b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.12.0rc1-0.31b0) - 2022-05-17 diff --git a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py index d2e42450a0..0722425d05 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py @@ -15,8 +15,7 @@ """ The opentelemetry-instrumentation-asgi package provides an ASGI middleware that can be used -on any ASGI framework (such as Django-channels / Quart) to track requests -timing through OpenTelemetry. +on any ASGI framework (such as Django-channels / Quart) to track request timing through OpenTelemetry. Usage (Quart) ------------- @@ -71,9 +70,14 @@ async def hello(): Request/Response hooks ********************** -Utilize request/reponse hooks to execute custom logic to be performed before/after performing a request. The server request hook takes in a server span and ASGI -scope object for every incoming request. The client request hook is called with the internal span and an ASGI scope which is sent as a dictionary for when the method recieve is called. -The client response hook is called with the internal span and an ASGI event which is sent as a dictionary for when the method send is called. +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. + +- The server request hook is passed a server span and ASGI scope object for every incoming request. +- The client request hook is called with the internal span and an ASGI scope when the method ``receive`` is called. +- The client response hook is called with the internal span and an ASGI event when the method ``send`` is called. + +For example, .. code-block:: python @@ -93,59 +97,99 @@ def client_response_hook(span: Span, message: dict): Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in ASGI are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" + +Would match all request headers that start with ``Accept`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in ASGI are case insensitive. So, giving header name as ``CUStom-Header`` in environment variable will be able capture header with name ``custom-header``. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in ASGI are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in ASGI are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Would match all response headers that start with ``Content`` and ``X-``. -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- """ +import re import typing import urllib from functools import wraps @@ -167,8 +211,10 @@ def client_response_hook(span: Span, message: dict): from opentelemetry.trace import Span, set_span_in_context from opentelemetry.trace.status import Status, StatusCode from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, + SanitizeValue, get_custom_headers, normalise_request_header_name, normalise_response_header_name, @@ -198,19 +244,21 @@ def get( if not headers: return None - # asgi header keys are in lower case + # ASGI header keys are in lower case key = key.lower() decoded = [ _value.decode("utf8") for (_key, _value) in headers - if _key.decode("utf8") == key + if _key.decode("utf8").lower() == key ] if not decoded: return None return decoded def keys(self, carrier: dict) -> typing.List[str]: - return list(carrier.keys()) + return [ + _key.decode("utf8") for (_key, _value) in carrier.get("headers") + ] asgi_getter = ASGIGetter() @@ -286,15 +334,37 @@ def collect_custom_request_headers_attributes(scope): Refer specification https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-request-and-response-headers""" attributes = {} - custom_request_headers = get_custom_headers( + + sanitized_fields = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + + s = SanitizeValue(sanitized_fields) + + custom_request_headers_name = get_custom_headers( OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST ) - for header in custom_request_headers: - values = asgi_getter.get(scope, header) - if values: - key = normalise_request_header_name(header) - attributes.setdefault(key, []).extend(values) + if custom_request_headers_name: + custom_request_headers_regex_compiled = re.compile( + "|".join("^" + i + "$" for i in custom_request_headers_name), + re.IGNORECASE, + ) + + for header_name in list( + filter( + custom_request_headers_regex_compiled.match, + asgi_getter.keys(scope), + ) + ): + header_values = asgi_getter.get(scope, header_name.lower()) + if header_values: + key = normalise_request_header_name(header_name.lower()) + attributes[key] = [ + s.sanitize_header_value( + header=header_name, value=header_values[0] + ) + ] return attributes @@ -303,15 +373,37 @@ def collect_custom_response_headers_attributes(message): """returns custom HTTP response headers to be added into SERVER span as span attributes Refer specification https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-request-and-response-headers""" attributes = {} - custom_response_headers = get_custom_headers( + + sanitized_fields = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + + s = SanitizeValue(sanitized_fields) + + custom_response_headers_name = get_custom_headers( OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE ) - for header in custom_response_headers: - values = asgi_getter.get(message, header) - if values: - key = normalise_response_header_name(header) - attributes.setdefault(key, []).extend(values) + if custom_response_headers_name: + custom_response_headers_regex_compiled = re.compile( + "|".join("^" + i + "$" for i in custom_response_headers_name), + re.IGNORECASE, + ) + + for header_name in list( + filter( + custom_response_headers_regex_compiled.match, + asgi_getter.keys(message), + ) + ): + header_values = asgi_getter.get(message, header_name.lower()) + if header_values: + key = normalise_response_header_name(header_name.lower()) + attributes[key] = [ + s.sanitize_header_value( + header=header_name, value=header_values[0] + ) + ] return attributes @@ -349,7 +441,7 @@ def set_status_code(span, status_code): def get_default_span_details(scope: dict) -> Tuple[str, dict]: """Default implementation for get_default_span_details Args: - scope: the asgi scope dictionary + scope: the ASGI scope dictionary Returns: a tuple of the span name, and any attributes to attach to the span. """ @@ -406,7 +498,7 @@ async def __call__(self, scope, receive, send): """The ASGI application Args: - scope: A ASGI environment. + scope: An ASGI environment. receive: An awaitable callable yielding dictionaries send: An awaitable callable taking a single dictionary as argument. """ diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py index 3a1e8424a8..eca9989df1 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py @@ -32,6 +32,7 @@ from opentelemetry.test.test_base import TestBase from opentelemetry.trace import SpanKind, format_span_id, format_trace_id from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, ) @@ -78,6 +79,15 @@ async def http_app_with_custom_headers(scope, receive, send): (b"Content-Type", b"text/plain"), (b"custom-test-header-1", b"test-header-value-1"), (b"custom-test-header-2", b"test-header-value-2"), + ( + b"my-custom-regex-header-1", + b"my-custom-regex-value-1,my-custom-regex-value-2", + ), + ( + b"My-Custom-Regex-Header-2", + b"my-custom-regex-value-3,my-custom-regex-value-4", + ), + (b"my-secret-header", b"my-secret-value"), ], } ) @@ -95,6 +105,15 @@ async def websocket_app_with_custom_headers(scope, receive, send): "headers": [ (b"custom-test-header-1", b"test-header-value-1"), (b"custom-test-header-2", b"test-header-value-2"), + ( + b"my-custom-regex-header-1", + b"my-custom-regex-value-1,my-custom-regex-value-2", + ), + ( + b"My-Custom-Regex-Header-2", + b"my-custom-regex-value-3,my-custom-regex-value-4", + ), + (b"my-secret-header", b"my-secret-value"), ], } ) @@ -628,13 +647,6 @@ async def wrapped_app(scope, receive, send): ) -@mock.patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, -) class TestCustomHeaders(AsgiTestBase, TestBase): def setUp(self): super().setUp() @@ -644,11 +656,21 @@ def setUp(self): simple_asgi, tracer_provider=self.tracer_provider ) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_http_custom_request_headers_in_span_attributes(self): self.scope["headers"].extend( [ (b"custom-test-header-1", b"test-header-value-1"), (b"custom-test-header-2", b"test-header-value-2"), + (b"Regex-Test-Header-1", b"Regex Test Value 1"), + (b"regex-test-header-2", b"RegexTestValue2,RegexTestValue3"), + (b"My-Secret-Header", b"My Secret Value"), ] ) self.seed_app(self.app) @@ -662,11 +684,23 @@ def test_http_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } for span in span_list: if span.kind == SpanKind.SERVER: self.assertSpanHasAttributes(span, expected) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_http_custom_request_headers_not_in_span_attributes(self): self.scope["headers"].extend( [ @@ -693,6 +727,13 @@ def test_http_custom_request_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_http_custom_response_headers_in_span_attributes(self): self.app = otel_asgi.OpenTelemetryMiddleware( http_app_with_custom_headers, tracer_provider=self.tracer_provider @@ -708,11 +749,25 @@ def test_http_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } for span in span_list: if span.kind == SpanKind.SERVER: self.assertSpanHasAttributes(span, expected) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_http_custom_response_headers_not_in_span_attributes(self): self.app = otel_asgi.OpenTelemetryMiddleware( http_app_with_custom_headers, tracer_provider=self.tracer_provider @@ -731,6 +786,13 @@ def test_http_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_websocket_custom_request_headers_in_span_attributes(self): self.scope = { "type": "websocket", @@ -741,6 +803,9 @@ def test_websocket_custom_request_headers_in_span_attributes(self): "headers": [ (b"custom-test-header-1", b"test-header-value-1"), (b"custom-test-header-2", b"test-header-value-2"), + (b"Regex-Test-Header-1", b"Regex Test Value 1"), + (b"regex-test-header-2", b"RegexTestValue2,RegexTestValue3"), + (b"My-Secret-Header", b"My Secret Value"), ], "client": ("127.0.0.1", 32767), "server": ("127.0.0.1", 80), @@ -759,11 +824,23 @@ def test_websocket_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } for span in span_list: if span.kind == SpanKind.SERVER: self.assertSpanHasAttributes(span, expected) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_websocket_custom_request_headers_not_in_span_attributes(self): self.scope = { "type": "websocket", @@ -795,6 +872,13 @@ def test_websocket_custom_request_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_websocket_custom_response_headers_in_span_attributes(self): self.scope = { "type": "websocket", @@ -823,11 +907,25 @@ def test_websocket_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } for span in span_list: if span.kind == SpanKind.SERVER: self.assertSpanHasAttributes(span, expected) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_websocket_custom_response_headers_not_in_span_attributes(self): self.scope = { "type": "websocket", @@ -859,6 +957,173 @@ def test_websocket_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_http_custom_request_headers_in_span_attributes_all(self): + self.scope["headers"].extend( + [ + (b"custom-test-header-1", b"test-header-value-1"), + (b"custom-test-header-2", b"test-header-value-2"), + (b"Regex-Test-Header-1", b"Regex Test Value 1"), + (b"regex-test-header-2", b"RegexTestValue2,RegexTestValue3"), + (b"My-Secret-Header", b"My Secret Value"), + ] + ) + self.seed_app(self.app) + self.send_default_request() + self.get_all_output() + span_list = self.exporter.get_finished_spans() + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + for span in span_list: + if span.kind == SpanKind.SERVER: + self.assertSpanHasAttributes(span, expected) + + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_http_custom_response_headers_in_span_attributes_all(self): + self.app = otel_asgi.OpenTelemetryMiddleware( + http_app_with_custom_headers, tracer_provider=self.tracer_provider + ) + self.seed_app(self.app) + self.send_default_request() + self.get_all_output() + span_list = self.exporter.get_finished_spans() + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + for span in span_list: + if span.kind == SpanKind.SERVER: + self.assertSpanHasAttributes(span, expected) + + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_websocket_custom_request_headers_in_span_attributes_all(self): + self.scope = { + "type": "websocket", + "http_version": "1.1", + "scheme": "ws", + "path": "/", + "query_string": b"", + "headers": [ + (b"custom-test-header-1", b"test-header-value-1"), + (b"custom-test-header-2", b"test-header-value-2"), + (b"Regex-Test-Header-1", b"Regex Test Value 1"), + (b"regex-test-header-2", b"RegexTestValue2,RegexTestValue3"), + (b"My-Secret-Header", b"My Secret Value"), + ], + "client": ("127.0.0.1", 32767), + "server": ("127.0.0.1", 80), + } + self.seed_app(self.app) + self.send_input({"type": "websocket.connect"}) + self.send_input({"type": "websocket.receive", "text": "ping"}) + self.send_input({"type": "websocket.disconnect"}) + + self.get_all_output() + span_list = self.exporter.get_finished_spans() + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + for span in span_list: + if span.kind == SpanKind.SERVER: + self.assertSpanHasAttributes(span, expected) + + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_websocket_custom_response_headers_in_span_attributes_all(self): + self.scope = { + "type": "websocket", + "http_version": "1.1", + "scheme": "ws", + "path": "/", + "query_string": b"", + "headers": [], + "client": ("127.0.0.1", 32767), + "server": ("127.0.0.1", 80), + } + self.app = otel_asgi.OpenTelemetryMiddleware( + websocket_app_with_custom_headers, + tracer_provider=self.tracer_provider, + ) + self.seed_app(self.app) + self.send_input({"type": "websocket.connect"}) + self.send_input({"type": "websocket.receive", "text": "ping"}) + self.send_input({"type": "websocket.disconnect"}) + self.get_all_output() + span_list = self.exporter.get_finished_spans() + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + for span in span_list: + if span.kind == SpanKind.SERVER: + self.assertSpanHasAttributes(span, expected) + if __name__ == "__main__": unittest.main() diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_getter.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_getter.py index 454162d715..187d63274a 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_getter.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_getter.py @@ -46,5 +46,5 @@ def test_get_(self): def test_keys(self): getter = ASGIGetter() - keys = getter.keys({}) + keys = getter.keys({"headers": []}) self.assertEqual(keys, []) diff --git a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py index 4b8dec4e64..bfa688413d 100644 --- a/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/__init__.py @@ -94,8 +94,9 @@ Exclude lists ************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_DJANGO_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_DJANGO_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. For example, @@ -107,8 +108,8 @@ Request attributes ******************** -To extract certain attributes from Django's request object and use them as span attributes, set the environment variable ``OTEL_PYTHON_DJANGO_TRACED_REQUEST_ATTRS`` to a comma -delimited list of request attribute names. +To extract attributes from Django's request object and use them as span attributes, set the environment variable +``OTEL_PYTHON_DJANGO_TRACED_REQUEST_ATTRS`` to a comma delimited list of request attribute names. For example, @@ -116,14 +117,15 @@ export OTEL_PYTHON_DJANGO_TRACED_REQUEST_ATTRS='path_info,content_type' -will extract path_info and content_type attributes from every traced request and add them as span attritbues. +will extract the ``path_info`` and ``content_type`` attributes from every traced request and add them as span attributes. Django Request object reference: https://docs.djangoproject.com/en/3.1/ref/request-response/#attributes Request and Response hooks *************************** -The instrumentation supports specifying request and response hooks. These are functions that get called back by the instrumentation right after a Span is created for a request -and right before the span is finished while processing a response. The hooks can be configured as follows: +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. +The hooks can be configured as follows: .. code:: python @@ -140,50 +142,94 @@ def response_hook(span, request, response): Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, :: - export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content_type,custom_request_header" + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract content_type and custom_request_header from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in django are case insensitive. So, giving header name as ``CUStom_Header`` in environment variable will be able capture header with name ``custom-header``. +Request header names in Django are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" + +Would match all request headers that start with ``Accept`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" -Example of the added span attribute, +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, :: - export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content_type,custom_response_header" + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" + +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. -will extract content_type and custom_response_header from response headers and add them as span attributes. +Response header names in Django are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in django are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" -Example of the added span attribute, +Would match all response headers that start with ``Content`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + +Note: + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. + API --- diff --git a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py index 05457de43d..8e0520e137 100644 --- a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware.py @@ -43,6 +43,7 @@ format_trace_id, ) from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, get_excluded_urls, @@ -477,18 +478,9 @@ def setUp(self): tracer_provider, exporter = self.create_tracer_provider() self.exporter = exporter _django_instrumentor.instrument(tracer_provider=tracer_provider) - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, - ) - self.env_patch.start() def tearDown(self): super().tearDown() - self.env_patch.stop() teardown_test_environment() _django_instrumentor.uninstrument() @@ -497,6 +489,13 @@ def tearDownClass(cls): super().tearDownClass() conf.settings = conf.LazySettings() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_http_custom_request_headers_in_span_attributes(self): expected = { "http.request.header.custom_test_header_1": ( @@ -505,10 +504,18 @@ def test_http_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } Client( HTTP_CUSTOM_TEST_HEADER_1="test-header-value-1", HTTP_CUSTOM_TEST_HEADER_2="test-header-value-2", + HTTP_REGEX_TEST_HEADER_1="Regex Test Value 1", + HTTP_REGEX_TEST_HEADER_2="RegexTestValue2,RegexTestValue3", + HTTP_MY_SECRET_HEADER="My Secret Value", ).get("/traced/") spans = self.exporter.get_finished_spans() self.assertEqual(len(spans), 1) @@ -518,6 +525,13 @@ def test_http_custom_request_headers_in_span_attributes(self): self.assertSpanHasAttributes(span, expected) self.memory_exporter.clear() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_http_custom_request_headers_not_in_span_attributes(self): not_expected = { "http.request.header.custom_test_header_2": ( @@ -534,6 +548,13 @@ def test_http_custom_request_headers_not_in_span_attributes(self): self.assertNotIn(key, span.attributes) self.memory_exporter.clear() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_http_custom_response_headers_in_span_attributes(self): expected = { "http.response.header.custom_test_header_1": ( @@ -542,6 +563,13 @@ def test_http_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } Client().get("/traced_custom_header/") spans = self.exporter.get_finished_spans() @@ -552,6 +580,13 @@ def test_http_custom_response_headers_in_span_attributes(self): self.assertSpanHasAttributes(span, expected) self.memory_exporter.clear() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_http_custom_response_headers_not_in_span_attributes(self): not_expected = { "http.response.header.custom_test_header_3": ( @@ -567,3 +602,71 @@ def test_http_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) self.memory_exporter.clear() + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_http_custom_request_headers_in_span_attributes_all(self): + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + Client( + HTTP_CUSTOM_TEST_HEADER_1="test-header-value-1", + HTTP_CUSTOM_TEST_HEADER_2="test-header-value-2", + HTTP_REGEX_TEST_HEADER_1="Regex Test Value 1", + HTTP_REGEX_TEST_HEADER_2="RegexTestValue2,RegexTestValue3", + HTTP_MY_SECRET_HEADER="My Secret Value", + ).get("/traced/") + spans = self.exporter.get_finished_spans() + self.assertEqual(len(spans), 1) + + span = spans[0] + self.assertEqual(span.kind, SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + self.memory_exporter.clear() + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_http_custom_response_headers_in_span_attributes_all(self): + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + Client().get("/traced_custom_header/") + spans = self.exporter.get_finished_spans() + self.assertEqual(len(spans), 1) + + span = spans[0] + self.assertEqual(span.kind, SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + self.memory_exporter.clear() diff --git a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py index 941fda49bb..f3b0697bd4 100644 --- a/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py +++ b/instrumentation/opentelemetry-instrumentation-django/tests/test_middleware_asgi.py @@ -43,6 +43,7 @@ format_trace_id, ) from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, get_excluded_urls, @@ -437,18 +438,9 @@ def setUp(self): tracer_provider, exporter = self.create_tracer_provider() self.exporter = exporter _django_instrumentor.instrument(tracer_provider=tracer_provider) - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, - ) - self.env_patch.start() def tearDown(self): super().tearDown() - self.env_patch.stop() teardown_test_environment() _django_instrumentor.uninstrument() @@ -457,6 +449,13 @@ def tearDownClass(cls): super().tearDownClass() conf.settings = conf.LazySettings() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) async def test_http_custom_request_headers_in_span_attributes(self): expected = { "http.request.header.custom_test_header_1": ( @@ -465,12 +464,20 @@ async def test_http_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } await self.async_client.get( "/traced/", **{ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) spans = self.exporter.get_finished_spans() @@ -481,6 +488,13 @@ async def test_http_custom_request_headers_in_span_attributes(self): self.assertSpanHasAttributes(span, expected) self.memory_exporter.clear() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) async def test_http_custom_request_headers_not_in_span_attributes(self): not_expected = { "http.request.header.custom_test_header_2": ( @@ -502,6 +516,13 @@ async def test_http_custom_request_headers_not_in_span_attributes(self): self.assertNotIn(key, span.attributes) self.memory_exporter.clear() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) async def test_http_custom_response_headers_in_span_attributes(self): expected = { "http.response.header.custom_test_header_1": ( @@ -510,6 +531,13 @@ async def test_http_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } await self.async_client.get("/traced_custom_header/") spans = self.exporter.get_finished_spans() @@ -520,6 +548,13 @@ async def test_http_custom_response_headers_in_span_attributes(self): self.assertSpanHasAttributes(span, expected) self.memory_exporter.clear() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) async def test_http_custom_response_headers_not_in_span_attributes(self): not_expected = { "http.response.header.custom_test_header_3": ( @@ -535,3 +570,74 @@ async def test_http_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) self.memory_exporter.clear() + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + async def test_http_custom_request_headers_in_span_attributes_all(self): + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + await self.async_client.get( + "/traced/", + **{ + "custom-test-header-1": "test-header-value-1", + "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + }, + ) + spans = self.exporter.get_finished_spans() + self.assertEqual(len(spans), 1) + + span = spans[0] + self.assertEqual(span.kind, SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + self.memory_exporter.clear() + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + async def test_http_custom_response_headers_in_span_attributes_all(self): + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + await self.async_client.get("/traced_custom_header/") + spans = self.exporter.get_finished_spans() + self.assertEqual(len(spans), 1) + + span = spans[0] + self.assertEqual(span.kind, SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + self.memory_exporter.clear() diff --git a/instrumentation/opentelemetry-instrumentation-django/tests/views.py b/instrumentation/opentelemetry-instrumentation-django/tests/views.py index f97933cfd8..452a7c0fdd 100644 --- a/instrumentation/opentelemetry-instrumentation-django/tests/views.py +++ b/instrumentation/opentelemetry-instrumentation-django/tests/views.py @@ -35,6 +35,13 @@ def response_with_custom_header(request): response = HttpResponse() response["custom-test-header-1"] = "test-header-value-1" response["custom-test-header-2"] = "test-header-value-2" + response[ + "my-custom-regex-header-1" + ] = "my-custom-regex-value-1,my-custom-regex-value-2" + response[ + "my-custom-regex-header-2" + ] = "my-custom-regex-value-3,my-custom-regex-value-4" + response["my-secret-header"] = "my-secret-value" return response diff --git a/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py b/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py index cdf2c3553f..00a1c2ca9a 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py @@ -19,15 +19,16 @@ * The Falcon resource and method name is used as the Span name. * The ``falcon.resource`` Span attribute is set so the matched resource. -* Error from Falcon resources are properly caught and recorded. +* Errors from Falcon resources are properly caught and recorded. Configuration ------------- Exclude lists ************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_FALCON_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_FALCON_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. For example, @@ -39,8 +40,8 @@ Request attributes ******************** -To extract certain attributes from Falcon's request object and use them as span attributes, set the environment variable ``OTEL_PYTHON_FALCON_TRACED_REQUEST_ATTRS`` to a comma -delimited list of request attribute names. +To extract attributes from Falcon's request object and use them as span attributes, set the environment variable +``OTEL_PYTHON_FALCON_TRACED_REQUEST_ATTRS`` to a comma delimited list of request attribute names. For example, @@ -48,7 +49,7 @@ export OTEL_PYTHON_FALCON_TRACED_REQUEST_ATTRS='query_string,uri_template' -will extract query_string and uri_template attributes from every traced request and add them as span attritbues. +will extract the ``query_string`` and ``uri_template`` attributes from every traced request and add them as span attributes. Falcon Request object reference: https://falcon.readthedocs.io/en/stable/api/request_and_response.html#id1 @@ -73,8 +74,9 @@ def on_get(self, req, resp): Request and Response hooks *************************** -The instrumentation supports specifying request and response hooks. These are functions that get called back by the instrumentation right after a Span is created for a request -and right before the span is finished while processing a response. The hooks can be configured as follows: +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. +The hooks can be configured as follows: :: @@ -88,54 +90,93 @@ def response_hook(span, req, resp): Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in Falcon are case-insensitive and ``-`` characters are replaced by ``_``. So, giving the header +name as ``CUStom_Header`` in the environment variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in falcon are case insensitive and - characters are replaced by _. So, giving header name as ``CUStom_Header`` in environment variable will be able capture header with name ``custom-header``. +Would match all request headers that start with ``Accept`` and ``X-``. -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: -Example of the added span attribute, + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" + +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in Falcon are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" + +Would match all response headers that start with ``Content`` and ``X-``. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in falcon are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- diff --git a/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py b/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py index 6cc60faee6..3e4c62ec3e 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py @@ -8,7 +8,13 @@ class HelloWorldResource: def _handle_request(self, _, resp): # pylint: disable=no-member resp.status = falcon.HTTP_201 - resp.body = "Hello World" + + _parsed_falcon_version = package_version.parse(falcon.__version__) + if _parsed_falcon_version < package_version.parse("3.0.0"): + # Falcon 1 and Falcon 2 + resp.body = "Hello World" + else: + resp.text = "Hello World" def on_get(self, req, resp): self._handle_request(req, resp) @@ -44,6 +50,15 @@ def on_get(self, _, resp): "my-custom-header", "my-custom-value-1,my-custom-header-2" ) resp.set_header("dont-capture-me", "test-value") + resp.set_header( + "my-custom-regex-header-1", + "my-custom-regex-value-1,my-custom-regex-value-2", + ) + resp.set_header( + "My-Custom-Regex-Header-2", + "my-custom-regex-value-3,my-custom-regex-value-4", + ) + resp.set_header("my-secret-header", "my-secret-value") def make_app(): diff --git a/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py b/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py index 5098937b2a..1492c01ee7 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py @@ -32,6 +32,7 @@ from opentelemetry.test.wsgitestutil import WsgiTestBase from opentelemetry.trace import StatusCode from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, ) @@ -307,19 +308,22 @@ def test_mark_span_internal_in_presence_of_span_from_other_framework(self): ) -@patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,invalid-header", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header", - }, -) class TestCustomRequestResponseHeaders(TestFalconBase): + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_header_added_in_server_span(self): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", "Custom-Test-Header-3": "TestValue4", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } self.client().simulate_request( method="GET", path="/hello", headers=headers @@ -332,6 +336,11 @@ def test_custom_request_header_added_in_server_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } not_expected = { "http.request.header.custom_test_header_3": ("TestValue4",), @@ -342,12 +351,22 @@ def test_custom_request_header_added_in_server_span(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_header_not_added_in_internal_span(self): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("test", kind=trace.SpanKind.SERVER): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } self.client().simulate_request( method="GET", path="/hello", headers=headers @@ -359,6 +378,13 @@ def test_custom_request_header_not_added_in_internal_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ( + "Regex Test Value 1", + ), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(span.kind, trace.SpanKind.INTERNAL) for key, _ in not_expected.items(): @@ -369,6 +395,13 @@ def test_custom_request_header_not_added_in_internal_span(self): < package_version.parse("2.0.0"), reason="falcon<2 does not implement custom response headers", ) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_header_added_in_server_span(self): self.client().simulate_request( method="GET", path="/test_custom_response_headers" @@ -383,6 +416,13 @@ def test_custom_response_header_added_in_server_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } not_expected = { "http.response.header.dont_capture_me": ("test-value",) @@ -397,6 +437,13 @@ def test_custom_response_header_added_in_server_span(self): < package_version.parse("2.0.0"), reason="falcon<2 does not implement custom response headers", ) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_header_not_added_in_internal_span(self): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("test", kind=trace.SpanKind.SERVER): @@ -413,7 +460,90 @@ def test_custom_response_header_not_added_in_internal_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(span.kind, trace.SpanKind.INTERNAL) for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_header_added_in_server_span_all(self): + headers = { + "Custom-Test-Header-1": "Test Value 1", + "Custom-Test-Header-2": "TestValue2,TestValue3", + "Custom-Test-Header-3": "TestValue4", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + } + self.client().simulate_request( + method="GET", path="/hello", headers=headers + ) + span = self.memory_exporter.get_finished_spans()[0] + assert span.status.is_ok + + expected = { + "http.request.header.custom_test_header_1": ("Test Value 1",), + "http.request.header.custom_test_header_2": ( + "TestValue2,TestValue3", + ), + "http.request.header.custom_test_header_3": ("TestValue4",), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + + self.assertEqual(span.kind, trace.SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + + @pytest.mark.skipif( + condition=package_version.parse(_falcon_verison) + < package_version.parse("2.0.0"), + reason="falcon<2 does not implement custom response headers", + ) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_header_added_in_server_span_all(self): + self.client().simulate_request( + method="GET", path="/test_custom_response_headers" + ) + span = self.memory_exporter.get_finished_spans()[0] + assert span.status.is_ok + expected = { + "http.response.header.content_type": ( + "text/plain; charset=utf-8", + ), + "http.response.header.content_length": ("0",), + "http.response.header.dont_capture_me": ("test-value",), + "http.response.header.my_custom_header": ( + "my-custom-value-1,my-custom-header-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + self.assertEqual(span.kind, trace.SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) diff --git a/instrumentation/opentelemetry-instrumentation-fastapi/src/opentelemetry/instrumentation/fastapi/__init__.py b/instrumentation/opentelemetry-instrumentation-fastapi/src/opentelemetry/instrumentation/fastapi/__init__.py index 5cf17d36f0..ba88cc973c 100644 --- a/instrumentation/opentelemetry-instrumentation-fastapi/src/opentelemetry/instrumentation/fastapi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-fastapi/src/opentelemetry/instrumentation/fastapi/__init__.py @@ -34,8 +34,9 @@ async def foobar(): Exclude lists ************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. For example, @@ -45,7 +46,7 @@ async def foobar(): will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``. -You can also pass the comma delimited regexes to the ``instrument_app`` method directly: +You can also pass comma delimited regexes directly to the ``instrument_app`` method: .. code-block:: python @@ -54,9 +55,12 @@ async def foobar(): Request/Response hooks ********************** -Utilize request/reponse hooks to execute custom logic to be performed before/after performing a request. The server request hook takes in a server span and ASGI -scope object for every incoming request. The client request hook is called with the internal span and an ASGI scope which is sent as a dictionary for when the method recieve is called. -The client response hook is called with the internal span and an ASGI event which is sent as a dictionary for when the method send is called. +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. + +- The server request hook is passed a server span and ASGI scope object for every incoming request. +- The client request hook is called with the internal span and an ASGI scope when the method ``receive`` is called. +- The client response hook is called with the internal span and an ASGI event when the method ``send`` is called. .. code-block:: python @@ -76,54 +80,93 @@ def client_response_hook(span: Span, message: dict): Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in FastAPI are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in fastapi are case insensitive. So, giving header name as ``CUStom-Header`` in environment variable will be able capture header with name ``custom-header``. +Would match all request headers that start with ``Accept`` and ``X-``. -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" + +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in FastAPI are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in fastapi are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" -Example of the added span attribute, +Would match all response headers that start with ``Content`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- diff --git a/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py b/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py index e4a0960a26..f74fa4b8fe 100644 --- a/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py +++ b/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py @@ -27,6 +27,7 @@ from opentelemetry.test.globals_test import reset_trace_globals from opentelemetry.test.test_base import TestBase from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, get_excluded_urls, @@ -386,21 +387,12 @@ def test_mark_span_internal_in_presence_of_span_from_other_framework(self): class TestHTTPAppWithCustomHeaders(TestBase): def setUp(self): super().setUp() - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, - ) - self.env_patch.start() self.app = self._create_app() otel_fastapi.FastAPIInstrumentor().instrument_app(self.app) self.client = TestClient(self.app) def tearDown(self) -> None: super().tearDown() - self.env_patch.stop() with self.disable_logging(): otel_fastapi.FastAPIInstrumentor().uninstrument_app(self.app) @@ -413,12 +405,22 @@ async def _(): headers = { "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "my-custom-regex-header-1": "my-custom-regex-value-1,my-custom-regex-value-2", + "My-Custom-Regex-Header-2": "my-custom-regex-value-3,my-custom-regex-value-4", + "My-Secret-Header": "My Secret Value", } content = {"message": "hello world"} return JSONResponse(content=content, headers=headers) return app + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_http_custom_request_headers_in_span_attributes(self): expected = { "http.request.header.custom_test_header_1": ( @@ -427,12 +429,20 @@ def test_http_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } resp = self.client.get( "/foobar", headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) self.assertEqual(200, resp.status_code) @@ -445,6 +455,13 @@ def test_http_custom_request_headers_in_span_attributes(self): self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_http_custom_request_headers_not_in_span_attributes(self): not_expected = { "http.request.header.custom_test_header_3": ( @@ -456,6 +473,9 @@ def test_http_custom_request_headers_not_in_span_attributes(self): headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) self.assertEqual(200, resp.status_code) @@ -469,6 +489,13 @@ def test_http_custom_request_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_http_custom_response_headers_in_span_attributes(self): expected = { "http.response.header.custom_test_header_1": ( @@ -477,6 +504,13 @@ def test_http_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } resp = self.client.get("/foobar") self.assertEqual(200, resp.status_code) @@ -488,6 +522,13 @@ def test_http_custom_response_headers_in_span_attributes(self): ][0] self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_http_custom_response_headers_not_in_span_attributes(self): not_expected = { "http.reponse.header.custom_test_header_3": ( @@ -506,25 +547,90 @@ def test_http_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_http_custom_request_headers_in_span_attributes_all(self): + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + resp = self.client.get( + "/foobar", + headers={ + "custom-test-header-1": "test-header-value-1", + "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + }, + ) + self.assertEqual(200, resp.status_code) + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 3) + + server_span = [ + span for span in span_list if span.kind == trace.SpanKind.SERVER + ][0] + + self.assertSpanHasAttributes(server_span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_http_custom_response_headers_in_span_attributes_all(self): + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + resp = self.client.get("/foobar") + self.assertEqual(200, resp.status_code) + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 3) + + server_span = [ + span for span in span_list if span.kind == trace.SpanKind.SERVER + ][0] + self.assertSpanHasAttributes(server_span, expected) + class TestWebSocketAppWithCustomHeaders(TestBase): def setUp(self): super().setUp() - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, - ) - self.env_patch.start() self.app = self._create_app() otel_fastapi.FastAPIInstrumentor().instrument_app(self.app) self.client = TestClient(self.app) def tearDown(self) -> None: super().tearDown() - self.env_patch.stop() with self.disable_logging(): otel_fastapi.FastAPIInstrumentor().uninstrument_app(self.app) @@ -542,6 +648,12 @@ async def _(websocket: fastapi.WebSocket): "headers": [ (b"custom-test-header-1", b"test-header-value-1"), (b"custom-test-header-2", b"test-header-value-2"), + (b"Regex-Test-Header-1", b"Regex Test Value 1"), + ( + b"regex-test-header-2", + b"RegexTestValue2,RegexTestValue3", + ), + (b"My-Secret-Header", b"My Secret Value"), ], } ) @@ -552,6 +664,13 @@ async def _(websocket: fastapi.WebSocket): return app + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_web_socket_custom_request_headers_in_span_attributes(self): expected = { "http.request.header.custom_test_header_1": ( @@ -581,6 +700,13 @@ def test_web_socket_custom_request_headers_in_span_attributes(self): self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_web_socket_custom_request_headers_not_in_span_attributes(self): not_expected = { "http.request.header.custom_test_header_3": ( @@ -608,6 +734,13 @@ def test_web_socket_custom_request_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_web_socket_custom_response_headers_in_span_attributes(self): expected = { "http.response.header.custom_test_header_1": ( @@ -631,6 +764,13 @@ def test_web_socket_custom_response_headers_in_span_attributes(self): self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_web_socket_custom_response_headers_not_in_span_attributes(self): not_expected = { "http.reponse.header.custom_test_header_3": ( @@ -652,17 +792,82 @@ def test_web_socket_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_web_socket_custom_request_headers_in_span_attributes_all(self): + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + } + + with self.client.websocket_connect( + "/foobar_web", + headers={ + "custom-test-header-1": "test-header-value-1", + "custom-test-header-2": "test-header-value-2", + }, + ) as websocket: + data = websocket.receive_json() + self.assertEqual(data, {"message": "hello world"}) + + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 5) + + server_span = [ + span for span in span_list if span.kind == trace.SpanKind.SERVER + ][0] + + self.assertSpanHasAttributes(server_span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_web_socket_custom_response_headers_in_span_attributes_all(self): + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + } + with self.client.websocket_connect("/foobar_web") as websocket: + data = websocket.receive_json() + self.assertEqual(data, {"message": "hello world"}) + + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 5) + + server_span = [ + span for span in span_list if span.kind == trace.SpanKind.SERVER + ][0] + + self.assertSpanHasAttributes(server_span, expected) + + +@patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", + }, +) class TestNonRecordingSpanWithCustomHeaders(TestBase): def setUp(self): super().setUp() - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, - ) - self.env_patch.start() self.app = fastapi.FastAPI() @self.app.get("/foobar") diff --git a/instrumentation/opentelemetry-instrumentation-flask/README.rst b/instrumentation/opentelemetry-instrumentation-flask/README.rst index d3092cc4a7..274247efb1 100644 --- a/instrumentation/opentelemetry-instrumentation-flask/README.rst +++ b/instrumentation/opentelemetry-instrumentation-flask/README.rst @@ -16,48 +16,6 @@ Installation pip install opentelemetry-instrumentation-flask -Configuration -------------- - -Exclude lists -************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_FLASK_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. - -For example, - -:: - - export OTEL_PYTHON_FLASK_EXCLUDED_URLS="client/.*/info,healthcheck" - -will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``. - -You can also pass the comma delimited regexes to the ``instrument_app`` method directly: - -.. code-block:: python - - FlaskInstrumentor().instrument_app(app, excluded_urls="client/.*/info,healthcheck") - -Request/Response hooks -********************** - -Utilize request/reponse hooks to execute custom logic to be performed before/after performing a request. Environ is an instance of WSGIEnvironment (flask.request.environ). -Response_headers is a list of key-value (tuples) representing the response headers returned from the response. - -.. code-block:: python - - def request_hook(span: Span, environ: WSGIEnvironment): - if span and span.is_recording(): - span.set_attribute("custom_user_attribute_from_request_hook", "some-value") - - def response_hook(span: Span, status: str, response_headers: List): - if span and span.is_recording(): - span.set_attribute("custom_user_attribute_from_response_hook", "some-value") - - FlaskInstrumentation().instrument(request_hook=request_hook, response_hook=response_hook) - -Flask Request object reference: https://flask.palletsprojects.com/en/2.0.x/api/#flask.Request - References ---------- diff --git a/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py b/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py index cca8743556..4fcc2622d1 100644 --- a/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py @@ -48,8 +48,9 @@ def hello(): Exclude lists ************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_FLASK_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_FLASK_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. For example, @@ -59,7 +60,7 @@ def hello(): will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``. -You can also pass the comma delimited regexes to the ``instrument_app`` method directly: +You can also pass comma delimited regexes directly to the ``instrument_app`` method: .. code-block:: python @@ -68,8 +69,15 @@ def hello(): Request/Response hooks ********************** -Utilize request/reponse hooks to execute custom logic to be performed before/after performing a request. Environ is an instance of WSGIEnvironment (flask.request.environ). -Response_headers is a list of key-value (tuples) representing the response headers returned from the response. +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. + +- The client request hook is called with the internal span and an instance of WSGIEnvironment (flask.request.environ) + when the method ``receive`` is called. +- The client response hook is called with the internal span, the status of the response and a list of key-value (tuples) + representing the response headers returned from the response when the method ``send`` is called. + +For example, .. code-block:: python @@ -83,58 +91,97 @@ def response_hook(span: Span, status: str, response_headers: List): FlaskInstrumentation().instrument(request_hook=request_hook, response_hook=response_hook) -Flask Request object reference: https://flask.palletsprojects.com/en/2.0.x/api/#flask.Request +Flask Request object reference: https://flask.palletsprojects.com/en/2.1.x/api/#flask.Request Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in Flask are case-insensitive and ``-`` characters are replaced by ``_``. So, giving the header +name as ``CUStom_Header`` in the environment variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in flask are case insensitive and - characters are replaced by _. So, giving header name as ``CUStom_Header`` in environment variable will be able capture header with name ``custom-header``. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Would match all request headers that start with ``Accept`` and ``X-``. -Example of the added span attribute, +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" + +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in flask are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Response header names in Flask are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -Example of the added span attribute, + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" + +Would match all response headers that start with ``Content`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- diff --git a/instrumentation/opentelemetry-instrumentation-flask/tests/base_test.py b/instrumentation/opentelemetry-instrumentation-flask/tests/base_test.py index 79b1edf6ab..ab3d8f8233 100644 --- a/instrumentation/opentelemetry-instrumentation-flask/tests/base_test.py +++ b/instrumentation/opentelemetry-instrumentation-flask/tests/base_test.py @@ -32,6 +32,13 @@ def _custom_response_headers(): resp.headers[ "my-custom-header" ] = "my-custom-value-1,my-custom-header-2" + resp.headers[ + "my-custom-regex-header-1" + ] = "my-custom-regex-value-1,my-custom-regex-value-2" + resp.headers[ + "My-Custom-Regex-Header-2" + ] = "my-custom-regex-value-3,my-custom-regex-value-4" + resp.headers["my-secret-header"] = "my-secret-value" return resp def _common_initialization(self): diff --git a/instrumentation/opentelemetry-instrumentation-flask/tests/test_programmatic.py b/instrumentation/opentelemetry-instrumentation-flask/tests/test_programmatic.py index 2bcb097c7b..7ba79ece52 100644 --- a/instrumentation/opentelemetry-instrumentation-flask/tests/test_programmatic.py +++ b/instrumentation/opentelemetry-instrumentation-flask/tests/test_programmatic.py @@ -27,7 +27,12 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.semconv.trace import SpanAttributes from opentelemetry.test.wsgitestutil import WsgiTestBase -from opentelemetry.util.http import get_excluded_urls +from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, + get_excluded_urls, +) # pylint: disable=import-error from .base_test import InstrumentationTest @@ -443,14 +448,6 @@ class TestCustomRequestResponseHeaders(InstrumentationTest, WsgiTestBase): def setUp(self): super().setUp() - self.env_patch = patch.dict( - "os.environ", - { - "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST": "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE": "content-type,content-length,my-custom-header,invalid-header", - }, - ) - self.env_patch.start() self.app = Flask(__name__) FlaskInstrumentor().instrument_app(self.app) @@ -458,14 +455,23 @@ def setUp(self): def tearDown(self): super().tearDown() - self.env_patch.stop() with self.disable_logging(): FlaskInstrumentor().uninstrument_app(self.app) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_header_added_in_server_span(self): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } resp = self.client.get("/hello/123", headers=headers) self.assertEqual(200, resp.status_code) @@ -475,16 +481,31 @@ def test_custom_request_header_added_in_server_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(span.kind, trace.SpanKind.SERVER) self.assertSpanHasAttributes(span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_header_not_added_in_internal_span(self): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("test", kind=trace.SpanKind.SERVER): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } resp = self.client.get("/hello/123", headers=headers) self.assertEqual(200, resp.status_code) @@ -494,11 +515,25 @@ def test_custom_request_header_not_added_in_internal_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ( + "Regex Test Value 1", + ), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(span.kind, trace.SpanKind.INTERNAL) for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_header_added_in_server_span(self): resp = self.client.get("/test_custom_response_headers") self.assertEqual(resp.status_code, 200) @@ -511,10 +546,24 @@ def test_custom_response_header_added_in_server_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(span.kind, trace.SpanKind.SERVER) self.assertSpanHasAttributes(span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_header_not_added_in_internal_span(self): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("test", kind=trace.SpanKind.SERVER): @@ -529,7 +578,76 @@ def test_custom_response_header_not_added_in_internal_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(span.kind, trace.SpanKind.INTERNAL) for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_header_added_in_server_span_all(self): + headers = { + "Custom-Test-Header-1": "Test Value 1", + "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + } + resp = self.client.get("/hello/123", headers=headers) + self.assertEqual(200, resp.status_code) + span = self.memory_exporter.get_finished_spans()[0] + expected = { + "http.request.header.custom_test_header_1": ("Test Value 1",), + "http.request.header.custom_test_header_2": ( + "TestValue2,TestValue3", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + self.assertEqual(span.kind, trace.SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_header_added_in_server_span_all(self): + resp = self.client.get("/test_custom_response_headers") + self.assertEqual(resp.status_code, 200) + span = self.memory_exporter.get_finished_spans()[0] + expected = { + "http.response.header.content_type": ( + "text/plain; charset=utf-8", + ), + "http.response.header.content_length": ("13",), + "http.response.header.my_custom_header": ( + "my-custom-value-1,my-custom-header-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + self.assertEqual(span.kind, trace.SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) diff --git a/instrumentation/opentelemetry-instrumentation-pyramid/src/opentelemetry/instrumentation/pyramid/__init__.py b/instrumentation/opentelemetry-instrumentation-pyramid/src/opentelemetry/instrumentation/pyramid/__init__.py index c225f2f11d..5c49b51b8c 100644 --- a/instrumentation/opentelemetry-instrumentation-pyramid/src/opentelemetry/instrumentation/pyramid/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-pyramid/src/opentelemetry/instrumentation/pyramid/__init__.py @@ -55,7 +55,7 @@ --------------------------------- If you use Method 2 and then set tweens for your application with the ``pyramid.tweens`` setting, -you need to add ``opentelemetry.instrumentation.pyramid.trace_tween_factory`` explicity to the list, +you need to explicitly add ``opentelemetry.instrumentation.pyramid.trace_tween_factory`` to the list, *as well as* instrumenting the config as shown above. For example: @@ -79,8 +79,9 @@ Exclude lists ************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_PYRAMID_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_PYRAMID_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. For example, @@ -92,54 +93,93 @@ Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in Pyramid are case-insensitive and ``-`` characters are replaced by ``_``. So, giving the header +name as ``CUStom_Header`` in the environment variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" + +Would match all request headers that start with ``Accept`` and ``X-``. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in pyramid are case insensitive and - characters are replaced by _. So, giving header name as ``CUStom_Header`` in environment variable will be able capture header with name ``custom-header``. +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in Pyramid are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" + +Would match all response headers that start with ``Content`` and ``X-``. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in pyramid are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- diff --git a/instrumentation/opentelemetry-instrumentation-pyramid/tests/pyramid_base_test.py b/instrumentation/opentelemetry-instrumentation-pyramid/tests/pyramid_base_test.py index 8f97cf2db7..f5dd9fd7d7 100644 --- a/instrumentation/opentelemetry-instrumentation-pyramid/tests/pyramid_base_test.py +++ b/instrumentation/opentelemetry-instrumentation-pyramid/tests/pyramid_base_test.py @@ -40,6 +40,9 @@ def _custom_response_header_endpoint(request): "content-type": "text/plain; charset=utf-8", "content-length": "7", "my-custom-header": "my-custom-value-1,my-custom-header-2", + "my-custom-regex-header-1": "my-custom-regex-value-1,my-custom-regex-value-2", + "My-Custom-Regex-Header-2": "my-custom-regex-value-3,my-custom-regex-value-4", + "my-secret-header": "my-secret-value", "dont-capture-me": "test-value", } return Response("Testing", headers=headers) diff --git a/instrumentation/opentelemetry-instrumentation-pyramid/tests/test_automatic.py b/instrumentation/opentelemetry-instrumentation-pyramid/tests/test_automatic.py index 89df49e49e..64b98970f4 100644 --- a/instrumentation/opentelemetry-instrumentation-pyramid/tests/test_automatic.py +++ b/instrumentation/opentelemetry-instrumentation-pyramid/tests/test_automatic.py @@ -23,6 +23,7 @@ from opentelemetry.trace import SpanKind from opentelemetry.trace.status import StatusCode from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, ) @@ -192,26 +193,27 @@ def setUp(self): PyramidInstrumentor().instrument() self.config = Configurator() self._common_initialization(self.config) - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,invalid-header", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header", - }, - ) - self.env_patch.start() def tearDown(self) -> None: super().tearDown() - self.env_patch.stop() with self.disable_logging(): PyramidInstrumentor().uninstrument() + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,invalid-header,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_header_added_in_server_span(self): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", "Custom-Test-Header-3": "TestValue4", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } resp = self.client.get("/hello/123", headers=headers) self.assertEqual(200, resp.status_code) @@ -221,6 +223,11 @@ def test_custom_request_header_added_in_server_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } not_expected = { "http.request.header.custom_test_header_3": ("TestValue4",), @@ -230,6 +237,13 @@ def test_custom_request_header_added_in_server_span(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,invalid-header,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_header_not_added_in_internal_span(self): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("test", kind=SpanKind.SERVER): @@ -250,6 +264,13 @@ def test_custom_request_header_not_added_in_internal_span(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_header_added_in_server_span(self): resp = self.client.get("/test_custom_response_headers") self.assertEqual(200, resp.status_code) @@ -262,6 +283,13 @@ def test_custom_response_header_added_in_server_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } not_expected = { "http.response.header.dont_capture_me": ("test-value",) @@ -271,6 +299,13 @@ def test_custom_response_header_added_in_server_span(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_header_not_added_in_internal_span(self): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("test", kind=SpanKind.SERVER): @@ -290,7 +325,79 @@ def test_custom_response_header_not_added_in_internal_span(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_header_added_in_server_span_all(self): + headers = { + "Custom-Test-Header-1": "Test Value 1", + "Custom-Test-Header-2": "TestValue2,TestValue3", + "Custom-Test-Header-3": "TestValue4", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + } + resp = self.client.get("/hello/123", headers=headers) + self.assertEqual(200, resp.status_code) + span = self.memory_exporter.get_finished_spans()[0] + expected = { + "http.request.header.custom_test_header_1": ("Test Value 1",), + "http.request.header.custom_test_header_2": ( + "TestValue2,TestValue3", + ), + "http.request.header.custom_test_header_3": ("TestValue4",), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + self.assertEqual(span.kind, SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_header_added_in_server_span_all(self): + resp = self.client.get("/test_custom_response_headers") + self.assertEqual(200, resp.status_code) + span = self.memory_exporter.get_finished_spans()[0] + expected = { + "http.response.header.content_type": ( + "text/plain; charset=utf-8", + ), + "http.response.header.content_length": ("7",), + "http.response.header.my_custom_header": ( + "my-custom-value-1,my-custom-header-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + "http.response.header.dont_capture_me": ("test-value",), + } + self.assertEqual(span.kind, SpanKind.SERVER) + self.assertSpanHasAttributes(span, expected) + +@patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,invalid-header", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header", + }, +) class TestCustomHeadersNonRecordingSpan(InstrumentationTest, WsgiTestBase): def setUp(self): super().setUp() @@ -302,18 +409,9 @@ def setUp(self): PyramidInstrumentor().instrument() self.config = Configurator() self._common_initialization(self.config) - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,invalid-header", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header", - }, - ) - self.env_patch.start() def tearDown(self) -> None: super().tearDown() - self.env_patch.stop() with self.disable_logging(): PyramidInstrumentor().uninstrument() diff --git a/instrumentation/opentelemetry-instrumentation-starlette/src/opentelemetry/instrumentation/starlette/__init__.py b/instrumentation/opentelemetry-instrumentation-starlette/src/opentelemetry/instrumentation/starlette/__init__.py index 259e054d32..0ad66c0b28 100644 --- a/instrumentation/opentelemetry-instrumentation-starlette/src/opentelemetry/instrumentation/starlette/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-starlette/src/opentelemetry/instrumentation/starlette/__init__.py @@ -36,8 +36,9 @@ def home(request): Exclude lists ************* -To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_STARLETTE_EXCLUDED_URLS`` -(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude. +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_STARLETTE_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. For example, @@ -50,9 +51,14 @@ def home(request): Request/Response hooks ********************** -Utilize request/response hooks to execute custom logic to be performed before/after performing a request. The server request hook takes in a server span and ASGI -scope object for every incoming request. The client request hook is called with the internal span and an ASGI scope which is sent as a dictionary for when the method recieve is called. -The client response hook is called with the internal span and an ASGI event which is sent as a dictionary for when the method send is called. +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. + +- The server request hook is passed a server span and ASGI scope object for every incoming request. +- The client request hook is called with the internal span and an ASGI scope when the method ``receive`` is called. +- The client response hook is called with the internal span and an ASGI event when the method ``send`` is called. + +For example, .. code-block:: python @@ -70,54 +76,93 @@ def client_response_hook(span: Span, message: dict): Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in Starlette are case-insensitive. So, giving the header name as ``CUStom-Header`` in the +environment variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in starlette are case insensitive. So, giving header name as ``CUStom-Header`` in environment variable will be able capture header with name ``custom-header``. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Would match all request headers that start with ``Accept`` and ``X-``. -Example of the added span attribute, +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" + +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in starlette are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Response header names in Starlette are case-insensitive. So, giving the header name as ``CUStom-Header`` in the +environment variable will capture the header named ``custom-header``. -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -Example of the added span attribute, + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" + +Would match all response headers that start with ``Content`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- diff --git a/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py b/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py index 8c98feca4e..8257536472 100644 --- a/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py +++ b/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py @@ -33,6 +33,7 @@ set_tracer_provider, ) from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, get_excluded_urls, @@ -265,21 +266,12 @@ def create_app(self): def setUp(self): super().setUp() - self.env_patch = patch.dict( - "os.environ", - { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", - }, - ) - self.env_patch.start() self._instrumentor = otel_starlette.StarletteInstrumentor() self._app = self.create_app() self._client = TestClient(self._app) def tearDown(self) -> None: super().tearDown() - self.env_patch.stop() with self.disable_logging(): self._instrumentor.uninstrument() @@ -294,6 +286,9 @@ def _(request): headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "my-custom-regex-header-1": "my-custom-regex-value-1,my-custom-regex-value-2", + "My-Custom-Regex-Header-2": "my-custom-regex-value-3,my-custom-regex-value-4", + "my-secret-header": "my-secret-value", }, ) @@ -307,6 +302,15 @@ async def _(websocket: WebSocket) -> None: "headers": [ (b"custom-test-header-1", b"test-header-value-1"), (b"custom-test-header-2", b"test-header-value-2"), + ( + b"my-custom-regex-header-1", + b"my-custom-regex-value-1,my-custom-regex-value-2", + ), + ( + b"My-Custom-Regex-Header-2", + b"my-custom-regex-value-3,my-custom-regex-value-4", + ), + (b"my-secret-header", b"my-secret-value"), ], } ) @@ -319,6 +323,13 @@ async def _(websocket: WebSocket) -> None: class TestHTTPAppWithCustomHeaders(TestBaseWithCustomHeaders): + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_headers_in_span_attributes(self): expected = { "http.request.header.custom_test_header_1": ( @@ -327,12 +338,20 @@ def test_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } resp = self._client.get( "/foobar", headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) self.assertEqual(200, resp.status_code) @@ -345,6 +364,13 @@ def test_custom_request_headers_in_span_attributes(self): self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_headers_not_in_span_attributes(self): not_expected = { "http.request.header.custom_test_header_3": ( @@ -356,6 +382,9 @@ def test_custom_request_headers_not_in_span_attributes(self): headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) self.assertEqual(200, resp.status_code) @@ -369,6 +398,13 @@ def test_custom_request_headers_not_in_span_attributes(self): for key in not_expected: self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_headers_in_span_attributes(self): expected = { "http.response.header.custom_test_header_1": ( @@ -377,6 +413,13 @@ def test_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } resp = self._client.get("/foobar") self.assertEqual(200, resp.status_code) @@ -389,6 +432,13 @@ def test_custom_response_headers_in_span_attributes(self): self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_headers_not_in_span_attributes(self): not_expected = { "http.response.header.custom_test_header_3": ( @@ -407,8 +457,90 @@ def test_custom_response_headers_not_in_span_attributes(self): for key in not_expected: self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_headers_in_span_attributes_all(self): + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + resp = self._client.get( + "/foobar", + headers={ + "custom-test-header-1": "test-header-value-1", + "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + }, + ) + self.assertEqual(200, resp.status_code) + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 3) + + server_span = [ + span for span in span_list if span.kind == SpanKind.SERVER + ][0] + + self.assertSpanHasAttributes(server_span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_headers_in_span_attributes_all(self): + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + resp = self._client.get("/foobar") + self.assertEqual(200, resp.status_code) + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 3) + + server_span = [ + span for span in span_list if span.kind == SpanKind.SERVER + ][0] + + self.assertSpanHasAttributes(server_span, expected) + class TestWebSocketAppWithCustomHeaders(TestBaseWithCustomHeaders): + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_headers_in_span_attributes(self): expected = { "http.request.header.custom_test_header_1": ( @@ -417,12 +549,20 @@ def test_custom_request_headers_in_span_attributes(self): "http.request.header.custom_test_header_2": ( "test-header-value-2", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } with self._client.websocket_connect( "/foobar_web", headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) as websocket: data = websocket.receive_json() @@ -436,6 +576,13 @@ def test_custom_request_headers_in_span_attributes(self): ][0] self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_request_headers_not_in_span_attributes(self): not_expected = { "http.request.header.custom_test_header_3": ( @@ -447,6 +594,9 @@ def test_custom_request_headers_not_in_span_attributes(self): headers={ "custom-test-header-1": "test-header-value-1", "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", }, ) as websocket: data = websocket.receive_json() @@ -462,6 +612,13 @@ def test_custom_request_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_headers_in_span_attributes(self): expected = { "http.response.header.custom_test_header_1": ( @@ -470,6 +627,13 @@ def test_custom_response_headers_in_span_attributes(self): "http.response.header.custom_test_header_2": ( "test-header-value-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } with self._client.websocket_connect("/foobar_web") as websocket: data = websocket.receive_json() @@ -484,6 +648,13 @@ def test_custom_response_headers_in_span_attributes(self): self.assertSpanHasAttributes(server_span, expected) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", + }, + ) def test_custom_response_headers_not_in_span_attributes(self): not_expected = { "http.response.header.custom_test_header_3": ( @@ -504,6 +675,84 @@ def test_custom_response_headers_not_in_span_attributes(self): for key, _ in not_expected.items(): self.assertNotIn(key, server_span.attributes) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_headers_in_span_attributes_all(self): + expected = { + "http.request.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.request.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + with self._client.websocket_connect( + "/foobar_web", + headers={ + "custom-test-header-1": "test-header-value-1", + "custom-test-header-2": "test-header-value-2", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + }, + ) as websocket: + data = websocket.receive_json() + self.assertEqual(data, {"message": "hello world"}) + + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 5) + + server_span = [ + span for span in span_list if span.kind == SpanKind.SERVER + ][0] + self.assertSpanHasAttributes(server_span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_headers_in_span_attributes_all(self): + expected = { + "http.response.header.custom_test_header_1": ( + "test-header-value-1", + ), + "http.response.header.custom_test_header_2": ( + "test-header-value-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + with self._client.websocket_connect("/foobar_web") as websocket: + data = websocket.receive_json() + self.assertEqual(data, {"message": "hello world"}) + + span_list = self.memory_exporter.get_finished_spans() + self.assertEqual(len(span_list), 5) + + server_span = [ + span for span in span_list if span.kind == SpanKind.SERVER + ][0] + + self.assertSpanHasAttributes(server_span, expected) + class TestNonRecordingSpanWithCustomHeaders(TestBaseWithCustomHeaders): def setUp(self): @@ -514,6 +763,13 @@ def setUp(self): self._app = self.create_app() self._client = TestClient(self._app) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", + }, + ) def test_custom_header_not_present_in_non_recording_span(self): resp = self._client.get( "/foobar", diff --git a/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py b/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py index 20ad51cd9c..cb77be2ab2 100644 --- a/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/__init__.py @@ -37,23 +37,25 @@ def get(self): Configuration ------------- -The following environment variables are supported as configuration options: +Exclude lists +************* +To exclude certain URLs from tracking, set the environment variable ``OTEL_PYTHON_TORNADO_EXCLUDED_URLS`` +(or ``OTEL_PYTHON_EXCLUDED_URLS`` to cover all instrumentations) to a string of comma delimited regexes that match the +URLs. -- OTEL_PYTHON_TORNADO_EXCLUDED_URLS - -A comma separated list of paths that should not be automatically traced. For example, if this is set to +For example, :: - export OTEL_PYTHON_TORNADO_EXCLUDED_URLS='/healthz,/ping' + export OTEL_PYTHON_TORNADO_EXCLUDED_URLS="client/.*/info,healthcheck" -Then any requests made to ``/healthz`` and ``/ping`` will not be automatically traced. +will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``. Request attributes ****************** -To extract certain attributes from Tornado's request object and use them as span attributes, set the environment variable ``OTEL_PYTHON_TORNADO_TRACED_REQUEST_ATTRS`` to a comma -delimited list of request attribute names. +To extract attributes from Tornado's request object and use them as span attributes, set the environment variable +``OTEL_PYTHON_TORNADO_TRACED_REQUEST_ATTRS`` to a comma delimited list of request attribute names. For example, @@ -61,14 +63,14 @@ def get(self): export OTEL_PYTHON_TORNADO_TRACED_REQUEST_ATTRS='uri,query' -will extract path_info and content_type attributes from every traced request and add them as span attributes. +will extract the ``uri`` and ``query`` attributes from every traced request and add them as span attributes. Request/Response hooks ********************** Tornado instrumentation supports extending tracing behaviour with the help of hooks. -Its ``instrument()`` method accepts three optional functions that get called back with the -created span and some other contextual information. Example: +Its ``instrument()`` method accepts three optional functions that get called with the +created span and some other contextual information. Examples: .. code-block:: python @@ -84,75 +86,115 @@ def server_request_hook(span, handler): def client_request_hook(span, request): pass - # will be called after a outgoing request made with + # will be called after an outgoing request made with # `tornado.httpclient.AsyncHTTPClient.fetch` finishes. # `response`` is an instance of ``Future[tornado.httpclient.HTTPResponse]`. - def client_resposne_hook(span, future): + def client_response_hook(span, future): pass # apply tornado instrumentation with hooks TornadoInstrumentor().instrument( server_request_hook=server_request_hook, client_request_hook=client_request_hook, - client_response_hook=client_resposne_hook + client_response_hook=client_response_hook ) Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in tornado are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Request header names in Tornado are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -Example of the added span attribute, + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" + +Would match all request headers that start with ``Accept`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" + +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in Tornado are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in tornado are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" -Example of the added span attribute, +Would match all response headers that start with ``Content`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- """ +import re from collections import namedtuple from functools import partial from logging import getLogger @@ -181,8 +223,10 @@ def client_resposne_hook(span, future): from opentelemetry.trace.status import Status, StatusCode from opentelemetry.util._time import _time_ns from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, + SanitizeValue, get_custom_headers, get_excluded_urls, get_traced_request_attrs, @@ -316,28 +360,76 @@ def _log_exception(tracer, func, handler, args, kwargs): def _collect_custom_request_headers_attributes(request_headers): + attributes = {} + + sanitized_fields = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + + s = SanitizeValue(sanitized_fields) + custom_request_headers_name = get_custom_headers( OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST ) - attributes = {} - for header_name in custom_request_headers_name: - header_values = request_headers.get(header_name) - if header_values: - key = normalise_request_header_name(header_name.lower()) - attributes[key] = [header_values] + + if custom_request_headers_name: + custom_request_headers_regex_compiled = re.compile( + "|".join("^" + i + "$" for i in custom_request_headers_name), + re.IGNORECASE, + ) + + for header_name in list( + filter( + custom_request_headers_regex_compiled.match, + request_headers.keys(), + ) + ): + header_values = request_headers.get(header_name) + if header_values: + key = normalise_request_header_name(header_name.lower()) + attributes[key] = [ + s.sanitize_header_value( + header=header_name, value=header_values + ) + ] + return attributes def _collect_custom_response_headers_attributes(response_headers): + attributes = {} + + sanitized_fields = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + + s = SanitizeValue(sanitized_fields) + custom_response_headers_name = get_custom_headers( OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE ) - attributes = {} - for header_name in custom_response_headers_name: - header_values = response_headers.get(header_name) - if header_values: - key = normalise_response_header_name(header_name.lower()) - attributes[key] = [header_values] + + if custom_response_headers_name: + custom_response_headers_regex_compiled = re.compile( + "|".join("^" + i + "$" for i in custom_response_headers_name), + re.IGNORECASE, + ) + + for header_name in list( + filter( + custom_response_headers_regex_compiled.match, + response_headers.keys(), + ) + ): + header_values = response_headers.get(header_name.lower()) + if header_values: + key = normalise_response_header_name(header_name) + attributes[key] = [ + s.sanitize_header_value( + header=header_name, value=header_values + ) + ] + return attributes diff --git a/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py b/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py index 8dcc94b683..90f46d5f69 100644 --- a/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py +++ b/instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py @@ -33,6 +33,7 @@ from opentelemetry.test.wsgitestutil import WsgiTestBase from opentelemetry.trace import SpanKind from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, get_excluded_urls, @@ -650,13 +651,17 @@ class TestTornadoCustomRequestResponseHeadersAddedWithServerSpan(TornadoTest): @patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3" + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", }, ) def test_custom_request_headers_added_in_server_span(self): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } response = self.fetch("/", headers=headers) self.assertEqual(response.code, 201) @@ -668,6 +673,11 @@ def test_custom_request_headers_added_in_server_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(tornado_span.kind, trace.SpanKind.SERVER) self.assertSpanHasAttributes(tornado_span, expected) @@ -675,7 +685,8 @@ def test_custom_request_headers_added_in_server_span(self): @patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header" + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", }, ) def test_custom_response_headers_added_in_server_span(self): @@ -692,6 +703,79 @@ def test_custom_response_headers_added_in_server_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + self.assertEqual(tornado_span.kind, trace.SpanKind.SERVER) + self.assertSpanHasAttributes(tornado_span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_headers_added_in_server_span_all(self): + headers = { + "Custom-Test-Header-1": "Test Value 1", + "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", + } + response = self.fetch("/", headers=headers) + self.assertEqual(response.code, 201) + _, tornado_span, _ = self.sorted_spans( + self.memory_exporter.get_finished_spans() + ) + expected = { + "http.request.header.custom_test_header_1": ("Test Value 1",), + "http.request.header.custom_test_header_2": ( + "TestValue2,TestValue3", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + self.assertEqual(tornado_span.kind, trace.SpanKind.SERVER) + self.assertSpanHasAttributes(tornado_span, expected) + + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_headers_added_in_server_span_all(self): + response = self.fetch("/test_custom_response_headers") + self.assertEqual(response.code, 200) + tornado_span, _ = self.sorted_spans( + self.memory_exporter.get_finished_spans() + ) + expected = { + "http.response.header.content_type": ( + "text/plain; charset=utf-8", + ), + "http.response.header.content_length": ("0",), + "http.response.header.my_custom_header": ( + "my-custom-value-1,my-custom-header-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(tornado_span.kind, trace.SpanKind.SERVER) self.assertSpanHasAttributes(tornado_span, expected) @@ -716,13 +800,17 @@ def middleware(request): @patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3" + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", }, ) def test_custom_request_headers_not_added_in_internal_span(self): headers = { "Custom-Test-Header-1": "Test Value 1", "Custom-Test-Header-2": "TestValue2,TestValue3", + "Regex-Test-Header-1": "Regex Test Value 1", + "regex-test-header-2": "RegexTestValue2,RegexTestValue3", + "My-Secret-Header": "My Secret Value", } response = self.fetch("/", headers=headers) self.assertEqual(response.code, 201) @@ -734,6 +822,11 @@ def test_custom_request_headers_not_added_in_internal_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(tornado_span.kind, trace.SpanKind.INTERNAL) for key, _ in not_expected.items(): @@ -742,7 +835,8 @@ def test_custom_request_headers_not_added_in_internal_span(self): @patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header" + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", }, ) def test_custom_response_headers_not_added_in_internal_span(self): @@ -759,6 +853,13 @@ def test_custom_response_headers_not_added_in_internal_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } self.assertEqual(tornado_span.kind, trace.SpanKind.INTERNAL) for key, _ in not_expected.items(): diff --git a/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py b/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py index 9e84c74aca..6790881f5c 100644 --- a/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py +++ b/instrumentation/opentelemetry-instrumentation-tornado/tests/tornado_test_app.py @@ -102,6 +102,15 @@ def get(self): self.set_header( "my-custom-header", "my-custom-value-1,my-custom-header-2" ) + self.set_header( + "my-custom-regex-header-1", + "my-custom-regex-value-1,my-custom-regex-value-2", + ) + self.set_header( + "My-Custom-Regex-Header-2", + "my-custom-regex-value-3,my-custom-regex-value-4", + ) + self.set_header("my-secret-header", "my-secret-value") self.set_status(200) diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py index 11c5acf643..488c213339 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py @@ -85,8 +85,15 @@ def GET(self): Request/Response hooks ********************** -Utilize request/reponse hooks to execute custom logic to be performed before/after performing a request. Environ is an instance of WSGIEnvironment. -Response_headers is a list of key-value (tuples) representing the response headers returned from the response. +This instrumentation supports request and response hooks. These are functions that get called +right after a span is created for a request and right before the span is finished for the response. + +- The client request hook is called with the internal span and an instance of WSGIEnvironment when the method + ``receive`` is called. +- The client response hook is called with the internal span, the status of the response and a list of key-value (tuples) + representing the response headers returned from the response when the method ``send`` is called. + +For example, .. code-block:: python @@ -102,60 +109,100 @@ def response_hook(span: Span, environ: WSGIEnvironment, status: str, response_he Capture HTTP request and response headers ***************************************** -You can configure the agent to capture predefined HTTP headers as span attributes, according to the `semantic convention `_. +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic convention `_. Request headers *************** -To capture predefined HTTP request headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` -to a comma-separated list of HTTP header names. +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header" -will extract ``content-type`` and ``custom_request_header`` from request headers and add them as span attributes. +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in WSGI are case-insensitive and ``-`` characters are replaced by ``_``. So, giving the header +name as ``CUStom_Header`` in the environment variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="Accept.*,X-.*" + +Would match all request headers that start with ``Accept`` and ``X-``. + +Additionally, the special keyword ``all`` can be used to capture all request headers. +:: -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Request header names in wsgi are case insensitive and - characters are replaced by _. So, giving header name as ``CUStom_Header`` in environment variable will be able capture header with name ``custom-header``. + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="all" -The name of the added span attribute will follow the format ``http.request.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. -Example of the added span attribute, +For example: ``http.request.header.custom_request_header = [","]`` Response headers **************** -To capture predefined HTTP response headers as span attributes, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` -to a comma-separated list of HTTP header names. +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE`` to a comma delimited list of HTTP header names. For example, - :: export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header" -will extract ``content-type`` and ``custom_response_header`` from response headers and add them as span attributes. +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in WSGI are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="Content.*,X-.*" -It is recommended that you should give the correct names of the headers to be captured in the environment variable. -Response header names captured in wsgi are case insensitive. So, giving header name as ``CUStomHeader`` in environment variable will be able capture header with name ``customheader``. +Would match all response headers that start with ``Content`` and ``X-``. -The name of the added span attribute will follow the format ``http.response.header.`` where ```` being the normalized HTTP header name (lowercase, with - characters replaced by _ ). -The value of the attribute will be single item list containing all the header values. +Additionally, the special keyword ``all`` can be used to capture all response headers. +:: -Example of the added span attribute, + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="all" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: ``http.response.header.custom_response_header = [","]`` +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. Regexes may be used, and all header names will be +matched in a case-insensitive manner. + +For example, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + Note: - Environment variable names to capture http headers are still experimental, and thus are subject to change. + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. API --- """ import functools +import re import typing import wsgiref.util as wsgiref_util from timeit import default_timer @@ -171,8 +218,10 @@ def response_hook(span: Span, environ: WSGIEnvironment, status: str, response_he from opentelemetry.semconv.trace import SpanAttributes from opentelemetry.trace.status import Status, StatusCode from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, + SanitizeValue, get_custom_headers, normalise_request_header_name, normalise_response_header_name, @@ -293,15 +342,42 @@ def collect_custom_request_headers_attributes(environ): from the PEP3333-conforming WSGI environ to be used as span creation attributes as described in the specification https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-request-and-response-headers""" attributes = {} + + sanitized_fields = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + + s = SanitizeValue( + [_CARRIER_KEY_PREFIX + i.replace("-", "_") for i in sanitized_fields] + ) + custom_request_headers_name = get_custom_headers( OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST ) - for header_name in custom_request_headers_name: - wsgi_env_var = header_name.upper().replace("-", "_") - header_values = environ.get(f"HTTP_{wsgi_env_var}") - if header_values: - key = normalise_request_header_name(header_name) - attributes[key] = [header_values] + + if custom_request_headers_name: + custom_request_headers_regex_compiled = re.compile( + "|".join( + "^" + _CARRIER_KEY_PREFIX + i.replace("-", "_") + "$" + for i in custom_request_headers_name + ), + re.IGNORECASE, + ) + + for header_name in list( + filter(custom_request_headers_regex_compiled.match, environ.keys()) + ): + header_values = environ.get(header_name) + if header_values: + key = normalise_request_header_name( + header_name[_CARRIER_KEY_PREFIX_LEN:] + ) + attributes[key] = [ + s.sanitize_header_value( + header=header_name, value=header_values + ) + ] + return attributes @@ -310,19 +386,43 @@ def collect_custom_response_headers_attributes(response_headers): PEP3333-conforming WSGI environ as described in the specification https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-request-and-response-headers""" attributes = {} + + sanitized_fields = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + + s = SanitizeValue(sanitized_fields) + custom_response_headers_name = get_custom_headers( OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE ) + response_headers_dict = {} if response_headers: for header_name, header_value in response_headers: response_headers_dict[header_name.lower()] = header_value - for header_name in custom_response_headers_name: - header_values = response_headers_dict.get(header_name.lower()) - if header_values: - key = normalise_response_header_name(header_name) - attributes[key] = [header_values] + if custom_response_headers_name: + custom_response_headers_regex_compiled = re.compile( + "|".join("^" + i + "$" for i in custom_response_headers_name), + re.IGNORECASE, + ) + + for header_name in list( + filter( + custom_response_headers_regex_compiled.match, + response_headers_dict.keys(), + ) + ): + header_values = response_headers_dict.get(header_name.lower()) + if header_values: + key = normalise_response_header_name(header_name) + attributes[key] = [ + s.sanitize_header_value( + header=header_name, value=header_values + ) + ] + return attributes diff --git a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py index 7bdfabc37f..d8a61f9735 100644 --- a/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-wsgi/tests/test_wsgi_middleware.py @@ -30,6 +30,7 @@ from opentelemetry.test.wsgitestutil import WsgiTestBase from opentelemetry.trace import StatusCode from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, ) @@ -98,6 +99,15 @@ def wsgi_with_custom_response_headers(environ, start_response): ("content-type", "text/plain; charset=utf-8"), ("content-length", "100"), ("my-custom-header", "my-custom-value-1,my-custom-header-2"), + ( + "my-custom-regex-header-1", + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + ( + "My-Custom-Regex-Header-2", + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + ("My-Secret-Header", "My Secret Value"), ], ) return [b"*"] @@ -522,7 +532,8 @@ def iterate_response(self, response): @mock.patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", }, ) def test_custom_request_headers_non_recording_span(self): @@ -532,6 +543,9 @@ def test_custom_request_headers_non_recording_span(self): { "HTTP_CUSTOM_TEST_HEADER_1": "Test Value 2", "HTTP_CUSTOM_TEST_HEADER_2": "TestValue2,TestValue3", + "HTTP_REGEX_TEST_HEADER_1": "Regex Test Value 1", + "HTTP_REGEX_TEST_HEADER_2": "RegexTestValue2,RegexTestValue3", + "HTTP_MY_SECRET_HEADER": "My Secret Value", } ) app = otel_wsgi.OpenTelemetryMiddleware( @@ -545,7 +559,8 @@ def test_custom_request_headers_non_recording_span(self): @mock.patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3" + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "Custom-Test-Header-1,Custom-Test-Header-2,Custom-Test-Header-3,Regex-Test-Header-.*,Regex-Invalid-Test-Header-.*,.*my-secret.*", }, ) def test_custom_request_headers_added_in_server_span(self): @@ -553,6 +568,9 @@ def test_custom_request_headers_added_in_server_span(self): { "HTTP_CUSTOM_TEST_HEADER_1": "Test Value 1", "HTTP_CUSTOM_TEST_HEADER_2": "TestValue2,TestValue3", + "HTTP_REGEX_TEST_HEADER_1": "Regex Test Value 1", + "HTTP_REGEX_TEST_HEADER_2": "RegexTestValue2,RegexTestValue3", + "HTTP_MY_SECRET_HEADER": "My Secret Value", } ) app = otel_wsgi.OpenTelemetryMiddleware(simple_wsgi) @@ -564,6 +582,11 @@ def test_custom_request_headers_added_in_server_span(self): "http.request.header.custom_test_header_2": ( "TestValue2,TestValue3", ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), } self.assertSpanHasAttributes(span, expected) @@ -596,7 +619,8 @@ def test_custom_request_headers_not_added_in_internal_span(self): @mock.patch.dict( "os.environ", { - OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header" + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "content-type,content-length,my-custom-header,invalid-header,my-custom-regex-header-.*,invalid-regex-header-.*,.*my-secret.*", }, ) def test_custom_response_headers_added_in_server_span(self): @@ -614,6 +638,13 @@ def test_custom_response_headers_added_in_server_span(self): "http.response.header.my_custom_header": ( "my-custom-value-1,my-custom-header-2", ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), } self.assertSpanHasAttributes(span, expected) @@ -641,6 +672,72 @@ def test_custom_response_headers_not_added_in_internal_span(self): for key, _ in not_expected.items(): self.assertNotIn(key, span.attributes) + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: "all", + }, + ) + def test_custom_request_headers_added_in_server_span_all(self): + self.environ.update( + { + "HTTP_CUSTOM_TEST_HEADER_1": "Test Value 1", + "HTTP_CUSTOM_TEST_HEADER_2": "TestValue2,TestValue3", + "HTTP_REGEX_TEST_HEADER_1": "Regex Test Value 1", + "HTTP_REGEX_TEST_HEADER_2": "RegexTestValue2,RegexTestValue3", + "HTTP_MY_SECRET_HEADER": "My Secret Value", + } + ) + app = otel_wsgi.OpenTelemetryMiddleware(simple_wsgi) + response = app(self.environ, self.start_response) + self.iterate_response(response) + span = self.memory_exporter.get_finished_spans()[0] + expected = { + "http.request.header.custom_test_header_1": ("Test Value 1",), + "http.request.header.custom_test_header_2": ( + "TestValue2,TestValue3", + ), + "http.request.header.regex_test_header_1": ("Regex Test Value 1",), + "http.request.header.regex_test_header_2": ( + "RegexTestValue2,RegexTestValue3", + ), + "http.request.header.my_secret_header": ("[REDACTED]",), + } + self.assertSpanHasAttributes(span, expected) + + @mock.patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: ".*my-secret.*", + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: "all", + }, + ) + def test_custom_response_headers_added_in_server_span_all(self): + app = otel_wsgi.OpenTelemetryMiddleware( + wsgi_with_custom_response_headers + ) + response = app(self.environ, self.start_response) + self.iterate_response(response) + span = self.memory_exporter.get_finished_spans()[0] + expected = { + "http.response.header.content_type": ( + "text/plain; charset=utf-8", + ), + "http.response.header.content_length": ("100",), + "http.response.header.my_custom_header": ( + "my-custom-value-1,my-custom-header-2", + ), + "http.response.header.my_custom_regex_header_1": ( + "my-custom-regex-value-1,my-custom-regex-value-2", + ), + "http.response.header.my_custom_regex_header_2": ( + "my-custom-regex-value-3,my-custom-regex-value-4", + ), + "http.response.header.my_secret_header": ("[REDACTED]",), + } + self.assertSpanHasAttributes(span, expected) + if __name__ == "__main__": unittest.main() diff --git a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py index aa34fb439a..1b7b2efc50 100644 --- a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py +++ b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py @@ -13,11 +13,15 @@ # limitations under the License. from os import environ +from re import IGNORECASE as RE_IGNORECASE from re import compile as re_compile from re import search from typing import Iterable, List from urllib.parse import urlparse, urlunparse +OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS = ( + "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS" +) OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST = ( "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST" ) @@ -38,6 +42,22 @@ def url_disabled(self, url: str) -> bool: return bool(self._excluded_urls and search(self._regex, url)) +class SanitizeValue: + """Class to sanitize (remove sensitive data from) certain headers (given as a list of regexes)""" + + def __init__(self, sanitized_fields: Iterable[str]): + self._sanitized_fields = sanitized_fields + if self._sanitized_fields: + self._regex = re_compile("|".join(sanitized_fields), RE_IGNORECASE) + + def sanitize_header_value(self, header: str, value: str) -> str: + return ( + "[REDACTED]" + if (self._sanitized_fields and search(self._regex, header)) + else value + ) + + _root = r"OTEL_PYTHON_{}" @@ -68,7 +88,7 @@ def get_excluded_urls(instrumentation: str) -> ExcludeList: def parse_excluded_urls(excluded_urls: str) -> ExcludeList: """ - Small helper to put an arbitrary url list inside of ExcludeList + Small helper to put an arbitrary url list inside an ExcludeList """ if excluded_urls: excluded_url_list = [ @@ -120,8 +140,11 @@ def normalise_response_header_name(header: str) -> str: def get_custom_headers(env_var: str) -> List[str]: custom_headers = environ.get(env_var, []) if custom_headers: - custom_headers = [ - custom_headers.strip() - for custom_headers in custom_headers.split(",") - ] + if custom_headers.strip().lower() == "all": + custom_headers = [".*"] + else: + custom_headers = [ + custom_headers.strip() + for custom_headers in custom_headers.split(",") + ] return custom_headers diff --git a/util/opentelemetry-util-http/tests/test_capture_custom_headers.py b/util/opentelemetry-util-http/tests/test_capture_custom_headers.py index eb1a4f6a7e..9b1b26d298 100644 --- a/util/opentelemetry-util-http/tests/test_capture_custom_headers.py +++ b/util/opentelemetry-util-http/tests/test_capture_custom_headers.py @@ -16,6 +16,7 @@ from opentelemetry.test.test_base import TestBase from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE, get_custom_headers, @@ -58,6 +59,21 @@ def test_get_custom_response_header(self): ], ) + @patch.dict( + "os.environ", + { + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS: "My-Secret-Header,My-Secret-Header-2" + }, + ) + def test_get_custom_sanitize_header(self): + custom_headers_to_capture = get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ) + self.assertEqual( + custom_headers_to_capture, + ["My-Secret-Header", "My-Secret-Header-2"], + ) + def test_normalise_request_header_name(self): key = normalise_request_header_name("Test-Header") self.assertEqual(key, "http.request.header.test_header")