Support log reporter safe mode

.github/workflows/CI.yaml

@@ -54,6 +54,7 @@ jobs:
     outputs:
       agent: ${{ steps.filter.outputs.agent }}
     steps:
+      - uses: actions/checkout@v3  # required for push event
       - name: Check for file changes
         uses: getsentry/paths-filter@v2
         id: filter

CHANGELOG.md

@@ -18,15 +18,16 @@
   - Fix local log stack depth overridden by agent log formatter (#192)
   - Fix typo that cause user sitecustomize.py not loaded (#193)
   - Fix instance property wrongly shown as UNKNOWN in OAP (#194)
-  - Fix multiple components inconsistently named on SkyWalking UI (#TBD)
+  - Fix multiple components inconsistently named on SkyWalking UI (#199)
   - Fix SW_AGENT_LOGGING_LEVEL not properly set during startup (#196)
 
 - Docs:
   - Add a FAQ doc on `how to use with uwsgi` (#188)
 
 - Others:
   - Add support for Python 3.10 (#167)
-  - Add E2E test coverage (#TBD)
+  - Add E2E test coverage for trace and logging (#199)
+  - Add a `SW_AGENT_LOG_REPORTER_SAFE_MODE` option to control the HTTP basic auth credential filter (#TBD)
 
 ### 0.7.0
 

skywalking/config.py

@@ -17,77 +17,73 @@
 import os
 import re
 import uuid
-from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
-    from typing import List
+# Change to future after Python3.6 support ends
+from typing import List, Pattern
 
-QUEUE_TIMEOUT = 1  # type: int
+QUEUE_TIMEOUT: int = 1
 
-RE_IGNORE_PATH = re.compile('^$')  # type: re.Pattern
-RE_HTTP_IGNORE_METHOD = RE_IGNORE_PATH  # type: re.Pattern
+RE_IGNORE_PATH: Pattern = re.compile('^$')
+RE_HTTP_IGNORE_METHOD: Pattern = RE_IGNORE_PATH
 
 options = None  # here to include 'options' in globals
 options = globals().copy()  # THIS MUST PRECEDE DIRECTLY BEFORE LIST OF CONFIG OPTIONS!
 
-service_name = os.getenv('SW_AGENT_NAME') or 'Python Service Name'  # type: str
-service_instance = os.getenv('SW_AGENT_INSTANCE') or str(uuid.uuid1()).replace('-', '')  # type: str
-agent_namespace = os.getenv('SW_AGENT_NAMESPACE')  # type: str
-collector_address = os.getenv('SW_AGENT_COLLECTOR_BACKEND_SERVICES') or '127.0.0.1:11800'  # type: str
-force_tls = os.getenv('SW_AGENT_FORCE_TLS', '').lower() == 'true'  # type: bool
-protocol = (os.getenv('SW_AGENT_PROTOCOL') or 'grpc').lower()  # type: str
-authentication = os.getenv('SW_AGENT_AUTHENTICATION')  # type: str
-logging_level = os.getenv('SW_AGENT_LOGGING_LEVEL') or 'INFO'  # type: str
-disable_plugins = (os.getenv('SW_AGENT_DISABLE_PLUGINS') or '').split(',')  # type: List[str]
-max_buffer_size = int(os.getenv('SW_AGENT_MAX_BUFFER_SIZE', '10000'))  # type: int
-sql_parameters_length = int(os.getenv('SW_SQL_PARAMETERS_LENGTH') or '0')  # type: int
-pymongo_trace_parameters = True if os.getenv('SW_PYMONGO_TRACE_PARAMETERS') and \
-                                   os.getenv('SW_PYMONGO_TRACE_PARAMETERS') == 'True' else False  # type: bool
-pymongo_parameters_max_length = int(os.getenv('SW_PYMONGO_PARAMETERS_MAX_LENGTH') or '512')  # type: int
-ignore_suffix = os.getenv('SW_IGNORE_SUFFIX') or '.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,' \
-                                                 '.mp4,.html,.svg '  # type: str
-flask_collect_http_params = True if os.getenv('SW_FLASK_COLLECT_HTTP_PARAMS') and \
-                                    os.getenv('SW_FLASK_COLLECT_HTTP_PARAMS') == 'True' else False  # type: bool
-sanic_collect_http_params = True if os.getenv('SW_SANIC_COLLECT_HTTP_PARAMS') and \
-                                    os.getenv('SW_SANIC_COLLECT_HTTP_PARAMS') == 'True' else False  # type: bool
-http_params_length_threshold = int(os.getenv('SW_HTTP_PARAMS_LENGTH_THRESHOLD') or '1024')  # type: int
-http_ignore_method = os.getenv('SW_HTTP_IGNORE_METHOD', '').upper()  # type: str
-django_collect_http_params = True if os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') and \
-                                     os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') == 'True' else False  # type: bool
-correlation_element_max_number = int(os.getenv('SW_CORRELATION_ELEMENT_MAX_NUMBER') or '3')  # type: int
-correlation_value_max_length = int(os.getenv('SW_CORRELATION_VALUE_MAX_LENGTH') or '128')  # type: int
-trace_ignore_path = os.getenv('SW_TRACE_IGNORE_PATH') or ''  # type: str
-elasticsearch_trace_dsl = True if os.getenv('SW_ELASTICSEARCH_TRACE_DSL') and \
-                                  os.getenv('SW_ELASTICSEARCH_TRACE_DSL') == 'True' else False  # type: bool
-kafka_bootstrap_servers = os.getenv('SW_KAFKA_REPORTER_BOOTSTRAP_SERVERS') or 'localhost:9092'  # type: str
-kafka_topic_management = os.getenv('SW_KAFKA_REPORTER_TOPIC_MANAGEMENT') or 'skywalking-managements'  # type: str
-kafka_topic_segment = os.getenv('SW_KAFKA_REPORTER_TOPIC_SEGMENT') or 'skywalking-segments'  # type: str
-kafka_topic_log = os.getenv('SW_KAFKA_REPORTER_TOPIC_LOG') or 'skywalking-logs'  # type: str
-celery_parameters_length = int(os.getenv('SW_CELERY_PARAMETERS_LENGTH') or '512')
-fastapi_collect_http_params = True if os.getenv('SW_FASTAPI_COLLECT_HTTP_PARAMS') and \
-                                      os.getenv('SW_FASTAPI_COLLECT_HTTP_PARAMS') == 'True' else False  # type: bool
-
-# profile configs
-get_profile_task_interval = int(os.getenv('SW_PROFILE_TASK_QUERY_INTERVAL') or '20')  # type: int
-profile_active = False if os.getenv('SW_AGENT_PROFILE_ACTIVE') and \
-                          os.getenv('SW_AGENT_PROFILE_ACTIVE') == 'False' else True  # type: bool
-profile_max_parallel = int(os.getenv('SW_AGENT_PROFILE_MAX_PARALLEL') or '5')  # type: int
-profile_duration = int(os.getenv('SW_AGENT_PROFILE_DURATION') or '10')  # type: int
-profile_dump_max_stack_depth = int(os.getenv('SW_AGENT_PROFILE_DUMP_MAX_STACK_DEPTH') or '500')  # type: int
-profile_snapshot_transport_buffer_size = int(os.getenv('SW_AGENT_PROFILE_SNAPSHOT_TRANSPORT_BUFFER_SIZE') or '50')
-
-log_reporter_active = True if os.getenv('SW_AGENT_LOG_REPORTER_ACTIVE') and \
-                              os.getenv('SW_AGENT_LOG_REPORTER_ACTIVE') == 'True' else False  # type: bool
-log_reporter_max_buffer_size = int(os.getenv('SW_AGENT_LOG_REPORTER_BUFFER_SIZE') or '10000')  # type: int
-log_reporter_level = os.getenv('SW_AGENT_LOG_REPORTER_LEVEL') or 'WARNING'  # type: str
-log_reporter_ignore_filter = True if os.getenv('SW_AGENT_LOG_REPORTER_IGNORE_FILTER') and \
-                                     os.getenv('SW_AGENT_LOG_REPORTER_IGNORE_FILTER') == 'True' else False  # type: bool
-log_reporter_formatted = False if os.getenv('SW_AGENT_LOG_REPORTER_FORMATTED') and \
-                                  os.getenv('SW_AGENT_LOG_REPORTER_FORMATTED') == 'False' else True  # type: bool
-log_reporter_layout = os.getenv('SW_AGENT_LOG_REPORTER_LAYOUT') or \
-                      '%(asctime)s [%(threadName)s] %(levelname)s %(name)s - %(message)s'  # type: str
-cause_exception_depth = int(os.getenv('SW_AGENT_CAUSE_EXCEPTION_DEPTH') or '5')  # type: int
-
+# Core level configurations
+service_name: str = os.getenv('SW_AGENT_NAME') or 'Python Service Name'
+service_instance: str = os.getenv('SW_AGENT_INSTANCE') or str(uuid.uuid1()).replace('-', '')
+agent_namespace: str = os.getenv('SW_AGENT_NAMESPACE')
+collector_address: str = os.getenv('SW_AGENT_COLLECTOR_BACKEND_SERVICES') or '127.0.0.1:11800'
+kafka_bootstrap_servers: str = os.getenv('SW_KAFKA_REPORTER_BOOTSTRAP_SERVERS') or 'localhost:9092'
+kafka_topic_management: str = os.getenv('SW_KAFKA_REPORTER_TOPIC_MANAGEMENT') or 'skywalking-managements'
+kafka_topic_segment: str = os.getenv('SW_KAFKA_REPORTER_TOPIC_SEGMENT') or 'skywalking-segments'
+kafka_topic_log: str = os.getenv('SW_KAFKA_REPORTER_TOPIC_LOG') or 'skywalking-logs'
+force_tls: bool = os.getenv('SW_AGENT_FORCE_TLS', '').lower() == 'true'
+protocol: str = (os.getenv('SW_AGENT_PROTOCOL') or 'grpc').lower()
+authentication: str = os.getenv('SW_AGENT_AUTHENTICATION')
+logging_level: str = os.getenv('SW_AGENT_LOGGING_LEVEL') or 'INFO'
+disable_plugins: List[str] = (os.getenv('SW_AGENT_DISABLE_PLUGINS') or '').split(',')
+max_buffer_size: int = int(os.getenv('SW_AGENT_MAX_BUFFER_SIZE', '10000'))
+trace_ignore_path: str = os.getenv('SW_TRACE_IGNORE_PATH') or ''
+ignore_suffix: str = os.getenv('SW_IGNORE_SUFFIX') or '.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,' \
+                                                      '.mp4,.html,.svg '
+correlation_element_max_number: int = int(os.getenv('SW_CORRELATION_ELEMENT_MAX_NUMBER') or '3')
+correlation_value_max_length: int = int(os.getenv('SW_CORRELATION_VALUE_MAX_LENGTH') or '128')
+
+# Plugin configurations
+sql_parameters_length: int = int(os.getenv('SW_SQL_PARAMETERS_LENGTH') or '0')
+pymongo_trace_parameters: bool = os.getenv('SW_PYMONGO_TRACE_PARAMETERS') == 'True'
+pymongo_parameters_max_length: int = int(os.getenv('SW_PYMONGO_PARAMETERS_MAX_LENGTH') or '512')
+elasticsearch_trace_dsl: bool = os.getenv('SW_ELASTICSEARCH_TRACE_DSL') == 'True'
+
+http_params_length_threshold: int = int(os.getenv('SW_HTTP_PARAMS_LENGTH_THRESHOLD') or '1024')
+http_ignore_method: str = os.getenv('SW_HTTP_IGNORE_METHOD', '').upper()
+flask_collect_http_params: bool = os.getenv('SW_FLASK_COLLECT_HTTP_PARAMS') == 'True'
+sanic_collect_http_params: bool = os.getenv('SW_SANIC_COLLECT_HTTP_PARAMS') == 'True'
+django_collect_http_params: bool = os.getenv('SW_DJANGO_COLLECT_HTTP_PARAMS') == 'True'
+fastapi_collect_http_params: bool = os.getenv('SW_FASTAPI_COLLECT_HTTP_PARAMS') == 'True'
+
+celery_parameters_length: int = int(os.getenv('SW_CELERY_PARAMETERS_LENGTH') or '512')
+
+# profiling configurations
+get_profile_task_interval: int = int(os.getenv('SW_PROFILE_TASK_QUERY_INTERVAL') or '20')
+profile_active: bool = os.getenv('SW_AGENT_PROFILE_ACTIVE') != 'False'
+profile_max_parallel: int = int(os.getenv('SW_AGENT_PROFILE_MAX_PARALLEL') or '5')
+profile_duration: int = int(os.getenv('SW_AGENT_PROFILE_DURATION') or '10')
+profile_dump_max_stack_depth: int = int(os.getenv('SW_AGENT_PROFILE_DUMP_MAX_STACK_DEPTH') or '500')
+profile_snapshot_transport_buffer_size: int = int(os.getenv('SW_AGENT_PROFILE_SNAPSHOT_TRANSPORT_BUFFER_SIZE') or '50')
+
+# log reporter configurations
+log_reporter_active: bool = os.getenv('SW_AGENT_LOG_REPORTER_ACTIVE') == 'True'
+log_reporter_safe_mode: bool = os.getenv('SW_AGENT_LOG_REPORTER_SAFE_MODE') == 'True'
+log_reporter_max_buffer_size: int = int(os.getenv('SW_AGENT_LOG_REPORTER_BUFFER_SIZE') or '10000')
+log_reporter_level: str = os.getenv('SW_AGENT_LOG_REPORTER_LEVEL') or 'WARNING'
+log_reporter_ignore_filter: bool = os.getenv('SW_AGENT_LOG_REPORTER_IGNORE_FILTER') == 'True'
+log_reporter_formatted: bool = os.getenv('SW_AGENT_LOG_REPORTER_FORMATTED') != 'False'
+log_reporter_layout: str = os.getenv('SW_AGENT_LOG_REPORTER_LAYOUT') or \
+                           '%(asctime)s [%(threadName)s] %(levelname)s %(name)s - %(message)s'
+# This configuration is shared by log reporter and tracer
+cause_exception_depth: int = int(os.getenv('SW_AGENT_CAUSE_EXCEPTION_DEPTH') or '5')
 
 options = {key for key in globals() if key not in options}  # THIS MUST FOLLOW DIRECTLY AFTER LIST OF CONFIG OPTIONS!
 

skywalking/utils/filter.py

@@ -36,8 +36,10 @@ def sw_urlparse(url):
 
 def sw_filter(target: str):
     # Remove user:pw from any valid full urls
-
-    return re.sub(r'://(.*?)@', r'://', target)
+    # this filter is disabled by default due to perf impact
+    if config.log_reporter_safe_mode:
+        return re.sub(r'://(.*?)@', r'://', target)
+    return target
 
 
 def sw_traceback():

tests/e2e/base/docker-compose.base.yml

@@ -45,6 +45,7 @@ services:
       SW_AGENT_LOGGING_LEVEL: DEBUG
       SW_AGENT_LOG_REPORTER_ACTIVE: "True"
       SW_AGENT_LOG_REPORTER_LEVEL: WARNING
+      SW_AGENT_LOG_REPORTER_SAFE_MODE: "True"
     healthcheck:
       test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090" ]
       interval: 5s
@@ -63,8 +64,6 @@ services:
       SW_AGENT_NAME: e2e-service-consumer
       SW_AGENT_INSTANCE: consumer1
       SW_AGENT_LOGGING_LEVEL: DEBUG
-      SW_AGENT_LOG_REPORTER_ACTIVE: "True"
-      SW_AGENT_LOG_REPORTER_LEVEL: WARNING
     healthcheck:
       test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090" ]
       interval: 5s

tests/e2e/base/provider/provider.py

@@ -82,6 +82,9 @@ async def application():
     time.sleep(0.5)
     # Warning is reported
     e2e_provider_logger.warning('E2E Provider Warning, this is reported!')
+    time.sleep(0.5)
+    # The user:password part will be removed
+    e2e_provider_logger.warning('Leak basic auth info at https://user:password@example.com')
     # Debug is not reported according to default agent setting
     e2e_provider_logger.debug('E2E Provider Debug, this is not reported!')
 

tests/e2e/case/expected/logs-list.yml

@@ -15,6 +15,26 @@
 
 logs:
   {{- contains .logs }}
+  - servicename: e2e-service-provider
+    serviceid: {{ b64enc "e2e-service-provider" }}.1
+    serviceinstancename: provider1
+    serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }}
+    endpointname: null
+    endpointid: null
+    traceid: {{ notEmpty .traceid }}
+    timestamp: {{ gt .timestamp 0 }}
+    contenttype: TEXT
+    content: {{ regexp .content "(?s).+Leak basic auth info at https://example.com.+"}}
+
+    tags:
+    {{- contains .tags }}
+    - key: level
+      value: WARNING
+    - key: logger
+      value: {{ notEmpty .value }}
+    - key: thread
+      value: {{ notEmpty .value }}
+    {{- end }}
   - servicename: e2e-service-provider
     serviceid: {{ b64enc "e2e-service-provider" }}.1
     serviceinstancename: provider1
@@ -54,4 +74,4 @@ logs:
       value: {{ notEmpty .value }}
     {{- end }}
   {{- end }}
-total: 2
+total: 3