diff --git a/CODEOWNERS b/CODEOWNERS index 1b7fcd94557b..99cda541b1cc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -50,6 +50,8 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/tracers/datadog @cgilmour @palazzem @mattklein123 # tracers.xray extension /*/extensions/tracers/xray @marcomagdy @lavignes @mattklein123 +# tracers.skywalking extension +/*/extensions/tracers/skywalking @wbpcode @dio @lizan # mysql_proxy extension /*/extensions/filters/network/mysql_proxy @rshriram @venilnoronha @mattklein123 # postgres_proxy extension diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index a12a0ea98b3a..983f15967b28 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -40,6 +40,10 @@ def api_dependencies(): name = "com_github_openzipkin_zipkinapi", build_file_content = ZIPKINAPI_BUILD_CONTENT, ) + external_http_archive( + name = "com_github_apache_skywalking_data_collect_protocol", + build_file_content = SKYWALKING_DATA_COLLECT_PROTOCOL_BUILD_CONTENT, + ) PROMETHEUSMETRICS_BUILD_CONTENT = """ load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") @@ -101,3 +105,30 @@ go_proto_library( visibility = ["//visibility:public"], ) """ + +SKYWALKING_DATA_COLLECT_PROTOCOL_BUILD_CONTENT = """ +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +proto_library( + name = "protocol", + srcs = [ + "common/Common.proto", + "language-agent/Tracing.proto", + ], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "protocol_cc_proto", + deps = [":protocol"], + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "protocol_go_proto", + proto = ":protocol", + visibility = ["//visibility:public"], +) +""" diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index e46f7d77f8e5..d72069046b85 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -88,4 +88,15 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2020-08-17", use_category = ["api"], ), + com_github_apache_skywalking_data_collect_protocol = dict( + project_name = "SkyWalking API", + project_desc = "SkyWalking's language independent model and gRPC API Definitions", + project_url = "https://github.com/apache/skywalking-data-collect-protocol", + version = "8.1.0", + sha256 = "ebea8a6968722524d1bcc4426fb6a29907ddc2902aac7de1559012d3eee90cf9", + strip_prefix = "skywalking-data-collect-protocol-{version}", + urls = ["https://github.com/apache/skywalking-data-collect-protocol/archive/v{version}.tar.gz"], + release_date = "2020-07-29", + use_category = ["api"], + ), ) diff --git a/api/envoy/config/trace/v3/skywalking.proto b/api/envoy/config/trace/v3/skywalking.proto new file mode 100644 index 000000000000..224d474ccf98 --- /dev/null +++ b/api/envoy/config/trace/v3/skywalking.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "envoy/config/core/v3/grpc_service.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/sensitive.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "SkywalkingProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.skywalking.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: SkyWalking tracer] + +// Configuration for the SkyWalking tracer. Please note that if SkyWalking tracer is used as the +// provider of http tracer, then +// :ref:`start_child_span ` +// in the router must be set to true to get the correct topology and tracing data. Moreover, SkyWalking +// Tracer does not support SkyWalking extension header (``sw8-x``) temporarily. +// [#extension: envoy.tracers.skywalking] +message SkyWalkingConfig { + // SkyWalking collector service. + core.v3.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; + + ClientConfig client_config = 2; +} + +// Client config for SkyWalking tracer. +message ClientConfig { + // Service name for SkyWalking tracer. If this field is empty, then local service cluster name + // that configured by :ref:`Bootstrap node ` + // message's :ref:`cluster ` field or command line + // option :option:`--service-cluster` will be used. If both this field and local service cluster + // name are empty, ``EnvoyProxy`` is used as the service name by default. + string service_name = 1; + + // Service instance name for SkyWalking tracer. If this field is empty, then local service node + // that configured by :ref:`Bootstrap node ` + // message's :ref:`id ` field or command line option + // :option:`--service-node` will be used. If both this field and local service node are empty, + // ``EnvoyProxy`` is used as the instance name by default. + string instance_name = 2; + + // Authentication token config for SkyWalking. SkyWalking can use token authentication to secure + // that monitoring application data can be trusted. In current version, Token is considered as a + // simple string. + // [#comment:TODO(wbpcode): Get backend token through the SDS API.] + oneof backend_token_specifier { + // Inline authentication token string. + string backend_token = 3 [(udpa.annotations.sensitive) = true]; + } + + // Envoy caches the segment in memory when the SkyWalking backend service is temporarily unavailable. + // This field specifies the maximum number of segments that can be cached. If not specified, the + // default is 1024. + google.protobuf.UInt32Value max_cache_size = 4; +} diff --git a/api/envoy/extensions/tracers/skywalking/v4alpha/BUILD b/api/envoy/extensions/tracers/skywalking/v4alpha/BUILD new file mode 100644 index 000000000000..1d56979cc466 --- /dev/null +++ b/api/envoy/extensions/tracers/skywalking/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v4alpha:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/tracers/skywalking/v4alpha/skywalking.proto b/api/envoy/extensions/tracers/skywalking/v4alpha/skywalking.proto new file mode 100644 index 000000000000..37936faa6133 --- /dev/null +++ b/api/envoy/extensions/tracers/skywalking/v4alpha/skywalking.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.skywalking.v4alpha; + +import "envoy/config/core/v4alpha/grpc_service.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/sensitive.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.skywalking.v4alpha"; +option java_outer_classname = "SkywalkingProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: SkyWalking tracer] + +// Configuration for the SkyWalking tracer. Please note that if SkyWalking tracer is used as the +// provider of http tracer, then +// :ref:`start_child_span ` +// in the router must be set to true to get the correct topology and tracing data. Moreover, SkyWalking +// Tracer does not support SkyWalking extension header (``sw8-x``) temporarily. +// [#extension: envoy.tracers.skywalking] +message SkyWalkingConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.SkyWalkingConfig"; + + // SkyWalking collector service. + config.core.v4alpha.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; + + ClientConfig client_config = 2; +} + +// Client config for SkyWalking tracer. +message ClientConfig { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.ClientConfig"; + + // Service name for SkyWalking tracer. If this field is empty, then local service cluster name + // that configured by :ref:`Bootstrap node ` + // message's :ref:`cluster ` field or command line + // option :option:`--service-cluster` will be used. If both this field and local service cluster + // name are empty, ``EnvoyProxy`` is used as the service name by default. + string service_name = 1; + + // Service instance name for SkyWalking tracer. If this field is empty, then local service node + // that configured by :ref:`Bootstrap node ` + // message's :ref:`id ` field or command line option + // :option:`--service-node` will be used. If both this field and local service node are empty, + // ``EnvoyProxy`` is used as the instance name by default. + string instance_name = 2; + + // Authentication token config for SkyWalking. SkyWalking can use token authentication to secure + // that monitoring application data can be trusted. In current version, Token is considered as a + // simple string. + // [#comment:TODO(wbpcode): Get backend token through the SDS API.] + oneof backend_token_specifier { + // Inline authentication token string. + string backend_token = 3 [(udpa.annotations.sensitive) = true]; + } + + // Envoy caches the segment in memory when the SkyWalking backend service is temporarily unavailable. + // This field specifies the maximum number of segments that can be cached. If not specified, the + // default is 1024. + google.protobuf.UInt32Value max_cache_size = 4; +} diff --git a/docs/root/start/sandboxes/index.rst b/docs/root/start/sandboxes/index.rst index 88db6644e856..6c181e5fb3c7 100644 --- a/docs/root/start/sandboxes/index.rst +++ b/docs/root/start/sandboxes/index.rst @@ -29,3 +29,4 @@ features. The following sandboxes are available: redis wasm-cc zipkin_tracing + skywalking_tracing diff --git a/docs/root/start/sandboxes/skywalking_tracing.rst b/docs/root/start/sandboxes/skywalking_tracing.rst new file mode 100644 index 000000000000..66f07dd15718 --- /dev/null +++ b/docs/root/start/sandboxes/skywalking_tracing.rst @@ -0,0 +1,89 @@ +.. _install_sandboxes_skywalking_tracing: + +SkyWalking Tracing +================== + +The SkyWalking tracing sandbox demonstrates Envoy's :ref:`request tracing ` +capabilities using `SkyWalking `_ as the tracing provider. This sandbox +is very similar to the Zipkin sandbox. All containers will be deployed inside a virtual network +called ``envoymesh``. + +All incoming requests are routed via the front Envoy, which is acting as a reverse proxy +sitting on the edge of the ``envoymesh`` network. Port ``8000`` is exposed +by docker compose (see :repo:`/examples/skywalking-tracing/docker-compose.yaml`). Notice that +all Envoys are configured to collect request traces (e.g., http_connection_manager/config/tracing setup in +:repo:`/examples/skywalking-tracing/front-envoy-skywalking.yaml`) and setup to propagate the spans generated +by the SkyWalking tracer to a SkyWalking cluster (trace driver setup +in :repo:`/examples/skywalking-tracing/front-envoy-skywalking.yaml`). + +When service1 accepts the request forwarded from front envoy, it will make an API call to service2 before +returning a response. + +.. include:: _include/docker-env-setup.rst + +Step 3: Build the sandbox +************************* + +To build this sandbox example, and start the example apps run the following commands: + +.. code-block:: console + + $ pwd + envoy/examples/skywalking-tracing + $ docker-compose pull + $ docker-compose up --build -d + $ docker-compose ps + + Name Command State Ports + -------------------------------------------------------------------------------------------------------------------------------------------------- + skywalking-tracing_elasticsearch_1 /tini -- /usr/local/bin/do ... Up (healthy) 0.0.0.0:9200->9200/tcp, 9300/tcp + skywalking-tracing_front-envoy_1 /docker-entrypoint.sh /bin ... Up 10000/tcp, 0.0.0.0:8000->8000/tcp, 0.0.0.0:8001->8001/tcp + skywalking-tracing_service1_1 /bin/sh /usr/local/bin/sta ... Up 10000/tcp + skywalking-tracing_service2_1 /bin/sh /usr/local/bin/sta ... Up 10000/tcp + skywalking-tracing_skywalking-oap_1 bash docker-entrypoint.sh Up (healthy) 0.0.0.0:11800->11800/tcp, 1234/tcp, 0.0.0.0:12800->12800/tcp + skywalking-tracing_skywalking-ui_1 bash docker-entrypoint.sh Up 0.0.0.0:8080->8080/tcp + +Step 4: Generate some load +************************** + +You can now send a request to service1 via the front-envoy as follows: + +.. code-block:: console + + $ curl -v localhost:8000/trace/1 + * Trying ::1... + * TCP_NODELAY set + * Connected to localhost (::1) port 8000 (#0) + > GET /trace/1 HTTP/1.1 + > Host: localhost:8000 + > User-Agent: curl/7.58.0 + > Accept: */* + > + < HTTP/1.1 200 OK + < content-type: text/html; charset=utf-8 + < content-length: 89 + < server: envoy + < date: Sat, 10 Oct 2020 01:56:08 GMT + < x-envoy-upstream-service-time: 27 + < + Hello from behind Envoy (service 1)! hostname: 1a2ba43d6d84 resolvedhostname: 172.19.0.6 + * Connection #0 to host localhost left intact + +You can get SkyWalking stats of front-envoy after some requests as follows: + +.. code-block:: console + + $ curl -s localhost:8001/stats | grep tracing.skywalking + tracing.skywalking.cache_flushed: 0 + tracing.skywalking.segments_dropped: 0 + tracing.skywalking.segments_flushed: 0 + tracing.skywalking.segments_sent: 13 + +Step 5: View the traces in SkyWalking UI +**************************************** + +Point your browser to http://localhost:8080 . You should see the SkyWalking dashboard. +Set the service to "front-envoy" and set the start time to a few minutes before +the start of the test (step 2) and hit enter. You should see traces from the front-proxy. +Click on a trace to explore the path taken by the request from front-proxy to service1 +to service2, as well as the latency incurred at each hop. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 43c4ce2f72b5..041ea6adaeac 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -54,6 +54,7 @@ New Features * ratelimit: added :ref:`disable_x_envoy_ratelimited_header ` option to disable `X-Envoy-RateLimited` header. * tcp: added a new :ref:`envoy.overload_actions.reject_incoming_connections ` action to reject incoming TCP connections. * tls: added support for RSA certificates with 4096-bit keys in FIPS mode. +* tracing: added SkyWalking tracer. * xds: added support for resource TTLs. A TTL is specified on the :ref:`Resource `. For SotW, a :ref:`Resource ` can be embedded in the list of resources to specify the TTL. diff --git a/examples/front-proxy/service.py b/examples/front-proxy/service.py index 1d5d5920a8e3..c0e008d73404 100644 --- a/examples/front-proxy/service.py +++ b/examples/front-proxy/service.py @@ -19,7 +19,10 @@ 'X-B3-Flags', # Jaeger header (for native client) - "uber-trace-id" + "uber-trace-id", + + # SkyWalking headers. + "sw8" ] diff --git a/examples/skywalking-tracing/Dockerfile-frontenvoy b/examples/skywalking-tracing/Dockerfile-frontenvoy new file mode 100644 index 000000000000..86d0a6b91b8b --- /dev/null +++ b/examples/skywalking-tracing/Dockerfile-frontenvoy @@ -0,0 +1,7 @@ +FROM envoyproxy/envoy-dev:latest + +RUN apt-get update && apt-get -q install -y \ + curl +COPY ./front-envoy-skywalking.yaml /etc/front-envoy.yaml +RUN chmod go+r /etc/front-envoy.yaml +CMD /usr/local/bin/envoy -c /etc/front-envoy.yaml --service-cluster front-proxy diff --git a/examples/skywalking-tracing/README.md b/examples/skywalking-tracing/README.md new file mode 100644 index 000000000000..5a9375b74006 --- /dev/null +++ b/examples/skywalking-tracing/README.md @@ -0,0 +1,2 @@ +To learn about this sandbox and for instructions on how to run it please head over +to the [envoy docs](https://www.envoyproxy.io/docs/envoy/latest/start/sandboxes/skywalking_tracing) diff --git a/examples/skywalking-tracing/docker-compose.yaml b/examples/skywalking-tracing/docker-compose.yaml new file mode 100644 index 000000000000..5ac0e647a7fe --- /dev/null +++ b/examples/skywalking-tracing/docker-compose.yaml @@ -0,0 +1,85 @@ +version: "3.7" +services: + # Front envoy. + front-envoy: + build: + context: . + dockerfile: Dockerfile-frontenvoy + networks: + - envoymesh + ports: + - 8000:8000 + - 8001:8001 + depends_on: + - skywalking-oap + # First service. + service1: + build: + context: ../front-proxy + dockerfile: Dockerfile-service + volumes: + - ./service1-envoy-skywalking.yaml:/etc/service-envoy.yaml + networks: + - envoymesh + environment: + - SERVICE_NAME=1 + depends_on: + - skywalking-oap + # Second service. + service2: + build: + context: ../front-proxy + dockerfile: Dockerfile-service + volumes: + - ./service2-envoy-skywalking.yaml:/etc/service-envoy.yaml + networks: + - envoymesh + environment: + - SERVICE_NAME=2 + depends_on: + - skywalking-oap + # Skywalking components. + elasticsearch: + image: elasticsearch:7.9.2 + networks: + - envoymesh + healthcheck: + test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + environment: + discovery.type: single-node + ulimits: + memlock: + soft: -1 + hard: -1 + skywalking-oap: + image: apache/skywalking-oap-server:8.1.0-es7 + networks: + - envoymesh + depends_on: + - elasticsearch + environment: + SW_STORAGE: elasticsearch7 + SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200 + healthcheck: + test: ["CMD-SHELL", "/skywalking/bin/swctl"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + restart: on-failure + skywalking-ui: + image: apache/skywalking-ui:8.1.0 + networks: + - envoymesh + depends_on: + - skywalking-oap + ports: + - 8080:8080 + environment: + SW_OAP_ADDRESS: skywalking-oap:12800 +networks: + envoymesh: {} diff --git a/examples/skywalking-tracing/front-envoy-skywalking.yaml b/examples/skywalking-tracing/front-envoy-skywalking.yaml new file mode 100644 index 000000000000..32d7de94dd07 --- /dev/null +++ b/examples/skywalking-tracing/front-envoy-skywalking.yaml @@ -0,0 +1,80 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + traffic_direction: OUTBOUND + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: true + tracing: + provider: + name: envoy.tracers.skywalking + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig + grpc_service: + envoy_grpc: + cluster_name: skywalking + timeout: 0.250s + client_config: + service_name: front-envoy + instance_name: front-envoy-1 + codec_type: auto + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: service1 + decorator: + operation: checkAvailability + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + start_child_span: true + clusters: + - name: service1 + connect_timeout: 0.250s + type: strict_dns + lb_policy: round_robin + http2_protocol_options: {} + load_assignment: + cluster_name: service1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service1 + port_value: 8000 + - name: skywalking + connect_timeout: 1s + type: strict_dns + lb_policy: round_robin + http2_protocol_options: {} + load_assignment: + cluster_name: skywalking + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: skywalking-oap + port_value: 11800 +admin: + access_log_path: "/dev/null" + address: + socket_address: + address: 0.0.0.0 + port_value: 8001 diff --git a/examples/skywalking-tracing/service1-envoy-skywalking.yaml b/examples/skywalking-tracing/service1-envoy-skywalking.yaml new file mode 100644 index 000000000000..a030c63f8c9f --- /dev/null +++ b/examples/skywalking-tracing/service1-envoy-skywalking.yaml @@ -0,0 +1,128 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + traffic_direction: INBOUND + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + tracing: + provider: + name: envoy.tracers.skywalking + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig + grpc_service: + envoy_grpc: + cluster_name: skywalking + timeout: 0.250s + client_config: + service_name: service1-envoy + instance_name: service1-envoy-1 + codec_type: auto + stat_prefix: ingress_http + route_config: + name: service1_route + virtual_hosts: + - name: service1 + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: local_service + decorator: + operation: checkAvailability + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + start_child_span: true + - address: + socket_address: + address: 0.0.0.0 + port_value: 9000 + traffic_direction: OUTBOUND + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + tracing: + provider: + name: envoy.tracers.skywalking + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig + grpc_service: + envoy_grpc: + cluster_name: skywalking + timeout: 0.250s + client_config: + service_name: service1-envoy + instance_name: service1-envoy-1 + codec_type: auto + stat_prefix: egress_http + route_config: + name: service2_route + virtual_hosts: + - name: service2 + domains: + - "*" + routes: + - match: + prefix: "/trace/2" + route: + cluster: service2 + decorator: + operation: checkStock + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + start_child_span: true + clusters: + - name: local_service + connect_timeout: 0.250s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: local_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 + - name: service2 + connect_timeout: 0.250s + type: strict_dns + lb_policy: round_robin + http2_protocol_options: {} + load_assignment: + cluster_name: service2 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: service2 + port_value: 8000 + - name: skywalking + connect_timeout: 1s + type: strict_dns + lb_policy: round_robin + http2_protocol_options: {} + load_assignment: + cluster_name: skywalking + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: skywalking-oap + port_value: 11800 diff --git a/examples/skywalking-tracing/service2-envoy-skywalking.yaml b/examples/skywalking-tracing/service2-envoy-skywalking.yaml new file mode 100644 index 000000000000..5f0ee1b834ea --- /dev/null +++ b/examples/skywalking-tracing/service2-envoy-skywalking.yaml @@ -0,0 +1,72 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + traffic_direction: INBOUND + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + tracing: + provider: + name: envoy.tracers.skywalking + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig + grpc_service: + envoy_grpc: + cluster_name: skywalking + timeout: 0.250s + client_config: + service_name: service2-envoy + instance_name: service2-envoy-1 + codec_type: auto + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: service2 + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: local_service + decorator: + operation: checkStock + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + start_child_span: true + clusters: + - name: local_service + connect_timeout: 0.250s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: local_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 + - name: skywalking + connect_timeout: 1s + type: strict_dns + lb_policy: round_robin + http2_protocol_options: {} + load_assignment: + cluster_name: skywalking + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: skywalking-oap + port_value: 11800 diff --git a/examples/skywalking-tracing/verify.sh b/examples/skywalking-tracing/verify.sh new file mode 100755 index 000000000000..3c5c4799ca90 --- /dev/null +++ b/examples/skywalking-tracing/verify.sh @@ -0,0 +1,51 @@ +#!/bin/bash -e + +export NAME=skywalking +export DELAY=200 + +# shellcheck source=examples/verify-common.sh +. "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh" + +run_log "Test connection" +responds_with \ + "Hello from behind Envoy (service 1)!" \ + http://localhost:8000/trace/1 + +run_log "Test stats" +responds_with \ + "tracing.skywalking.segments_sent: 1" \ + http://localhost:8001/stats + +run_log "Test dashboard" +responds_with \ + "" \ + http://localhost:8080 + +run_log "Test OAP Server" +responds_with \ + "getEndpoints" \ + http://localhost:8080/graphql \ + -X POST \ + -H "Content-Type:application/json" \ + -d "{ \"query\": \"query queryEndpoints(\$serviceId: ID!, \$keyword: String!) { + getEndpoints: searchEndpoint(serviceId: \$serviceId, keyword: \$keyword, limit: 100) { + key: id + label: name + } + }\", + \"variables\": { \"serviceId\": \"\", \"keyword\": \"\" } + }" + +responds_with \ + "currentTimestamp" \ + http://localhost:8080/graphql \ + -X POST \ + -H "Content-Type:application/json" \ + -d "{ \"query\": \"query queryOAPTimeInfo { + getTimeInfo { + timezone + currentTimestamp + } + }\", + \"variables\": {} + }" diff --git a/generated_api_shadow/bazel/repositories.bzl b/generated_api_shadow/bazel/repositories.bzl index a12a0ea98b3a..983f15967b28 100644 --- a/generated_api_shadow/bazel/repositories.bzl +++ b/generated_api_shadow/bazel/repositories.bzl @@ -40,6 +40,10 @@ def api_dependencies(): name = "com_github_openzipkin_zipkinapi", build_file_content = ZIPKINAPI_BUILD_CONTENT, ) + external_http_archive( + name = "com_github_apache_skywalking_data_collect_protocol", + build_file_content = SKYWALKING_DATA_COLLECT_PROTOCOL_BUILD_CONTENT, + ) PROMETHEUSMETRICS_BUILD_CONTENT = """ load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") @@ -101,3 +105,30 @@ go_proto_library( visibility = ["//visibility:public"], ) """ + +SKYWALKING_DATA_COLLECT_PROTOCOL_BUILD_CONTENT = """ +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +proto_library( + name = "protocol", + srcs = [ + "common/Common.proto", + "language-agent/Tracing.proto", + ], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "protocol_cc_proto", + deps = [":protocol"], + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "protocol_go_proto", + proto = ":protocol", + visibility = ["//visibility:public"], +) +""" diff --git a/generated_api_shadow/bazel/repository_locations.bzl b/generated_api_shadow/bazel/repository_locations.bzl index e46f7d77f8e5..d72069046b85 100644 --- a/generated_api_shadow/bazel/repository_locations.bzl +++ b/generated_api_shadow/bazel/repository_locations.bzl @@ -88,4 +88,15 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2020-08-17", use_category = ["api"], ), + com_github_apache_skywalking_data_collect_protocol = dict( + project_name = "SkyWalking API", + project_desc = "SkyWalking's language independent model and gRPC API Definitions", + project_url = "https://github.com/apache/skywalking-data-collect-protocol", + version = "8.1.0", + sha256 = "ebea8a6968722524d1bcc4426fb6a29907ddc2902aac7de1559012d3eee90cf9", + strip_prefix = "skywalking-data-collect-protocol-{version}", + urls = ["https://github.com/apache/skywalking-data-collect-protocol/archive/v{version}.tar.gz"], + release_date = "2020-07-29", + use_category = ["api"], + ), ) diff --git a/generated_api_shadow/envoy/config/trace/v3/skywalking.proto b/generated_api_shadow/envoy/config/trace/v3/skywalking.proto new file mode 100644 index 000000000000..224d474ccf98 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/skywalking.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "envoy/config/core/v3/grpc_service.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/sensitive.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "SkywalkingProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.skywalking.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: SkyWalking tracer] + +// Configuration for the SkyWalking tracer. Please note that if SkyWalking tracer is used as the +// provider of http tracer, then +// :ref:`start_child_span ` +// in the router must be set to true to get the correct topology and tracing data. Moreover, SkyWalking +// Tracer does not support SkyWalking extension header (``sw8-x``) temporarily. +// [#extension: envoy.tracers.skywalking] +message SkyWalkingConfig { + // SkyWalking collector service. + core.v3.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; + + ClientConfig client_config = 2; +} + +// Client config for SkyWalking tracer. +message ClientConfig { + // Service name for SkyWalking tracer. If this field is empty, then local service cluster name + // that configured by :ref:`Bootstrap node ` + // message's :ref:`cluster ` field or command line + // option :option:`--service-cluster` will be used. If both this field and local service cluster + // name are empty, ``EnvoyProxy`` is used as the service name by default. + string service_name = 1; + + // Service instance name for SkyWalking tracer. If this field is empty, then local service node + // that configured by :ref:`Bootstrap node ` + // message's :ref:`id ` field or command line option + // :option:`--service-node` will be used. If both this field and local service node are empty, + // ``EnvoyProxy`` is used as the instance name by default. + string instance_name = 2; + + // Authentication token config for SkyWalking. SkyWalking can use token authentication to secure + // that monitoring application data can be trusted. In current version, Token is considered as a + // simple string. + // [#comment:TODO(wbpcode): Get backend token through the SDS API.] + oneof backend_token_specifier { + // Inline authentication token string. + string backend_token = 3 [(udpa.annotations.sensitive) = true]; + } + + // Envoy caches the segment in memory when the SkyWalking backend service is temporarily unavailable. + // This field specifies the maximum number of segments that can be cached. If not specified, the + // default is 1024. + google.protobuf.UInt32Value max_cache_size = 4; +} diff --git a/generated_api_shadow/envoy/extensions/tracers/skywalking/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/skywalking/v4alpha/BUILD new file mode 100644 index 000000000000..1d56979cc466 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/skywalking/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v4alpha:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/tracers/skywalking/v4alpha/skywalking.proto b/generated_api_shadow/envoy/extensions/tracers/skywalking/v4alpha/skywalking.proto new file mode 100644 index 000000000000..37936faa6133 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/skywalking/v4alpha/skywalking.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.skywalking.v4alpha; + +import "envoy/config/core/v4alpha/grpc_service.proto"; + +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/sensitive.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.skywalking.v4alpha"; +option java_outer_classname = "SkywalkingProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: SkyWalking tracer] + +// Configuration for the SkyWalking tracer. Please note that if SkyWalking tracer is used as the +// provider of http tracer, then +// :ref:`start_child_span ` +// in the router must be set to true to get the correct topology and tracing data. Moreover, SkyWalking +// Tracer does not support SkyWalking extension header (``sw8-x``) temporarily. +// [#extension: envoy.tracers.skywalking] +message SkyWalkingConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.SkyWalkingConfig"; + + // SkyWalking collector service. + config.core.v4alpha.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; + + ClientConfig client_config = 2; +} + +// Client config for SkyWalking tracer. +message ClientConfig { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.ClientConfig"; + + // Service name for SkyWalking tracer. If this field is empty, then local service cluster name + // that configured by :ref:`Bootstrap node ` + // message's :ref:`cluster ` field or command line + // option :option:`--service-cluster` will be used. If both this field and local service cluster + // name are empty, ``EnvoyProxy`` is used as the service name by default. + string service_name = 1; + + // Service instance name for SkyWalking tracer. If this field is empty, then local service node + // that configured by :ref:`Bootstrap node ` + // message's :ref:`id ` field or command line option + // :option:`--service-node` will be used. If both this field and local service node are empty, + // ``EnvoyProxy`` is used as the instance name by default. + string instance_name = 2; + + // Authentication token config for SkyWalking. SkyWalking can use token authentication to secure + // that monitoring application data can be trusted. In current version, Token is considered as a + // simple string. + // [#comment:TODO(wbpcode): Get backend token through the SDS API.] + oneof backend_token_specifier { + // Inline authentication token string. + string backend_token = 3 [(udpa.annotations.sensitive) = true]; + } + + // Envoy caches the segment in memory when the SkyWalking backend service is temporarily unavailable. + // This field specifies the maximum number of segments that can be cached. If not specified, the + // default is 1024. + google.protobuf.UInt32Value max_cache_size = 4; +} diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 5a66f8f0ea59..7d89713a3f76 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -59,6 +59,7 @@ class CustomHeaderValues { const LowerCaseString AccessControlExposeHeaders{"access-control-expose-headers"}; const LowerCaseString AccessControlMaxAge{"access-control-max-age"}; const LowerCaseString AccessControlAllowCredentials{"access-control-allow-credentials"}; + const LowerCaseString Authentication{"authentication"}; const LowerCaseString Authorization{"authorization"}; const LowerCaseString CacheControl{"cache-control"}; const LowerCaseString CdnLoop{"cdn-loop"}; diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index e3ec724d9339..664b561fb0d2 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -166,6 +166,7 @@ EXTENSIONS = { "envoy.tracers.opencensus": "//source/extensions/tracers/opencensus:config", # WiP "envoy.tracers.xray": "//source/extensions/tracers/xray:config", + "envoy.tracers.skywalking": "//source/extensions/tracers/skywalking:config", # # Transport sockets diff --git a/source/extensions/tracers/skywalking/BUILD b/source/extensions/tracers/skywalking/BUILD new file mode 100644 index 000000000000..5cf90c3f976f --- /dev/null +++ b/source/extensions/tracers/skywalking/BUILD @@ -0,0 +1,107 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +# Trace driver for Apache SkyWalking. + +envoy_extension_package() + +envoy_cc_library( + name = "trace_segment_reporter_lib", + srcs = ["trace_segment_reporter.cc"], + hdrs = ["trace_segment_reporter.h"], + deps = [ + ":skywalking_client_config_lib", + ":skywalking_stats_lib", + ":skywalking_types_lib", + "//include/envoy/grpc:async_client_manager_interface", + "//source/common/common:backoff_lib", + "//source/common/grpc:async_client_lib", + "@com_github_apache_skywalking_data_collect_protocol//:protocol_cc_proto", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "skywalking_types_lib", + srcs = ["skywalking_types.cc"], + hdrs = ["skywalking_types.h"], + deps = [ + ":skywalking_stats_lib", + "//include/envoy/common:random_generator_interface", + "//include/envoy/common:time_interface", + "//include/envoy/http:header_map_interface", + "//include/envoy/tracing:http_tracer_interface", + "//source/common/common:base64_lib", + "//source/common/common:hex_lib", + "//source/common/common:utility_lib", + "@com_github_apache_skywalking_data_collect_protocol//:protocol_cc_proto", + ], +) + +envoy_cc_library( + name = "skywalking_client_config_lib", + srcs = ["skywalking_client_config.cc"], + hdrs = ["skywalking_client_config.h"], + deps = [ + "//include/envoy/secret:secret_provider_interface", + "//include/envoy/server:factory_context_interface", + "//include/envoy/server:tracer_config_interface", + "//source/common/config:datasource_lib", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "skywalking_tracer_lib", + srcs = [ + "skywalking_tracer_impl.cc", + "tracer.cc", + ], + hdrs = [ + "skywalking_tracer_impl.h", + "tracer.h", + ], + deps = [ + ":skywalking_client_config_lib", + ":skywalking_types_lib", + ":trace_segment_reporter_lib", + "//include/envoy/common:time_interface", + "//include/envoy/server:tracer_config_interface", + "//include/envoy/tracing:http_tracer_interface", + "//source/common/common:macros", + "//source/common/http:header_map_lib", + "//source/common/runtime:runtime_lib", + "//source/common/tracing:http_tracer_lib", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "skywalking_stats_lib", + hdrs = [ + "skywalking_stats.h", + ], + deps = [ + "//include/envoy/stats:stats_macros", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + security_posture = "robust_to_untrusted_downstream", + status = "wip", + deps = [ + ":skywalking_tracer_lib", + "//source/common/config:datasource_lib", + "//source/extensions/tracers/common:factory_base_lib", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/tracers/skywalking/config.cc b/source/extensions/tracers/skywalking/config.cc new file mode 100644 index 000000000000..4f9e15b12f2d --- /dev/null +++ b/source/extensions/tracers/skywalking/config.cc @@ -0,0 +1,36 @@ +#include "extensions/tracers/skywalking/config.h" + +#include "envoy/config/trace/v3/skywalking.pb.h" +#include "envoy/config/trace/v3/skywalking.pb.validate.h" +#include "envoy/registry/registry.h" + +#include "common/common/utility.h" +#include "common/tracing/http_tracer_impl.h" + +#include "extensions/tracers/skywalking/skywalking_tracer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +SkyWalkingTracerFactory::SkyWalkingTracerFactory() : FactoryBase("envoy.tracers.skywalking") {} + +Tracing::HttpTracerSharedPtr SkyWalkingTracerFactory::createHttpTracerTyped( + const envoy::config::trace::v3::SkyWalkingConfig& proto_config, + Server::Configuration::TracerFactoryContext& context) { + Tracing::DriverPtr skywalking_driver = + std::make_unique(proto_config, context); + return std::make_shared(std::move(skywalking_driver), + context.serverFactoryContext().localInfo()); +} + +/** + * Static registration for the SkyWalking tracer. @see RegisterFactory. + */ +REGISTER_FACTORY(SkyWalkingTracerFactory, Server::Configuration::TracerFactory); + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/config.h b/source/extensions/tracers/skywalking/config.h new file mode 100644 index 000000000000..abeffe373e5d --- /dev/null +++ b/source/extensions/tracers/skywalking/config.h @@ -0,0 +1,31 @@ +#pragma once + +#include "envoy/config/trace/v3/skywalking.pb.h" +#include "envoy/config/trace/v3/skywalking.pb.validate.h" + +#include "extensions/tracers/common/factory_base.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +/** + * Config registration for the SkyWalking tracer. @see TracerFactory. + */ +class SkyWalkingTracerFactory + : public Common::FactoryBase { +public: + SkyWalkingTracerFactory(); + +private: + // FactoryBase + Tracing::HttpTracerSharedPtr + createHttpTracerTyped(const envoy::config::trace::v3::SkyWalkingConfig& proto_config, + Server::Configuration::TracerFactoryContext& context) override; +}; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_client_config.cc b/source/extensions/tracers/skywalking/skywalking_client_config.cc new file mode 100644 index 000000000000..ed692b3f72bd --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_client_config.cc @@ -0,0 +1,43 @@ +#include "extensions/tracers/skywalking/skywalking_client_config.h" + +#include "common/config/datasource.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +constexpr uint32_t DEFAULT_DELAYED_SEGMENTS_CACHE_SIZE = 1024; + +// When the user does not provide any available configuration, in order to ensure that the service +// name and instance name are not empty, use this value as the default identifier. In practice, +// user should provide accurate configuration as much as possible to avoid using the default value. +constexpr absl::string_view DEFAULT_SERVICE_AND_INSTANCE = "EnvoyProxy"; + +SkyWalkingClientConfig::SkyWalkingClientConfig(Server::Configuration::TracerFactoryContext& context, + const envoy::config::trace::v3::ClientConfig& config) + : factory_context_(context.serverFactoryContext()), + max_cache_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_cache_size, + DEFAULT_DELAYED_SEGMENTS_CACHE_SIZE)), + service_(config.service_name().empty() ? factory_context_.localInfo().clusterName().empty() + ? DEFAULT_SERVICE_AND_INSTANCE + : factory_context_.localInfo().clusterName() + : config.service_name()), + instance_(config.instance_name().empty() ? factory_context_.localInfo().nodeName().empty() + ? DEFAULT_SERVICE_AND_INSTANCE + : factory_context_.localInfo().nodeName() + : config.instance_name()) { + // Since the SDS API to get backend token is not supported yet, we can get the value of token + // from the backend_token field directly. If the user does not provide the configuration, the + // value of token is kept empty. + backend_token_ = config.backend_token(); +} + +// TODO(wbpcode): currently, backend authentication token can only be configured with inline string. +// It will be possible to get authentication through the SDS API later. +const std::string& SkyWalkingClientConfig::backendToken() const { return backend_token_; } + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_client_config.h b/source/extensions/tracers/skywalking/skywalking_client_config.h new file mode 100644 index 000000000000..8983a1dbf758 --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_client_config.h @@ -0,0 +1,43 @@ +#pragma once + +#include "envoy/config/trace/v3/skywalking.pb.h" +#include "envoy/secret/secret_provider.h" +#include "envoy/server/factory_context.h" +#include "envoy/server/tracer_config.h" + +#include "absl/synchronization/mutex.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +class SkyWalkingClientConfig { +public: + SkyWalkingClientConfig(Server::Configuration::TracerFactoryContext& context, + const envoy::config::trace::v3::ClientConfig& config); + + uint32_t maxCacheSize() const { return max_cache_size_; } + + const std::string& service() const { return service_; } + const std::string& serviceInstance() const { return instance_; } + + const std::string& backendToken() const; + +private: + Server::Configuration::ServerFactoryContext& factory_context_; + + const uint32_t max_cache_size_{0}; + + const std::string service_; + const std::string instance_; + + std::string backend_token_; +}; + +using SkyWalkingClientConfigPtr = std::unique_ptr; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_stats.h b/source/extensions/tracers/skywalking/skywalking_stats.h new file mode 100644 index 000000000000..0ad8a58f5ab5 --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_stats.h @@ -0,0 +1,21 @@ +#include "envoy/stats/stats_macros.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +#define SKYWALKING_TRACER_STATS(COUNTER) \ + COUNTER(cache_flushed) \ + COUNTER(segments_dropped) \ + COUNTER(segments_flushed) \ + COUNTER(segments_sent) + +struct SkyWalkingTracerStats { + SKYWALKING_TRACER_STATS(GENERATE_COUNTER_STRUCT) +}; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc new file mode 100644 index 000000000000..130cbd6a9801 --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.cc @@ -0,0 +1,63 @@ +#include "extensions/tracers/skywalking/skywalking_tracer_impl.h" + +#include + +#include "common/common/macros.h" +#include "common/common/utility.h" +#include "common/http/path_utility.h" + +#include "extensions/tracers/skywalking/skywalking_types.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +Driver::Driver(const envoy::config::trace::v3::SkyWalkingConfig& proto_config, + Server::Configuration::TracerFactoryContext& context) + : tracing_stats_{SKYWALKING_TRACER_STATS( + POOL_COUNTER_PREFIX(context.serverFactoryContext().scope(), "tracing.skywalking."))}, + client_config_( + std::make_unique(context, proto_config.client_config())), + random_generator_(context.serverFactoryContext().api().randomGenerator()), + tls_slot_ptr_(context.serverFactoryContext().threadLocal().allocateSlot()) { + + auto& factory_context = context.serverFactoryContext(); + tls_slot_ptr_->set([proto_config, &factory_context, this](Event::Dispatcher& dispatcher) { + TracerPtr tracer = std::make_unique(factory_context.timeSource()); + tracer->setReporter(std::make_unique( + factory_context.clusterManager().grpcAsyncClientManager().factoryForGrpcService( + proto_config.grpc_service(), factory_context.scope(), false), + dispatcher, factory_context.api().randomGenerator(), tracing_stats_, *client_config_)); + return std::make_shared(std::move(tracer)); + }); +} + +Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, + Http::RequestHeaderMap& request_headers, + const std::string& operation_name, Envoy::SystemTime start_time, + const Tracing::Decision decision) { + auto& tracer = *tls_slot_ptr_->getTyped().tracer_; + + try { + SpanContextPtr previous_span_context = SpanContext::spanContextFromRequest(request_headers); + auto segment_context = std::make_shared(std::move(previous_span_context), + decision, random_generator_); + + // Initialize fields of current span context. + segment_context->setService(client_config_->service()); + segment_context->setServiceInstance(client_config_->serviceInstance()); + + return tracer.startSpan(config, start_time, operation_name, std::move(segment_context), + nullptr); + + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "New SkyWalking Span/Segment cannot be created for error: {}", e.what()); + return std::make_unique(); + } +} + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_tracer_impl.h b/source/extensions/tracers/skywalking/skywalking_tracer_impl.h new file mode 100644 index 000000000000..f073461deb5d --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_tracer_impl.h @@ -0,0 +1,48 @@ +#pragma once + +#include "envoy/config/trace/v3/skywalking.pb.h" +#include "envoy/server/tracer_config.h" +#include "envoy/thread_local/thread_local.h" +#include "envoy/tracing/http_tracer.h" + +#include "common/tracing/http_tracer_impl.h" + +#include "extensions/tracers/skywalking/skywalking_client_config.h" +#include "extensions/tracers/skywalking/tracer.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +class Driver : public Tracing::Driver, public Logger::Loggable { +public: + explicit Driver(const envoy::config::trace::v3::SkyWalkingConfig& config, + Server::Configuration::TracerFactoryContext& context); + + Tracing::SpanPtr startSpan(const Tracing::Config& config, Http::RequestHeaderMap& request_headers, + const std::string& operation, Envoy::SystemTime start_time, + const Tracing::Decision decision) override; + +private: + struct TlsTracer : ThreadLocal::ThreadLocalObject { + TlsTracer(TracerPtr tracer) : tracer_(std::move(tracer)) {} + + TracerPtr tracer_; + }; + + SkyWalkingTracerStats tracing_stats_; + + SkyWalkingClientConfigPtr client_config_; + + // This random_generator_ will be used to create SkyWalking trace id and segment id. + Random::RandomGenerator& random_generator_; + ThreadLocal::SlotPtr tls_slot_ptr_; +}; + +using DriverPtr = std::unique_ptr; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_types.cc b/source/extensions/tracers/skywalking/skywalking_types.cc new file mode 100644 index 000000000000..9c750884bc11 --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_types.cc @@ -0,0 +1,175 @@ +#include "extensions/tracers/skywalking/skywalking_types.h" + +#include "envoy/common/exception.h" + +#include "common/common/base64.h" +#include "common/common/empty_string.h" +#include "common/common/fmt.h" +#include "common/common/hex.h" +#include "common/common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +namespace { + +// The standard header name is "sw8", as mentioned in: +// https://github.com/apache/skywalking/blob/v8.1.0/docs/en/protocols/Skywalking-Cross-Process-Propagation-Headers-Protocol-v3.md. +const Http::LowerCaseString& propagationHeader() { + CONSTRUCT_ON_FIRST_USE(Http::LowerCaseString, "sw8"); +} + +std::string generateId(Random::RandomGenerator& random_generator) { + return absl::StrCat(Hex::uint64ToHex(random_generator.random()), + Hex::uint64ToHex(random_generator.random())); +} + +std::string base64Encode(const absl::string_view input) { + return Base64::encode(input.data(), input.length()); +} + +// Decode and validate fields of propagation header. +std::string base64Decode(absl::string_view input) { + // The input can be Base64 string with or without padding. + std::string result = Base64::decodeWithoutPadding(input); + if (result.empty()) { + throw EnvoyException("Invalid propagation header for SkyWalking: parse error"); + } + return result; +} + +} // namespace + +SpanContextPtr SpanContext::spanContextFromRequest(Http::RequestHeaderMap& headers) { + auto propagation_header = headers.get(propagationHeader()); + if (propagation_header.empty()) { + // No propagation header then Envoy is first hop. + return nullptr; + } + + auto header_value_string = propagation_header[0]->value().getStringView(); + const auto parts = StringUtil::splitToken(header_value_string, "-", false, true); + // Reference: + // https://github.com/apache/skywalking/blob/v8.1.0/docs/en/protocols/Skywalking-Cross-Process-Propagation-Headers-Protocol-v3.md. + if (parts.size() != 8) { + throw EnvoyException( + fmt::format("Invalid propagation header for SkyWalking: {}", header_value_string)); + } + + SpanContextPtr previous_span_context = std::unique_ptr(new SpanContext()); + + // Parse and validate sampling flag. + if (parts[0] == "0") { + previous_span_context->sampled_ = 0; + } else if (parts[0] == "1") { + previous_span_context->sampled_ = 1; + } else { + throw EnvoyException(fmt::format("Invalid propagation header for SkyWalking: sampling flag can " + "only be '0' or '1' but '{}' was provided", + parts[0])); + } + + // Parse trace id. + previous_span_context->trace_id_ = base64Decode(parts[1]); + // Parse segment id. + previous_span_context->trace_segment_id_ = base64Decode(parts[2]); + + // Parse span id. + if (!absl::SimpleAtoi(parts[3], &previous_span_context->span_id_)) { + throw EnvoyException(fmt::format( + "Invalid propagation header for SkyWalking: connot convert '{}' to valid span id", + parts[3])); + } + + // Parse service. + previous_span_context->service_ = base64Decode(parts[4]); + // Parse service instance. + previous_span_context->service_instance_ = base64Decode(parts[5]); + // Parse endpoint. Operation Name of the first entry span in the previous segment. + previous_span_context->endpoint_ = base64Decode(parts[6]); + // Parse target address used at downstream side of this request. + previous_span_context->target_address_ = base64Decode(parts[7]); + + return previous_span_context; +} + +SegmentContext::SegmentContext(SpanContextPtr&& previous_span_context, Tracing::Decision decision, + Random::RandomGenerator& random_generator) + : previous_span_context_(std::move(previous_span_context)) { + + if (previous_span_context_) { + trace_id_ = previous_span_context_->trace_id_; + sampled_ = previous_span_context_->sampled_; + } else { + trace_id_ = generateId(random_generator); + sampled_ = decision.traced; + } + trace_segment_id_ = generateId(random_generator); + + // Some detailed log for debugging. + ENVOY_LOG(trace, "{} and create new SkyWalking segment:", + previous_span_context_ ? "Has previous span context" : "No previous span context"); + + ENVOY_LOG(trace, " Trace ID: {}", trace_id_); + ENVOY_LOG(trace, " Segment ID: {}", trace_segment_id_); + ENVOY_LOG(trace, " Sampled: {}", sampled_); +} + +SpanStore* SegmentContext::createSpanStore(const SpanStore* parent_span_store) { + ENVOY_LOG(trace, "Create new SpanStore object for current segment: {}", trace_segment_id_); + SpanStorePtr new_span_store = std::make_unique(this); + new_span_store->setSpanId(span_list_.size()); + if (!parent_span_store) { + // The parent SpanStore object does not exist. Create the root SpanStore object in the current + // segment. + new_span_store->setSampled(sampled_); + new_span_store->setParentSpanId(-1); + // First span of current segment for Envoy Proxy must be a Entry Span. It is created for + // downstream HTTP request. + new_span_store->setAsEntrySpan(true); + } else { + // Create child SpanStore object. + new_span_store->setSampled(parent_span_store->sampled()); + new_span_store->setParentSpanId(parent_span_store->spanId()); + new_span_store->setAsEntrySpan(false); + } + SpanStore* ref = new_span_store.get(); + span_list_.emplace_back(std::move(new_span_store)); + return ref; +} + +void SpanStore::injectContext(Http::RequestHeaderMap& request_headers) const { + ASSERT(segment_context_); + + // For SkyWalking Entry Span, Envoy does not need to inject tracing context into the request + // headers. + if (is_entry_span_) { + ENVOY_LOG(debug, "Skip tracing context injection for SkyWalking Entry Span"); + return; + } + + ENVOY_LOG(debug, "Inject or update SkyWalking propagation header in upstream request headers"); + const_cast(this)->setPeerAddress(std::string(request_headers.getHostValue())); + + ENVOY_LOG(trace, "'sw8' header: '({}) - ({}) - ({}) - ({}) - ({}) - ({}) - ({}) - ({})'", + sampled_, segment_context_->traceId(), segment_context_->traceSegmentId(), span_id_, + segment_context_->service(), segment_context_->serviceInstance(), + segment_context_->rootSpanStore()->operation(), peer_address_); + + // Reference: + // https://github.com/apache/skywalking/blob/v8.1.0/docs/en/protocols/Skywalking-Cross-Process-Propagation-Headers-Protocol-v3.md. + const auto value = absl::StrCat(sampled_, "-", base64Encode(segment_context_->traceId()), "-", + base64Encode(segment_context_->traceSegmentId()), "-", span_id_, + "-", base64Encode(segment_context_->service()), "-", + base64Encode(segment_context_->serviceInstance()), "-", + base64Encode(segment_context_->rootSpanStore()->operation()), "-", + base64Encode(peer_address_)); + request_headers.setReferenceKey(propagationHeader(), value); +} + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/skywalking_types.h b/source/extensions/tracers/skywalking/skywalking_types.h new file mode 100644 index 000000000000..eacd9a94a075 --- /dev/null +++ b/source/extensions/tracers/skywalking/skywalking_types.h @@ -0,0 +1,313 @@ +#pragma once + +#include +#include + +#include "envoy/common/random_generator.h" +#include "envoy/common/time.h" +#include "envoy/http/header_map.h" +#include "envoy/tracing/http_tracer.h" + +#include "language-agent/Tracing.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +class SegmentContext; +using SegmentContextSharedPtr = std::shared_ptr; + +class SpanStore; +using SpanStorePtr = std::unique_ptr; + +class SpanContext; +using SpanContextPtr = std::unique_ptr; + +class SpanContext : public Logger::Loggable { +public: + /* + * Parse the context of the previous span from the request and decide whether to sample it or + * not. + * + * @param headers The request headers. + * @return SpanContextPtr The previous span context parsed from request headers. + */ + static SpanContextPtr spanContextFromRequest(Http::RequestHeaderMap& headers); + + // Sampling flag. This field can only be 0 or 1. 1 means this trace need to be sampled and send to + // backend. + int sampled_{0}; + + // This span id points to the parent span in parent trace segment. + int span_id_{0}; + + std::string trace_id_; + + // This trace segment id points to the parent trace segment. + std::string trace_segment_id_; + + std::string service_; + std::string service_instance_; + + // Operation Name of the first entry span in the parent segment. + std::string endpoint_; + + // Target address used at client side of this request. The network address(not must be IP + port) + // used at client side to access this target service. + std::string target_address_; + +private: + // Private default constructor. We can only create SpanContext by 'spanContextFromRequest'. + SpanContext() = default; +}; + +class SegmentContext : public Logger::Loggable { +public: + /* + * Create a new segment context based on the previous span context that parsed from request + * headers. + * + * @param previous_span_context The previous span context. + * @param random_generator The random generator that used to create trace id and segment id. + * @param decision The tracing decision. + */ + SegmentContext(SpanContextPtr&& previous_span_context, Tracing::Decision decision, + Random::RandomGenerator& random_generator); + + /* + * Set service name. + * + * @param service The service name. + */ + void setService(const std::string& service) { service_ = service; } + + /* + * Set service instance name. + * + * @param service_instance The service instance name. + */ + void setServiceInstance(const std::string& service_instance) { + service_instance_ = service_instance; + } + + /* + * Create a new SpanStore object and return its pointer. The ownership of the newly created + * SpanStore object belongs to the current segment context. + * + * @param parent_store The pointer that point to parent SpanStore object. + * @return SpanStore* The pointer that point to newly created SpanStore object. + */ + SpanStore* createSpanStore(const SpanStore* parent_store); + + /* + * Get all SpanStore objects in the current segment. + */ + const std::vector& spanList() const { return span_list_; } + + /* + * Get root SpanStore object in the current segment. + */ + const SpanStore* rootSpanStore() { return span_list_.empty() ? nullptr : span_list_[0].get(); } + + int sampled() const { return sampled_; } + const std::string& traceId() const { return trace_id_; } + const std::string& traceSegmentId() const { return trace_segment_id_; } + + const std::string& service() const { return service_; } + const std::string& serviceInstance() const { return service_instance_; } + + SpanContext* previousSpanContext() const { return previous_span_context_.get(); } + +private: + int sampled_{0}; + // This value is unique in the entire tracing link. If previous_context is null, we will use + // random_generator to create a trace id. + std::string trace_id_; + // Envoy creates a new span when it accepts a new HTTP request. This span and all of its child + // spans belong to the same segment and share the segment id. + std::string trace_segment_id_; + + std::string service_; + std::string service_instance_; + + // The SegmentContext parsed from the request headers. If no propagation headers in request then + // this will be nullptr. + SpanContextPtr previous_span_context_; + + std::vector span_list_; +}; + +using Tag = std::pair; + +/* + * A helper class for the SkyWalking span and is used to store all span-related data, including span + * id, parent span id, tags and so on. Whenever we create a new span, we create a new SpanStore + * object. The new span will hold a pointer to the newly created SpanStore object and write data to + * it or get data from it. + */ +class SpanStore : public Logger::Loggable { +public: + /* + * Construct a SpanStore object using span context and time source. + * + * @param segment_context The pointer that point to current span context. This can not be null. + * @param time_source A time source to get the span end time. + */ + explicit SpanStore(SegmentContext* segment_context) : segment_context_(segment_context) {} + + /* + * Get operation name of span. + */ + const std::string& operation() const { return operation_; } + + /* + * Get peer address. The peer in SkyWalking is different with the tag value of 'peer.address'. The + * tag value of 'peer.address' in Envoy is downstream address and the peer in SkyWalking is + * upstream address. + */ + const std::string& peerAddress() const { return peer_address_; } + + /* + * Get span start time. + */ + uint64_t startTime() const { return start_time_; } + + /* + * Get span end time. + */ + uint64_t endTime() const { return end_time_; } + + /* + * Get span tags. + */ + const std::vector& tags() const { return tags_; } + + /* + * Get span logs. + */ + const std::vector& logs() const { return logs_; } + + /* + * Get span sampling flag. + */ + int sampled() const { return sampled_; } + + /* + * Get span id. + */ + int spanId() const { return span_id_; } + + /* + * Get parent span id. + */ + int parentSpanId() const { return parent_span_id_; } + + /* + * Determines if an error has occurred in the current span. + */ + bool isError() const { return is_error_; } + + /* + * Determines if the current span is an entry span. + * + * Reference: + * https://github.com/apache/skywalking/blob/v8.1.0/docs/en/protocols/Trace-Data-Protocol-v3.md + */ + bool isEntrySpan() const { return is_entry_span_; } + + /* + * Set span start time. This is the time when the HTTP request started, not the time when the span + * was created. + */ + void setStartTime(uint64_t start_time) { start_time_ = start_time; } + + /* + * Set span end time. It is meaningless for now. End time will be set by finish. + */ + void setEndTime(uint64_t end_time) { end_time_ = end_time; } + + /* + * Set operation name. + */ + void setOperation(const std::string& operation) { operation_ = operation; } + + /* + * Set peer address. In SkyWalking, the peer address is only set in Exit Span. And it should the + * upstream address. Since the upstream address cannot be obtained at the request stage, the + * request host is used instead. + */ + void setPeerAddress(const std::string& peer_address) { peer_address_ = peer_address; } + + /* + * Set if the current span has an error. + */ + void setAsError(bool is_error) { is_error_ = is_error; } + + /* + * Set if the current span is a entry span. + */ + void setAsEntrySpan(bool is_entry_span) { is_entry_span_ = is_entry_span; } + + /* + * Add a new tag entry to current span. + */ + void addTag(absl::string_view name, absl::string_view value) { tags_.emplace_back(name, value); } + + /* + * Add a new log entry to current span. Due to different data formats, log is temporarily not + * supported. + */ + void addLog(SystemTime, const std::string&) {} + + /* + * Set span id of current span. The span id in each segment is started from 0. When new span is + * created, its span id is the current max span id plus 1. + */ + void setSpanId(int span_id) { span_id_ = span_id; } + + /* + * Set parent span id. Notice that in SkyWalking, the parent span and the child span belong to the + * same segment. The first span of each segment has a parent span id of -1. + */ + void setParentSpanId(int parent_span_id) { parent_span_id_ = parent_span_id; } + + /* + * Set sampling flag. In general, the sampling flag of span is consistent with the current span + * context. + */ + void setSampled(int sampled) { sampled_ = sampled == 0 ? 0 : 1; } + + /* + * Inject current span context information to request headers. This will update original + * propagation headers. + * + * @param request_headers The request headers. + */ + void injectContext(Http::RequestHeaderMap& request_headers) const; + +private: + SegmentContext* segment_context_{nullptr}; + + int sampled_{0}; + + int span_id_{0}; + int parent_span_id_{-1}; + + uint64_t start_time_{0}; + uint64_t end_time_{0}; + + std::string operation_; + std::string peer_address_; + + bool is_error_{false}; + bool is_entry_span_{true}; + + std::vector tags_; + std::vector logs_; +}; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/trace_segment_reporter.cc b/source/extensions/tracers/skywalking/trace_segment_reporter.cc new file mode 100644 index 000000000000..5ef0046dc800 --- /dev/null +++ b/source/extensions/tracers/skywalking/trace_segment_reporter.cc @@ -0,0 +1,178 @@ +#include "extensions/tracers/skywalking/trace_segment_reporter.h" + +#include "envoy/http/header_map.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +namespace { + +Http::RegisterCustomInlineHeader + authentication_handle(Http::CustomHeaders::get().Authentication); + +// Convert SegmentContext to SegmentObject. +TraceSegmentPtr toSegmentObject(const SegmentContext& segment_context) { + auto new_segment_ptr = std::make_unique(); + SegmentObject& segment_object = *new_segment_ptr; + + segment_object.set_traceid(segment_context.traceId()); + segment_object.set_tracesegmentid(segment_context.traceSegmentId()); + segment_object.set_service(segment_context.service()); + segment_object.set_serviceinstance(segment_context.serviceInstance()); + + for (const auto& span_store : segment_context.spanList()) { + if (!span_store->sampled()) { + continue; + } + auto* span = segment_object.mutable_spans()->Add(); + + span->set_spanlayer(SpanLayer::Http); + span->set_spantype(span_store->isEntrySpan() ? SpanType::Entry : SpanType::Exit); + // Please check + // https://github.com/apache/skywalking/blob/master/oap-server/server-bootstrap/src/main/resources/component-libraries.yml + // get more information. + span->set_componentid(9000); + + if (!span_store->peerAddress().empty() && span_store->isEntrySpan()) { + span->set_peer(span_store->peerAddress()); + } + + span->set_spanid(span_store->spanId()); + span->set_parentspanid(span_store->parentSpanId()); + + span->set_starttime(span_store->startTime()); + span->set_endtime(span_store->endTime()); + + span->set_iserror(span_store->isError()); + + span->set_operationname(span_store->operation()); + + auto& tags = *span->mutable_tags(); + tags.Reserve(span_store->tags().size()); + + for (auto& span_tag : span_store->tags()) { + KeyStringValuePair* new_tag = tags.Add(); + new_tag->set_key(span_tag.first); + new_tag->set_value(span_tag.second); + } + + SpanContext* previous_span_context = segment_context.previousSpanContext(); + + if (!previous_span_context || !span_store->isEntrySpan()) { + continue; + } + + auto* ref = span->mutable_refs()->Add(); + ref->set_traceid(previous_span_context->trace_id_); + ref->set_parenttracesegmentid(previous_span_context->trace_segment_id_); + ref->set_parentspanid(previous_span_context->span_id_); + ref->set_parentservice(previous_span_context->service_); + ref->set_parentserviceinstance(previous_span_context->service_instance_); + ref->set_parentendpoint(previous_span_context->endpoint_); + ref->set_networkaddressusedatpeer(previous_span_context->target_address_); + } + return new_segment_ptr; +} + +} // namespace + +TraceSegmentReporter::TraceSegmentReporter(Grpc::AsyncClientFactoryPtr&& factory, + Event::Dispatcher& dispatcher, + Random::RandomGenerator& random_generator, + SkyWalkingTracerStats& stats, + const SkyWalkingClientConfig& client_config) + : tracing_stats_(stats), client_config_(client_config), client_(factory->create()), + service_method_(*Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "TraceSegmentReportService.collect")), + random_generator_(random_generator) { + + static constexpr uint32_t RetryInitialDelayMs = 500; + static constexpr uint32_t RetryMaxDelayMs = 30000; + backoff_strategy_ = std::make_unique( + RetryInitialDelayMs, RetryMaxDelayMs, random_generator_); + + retry_timer_ = dispatcher.createTimer([this]() -> void { establishNewStream(); }); + establishNewStream(); +} + +void TraceSegmentReporter::onCreateInitialMetadata(Http::RequestHeaderMap& metadata) { + if (!client_config_.backendToken().empty()) { + metadata.setInline(authentication_handle.handle(), client_config_.backendToken()); + } +} + +void TraceSegmentReporter::report(const SegmentContext& segment_context) { + sendTraceSegment(toSegmentObject(segment_context)); +} + +void TraceSegmentReporter::sendTraceSegment(TraceSegmentPtr request) { + ASSERT(request); + ENVOY_LOG(trace, "Try to report segment to SkyWalking Server:\n{}", request->DebugString()); + + if (stream_ != nullptr) { + tracing_stats_.segments_sent_.inc(); + stream_->sendMessage(*request, false); + return; + } + // Null stream_ and cache segment data temporarily. + delayed_segments_cache_.emplace(std::move(request)); + if (delayed_segments_cache_.size() > client_config_.maxCacheSize()) { + tracing_stats_.segments_dropped_.inc(); + delayed_segments_cache_.pop(); + } +} + +void TraceSegmentReporter::flushTraceSegments() { + ENVOY_LOG(debug, "Flush segments in cache to SkyWalking backend service"); + while (!delayed_segments_cache_.empty() && stream_ != nullptr) { + tracing_stats_.segments_sent_.inc(); + tracing_stats_.segments_flushed_.inc(); + stream_->sendMessage(*delayed_segments_cache_.front(), false); + delayed_segments_cache_.pop(); + } + tracing_stats_.cache_flushed_.inc(); +} + +void TraceSegmentReporter::closeStream() { + if (stream_ != nullptr) { + flushTraceSegments(); + stream_->closeStream(); + } +} + +void TraceSegmentReporter::onRemoteClose(Grpc::Status::GrpcStatus status, + const std::string& message) { + ENVOY_LOG(debug, "{} gRPC stream closed: {}, {}", service_method_.name(), status, message); + stream_ = nullptr; + handleFailure(); +} + +void TraceSegmentReporter::establishNewStream() { + ENVOY_LOG(debug, "Try to create new {} gRPC stream for reporter", service_method_.name()); + stream_ = client_->start(service_method_, *this, Http::AsyncClient::StreamOptions()); + if (stream_ == nullptr) { + ENVOY_LOG(debug, "Failed to create {} gRPC stream", service_method_.name()); + return; + } + // TODO(wbpcode): Even if stream_ is not empty, there is no guarantee that the connection will be + // established correctly. If there is a connection failure, the onRemoteClose method will be + // called. Currently, we lack a way to determine whether the connection is truly available. This + // may cause partial data loss. + if (!delayed_segments_cache_.empty()) { + flushTraceSegments(); + } + backoff_strategy_->reset(); +} + +void TraceSegmentReporter::handleFailure() { setRetryTimer(); } + +void TraceSegmentReporter::setRetryTimer() { + retry_timer_->enableTimer(std::chrono::milliseconds(backoff_strategy_->nextBackOffMs())); +} + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/trace_segment_reporter.h b/source/extensions/tracers/skywalking/trace_segment_reporter.h new file mode 100644 index 000000000000..fd70d819917b --- /dev/null +++ b/source/extensions/tracers/skywalking/trace_segment_reporter.h @@ -0,0 +1,83 @@ +#pragma once + +#include + +#include "envoy/config/trace/v3/skywalking.pb.h" +#include "envoy/grpc/async_client_manager.h" + +#include "common/Common.pb.h" +#include "common/common/backoff_strategy.h" +#include "common/grpc/async_client_impl.h" + +#include "extensions/tracers/skywalking/skywalking_client_config.h" +#include "extensions/tracers/skywalking/skywalking_stats.h" +#include "extensions/tracers/skywalking/skywalking_types.h" + +#include "language-agent/Tracing.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +using TraceSegmentPtr = std::unique_ptr; + +class TraceSegmentReporter : public Logger::Loggable, + public Grpc::AsyncStreamCallbacks { +public: + explicit TraceSegmentReporter(Grpc::AsyncClientFactoryPtr&& factory, + Event::Dispatcher& dispatcher, Random::RandomGenerator& random, + SkyWalkingTracerStats& stats, + const SkyWalkingClientConfig& client_config); + + // Grpc::AsyncStreamCallbacks + void onCreateInitialMetadata(Http::RequestHeaderMap& metadata) override; + void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override {} + void onReceiveMessage(std::unique_ptr&&) override {} + void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} + void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override; + + /* + * Flush all cached segment objects to the back-end tracing service and close the GRPC stream. + */ + void closeStream(); + + /* + * Convert the current span context into a segment object and report it to the back-end tracing + * service through the GRPC stream. + * + * @param span_context The span context. + */ + void report(const SegmentContext& span_context); + +private: + void flushTraceSegments(); + + void sendTraceSegment(TraceSegmentPtr request); + void establishNewStream(); + void handleFailure(); + void setRetryTimer(); + + SkyWalkingTracerStats& tracing_stats_; + + const SkyWalkingClientConfig& client_config_; + + Grpc::AsyncClient client_; + Grpc::AsyncStream stream_{}; + const Protobuf::MethodDescriptor& service_method_; + + Random::RandomGenerator& random_generator_; + // If the connection is unavailable when reporting data, the created SegmentObject will be cached + // in the queue, and when a new connection is established, the cached data will be reported. + std::queue delayed_segments_cache_; + + Event::TimerPtr retry_timer_; + BackOffStrategyPtr backoff_strategy_; +}; + +using TraceSegmentReporterPtr = std::unique_ptr; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/tracer.cc b/source/extensions/tracers/skywalking/tracer.cc new file mode 100644 index 000000000000..f3845b9c4213 --- /dev/null +++ b/source/extensions/tracers/skywalking/tracer.cc @@ -0,0 +1,82 @@ +#include "extensions/tracers/skywalking/tracer.h" + +#include + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +constexpr absl::string_view StatusCodeTag = "status_code"; +constexpr absl::string_view UrlTag = "url"; + +namespace { + +uint64_t getTimestamp(SystemTime time) { + return std::chrono::duration_cast(time.time_since_epoch()).count(); +} + +} // namespace + +Tracing::SpanPtr Tracer::startSpan(const Tracing::Config&, SystemTime start_time, + const std::string& operation, + SegmentContextSharedPtr segment_context, Span* parent) { + SpanStore* span_store = segment_context->createSpanStore(parent ? parent->spanStore() : nullptr); + + span_store->setStartTime(getTimestamp(start_time)); + + span_store->setOperation(operation); + + return std::make_unique(std::move(segment_context), span_store, *this); +} + +void Span::setOperation(absl::string_view operation) { + span_store_->setOperation(std::string(operation)); +} + +void Span::setTag(absl::string_view name, absl::string_view value) { + if (name == Tracing::Tags::get().HttpUrl) { + span_store_->addTag(UrlTag, value); + return; + } + + if (name == Tracing::Tags::get().HttpStatusCode) { + span_store_->addTag(StatusCodeTag, value); + return; + } + + if (name == Tracing::Tags::get().Error) { + span_store_->setAsError(value == Tracing::Tags::get().True); + } + + span_store_->addTag(name, value); +} + +// Logs in the SkyWalking format are temporarily unsupported. +void Span::log(SystemTime, const std::string&) {} + +void Span::finishSpan() { + span_store_->setEndTime(DateUtil::nowToMilliseconds(tracer_.time_source_)); + tryToReportSpan(); +} + +void Span::injectContext(Http::RequestHeaderMap& request_headers) { + span_store_->injectContext(request_headers); +} + +Tracing::SpanPtr Span::spawnChild(const Tracing::Config& config, const std::string& operation_name, + SystemTime start_time) { + // The new child span will share the same context with the parent span. + return tracer_.startSpan(config, start_time, operation_name, segment_context_, this); +} + +void Span::setSampled(bool sampled) { span_store_->setSampled(sampled ? 1 : 0); } + +std::string Span::getBaggage(absl::string_view) { return EMPTY_STRING; } + +void Span::setBaggage(absl::string_view, absl::string_view) {} + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/tracers/skywalking/tracer.h b/source/extensions/tracers/skywalking/tracer.h new file mode 100644 index 000000000000..d28276b232cd --- /dev/null +++ b/source/extensions/tracers/skywalking/tracer.h @@ -0,0 +1,117 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" + +#include "common/tracing/http_tracer_impl.h" + +#include "extensions/tracers/skywalking/skywalking_types.h" +#include "extensions/tracers/skywalking/trace_segment_reporter.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +class Span; + +class Tracer { +public: + explicit Tracer(TimeSource& time_source) : time_source_(time_source) {} + virtual ~Tracer() { reporter_->closeStream(); } + + /* + * Set a trace segment reporter to the current Tracer. Whenever a SkyWalking segment ends, the + * reporter will be used to report segment data. + * + * @param reporter The unique ptr of trace segment reporter. + */ + void setReporter(TraceSegmentReporterPtr&& reporter) { reporter_ = std::move(reporter); } + + /* + * Report trace segment data to backend tracing service. + * + * @param segment_context The segment context. + */ + void report(const SegmentContext& segment_context) { return reporter_->report(segment_context); } + + /* + * Create a new span based on the segment context and parent span. + * + * @param config The tracing config. + * @param start_time Start time of span. + * @param operation Operation name of span. + * @param segment_context The SkyWalking segment context. The newly created span belongs to this + * segment. + * @param parent The parent span pointer. If parent is null, then the newly created span is first + * span of this segment. + * + * @return The unique ptr to the newly created span. + */ + Tracing::SpanPtr startSpan(const Tracing::Config& config, SystemTime start_time, + const std::string& operation, SegmentContextSharedPtr segment_context, + Span* parent); + + TimeSource& time_source_; + +private: + TraceSegmentReporterPtr reporter_; +}; + +using TracerPtr = std::unique_ptr; + +class Span : public Tracing::Span { +public: + /* + * Constructor of span. + * + * @param segment_context The SkyWalking segment context. + * @param span_store Pointer to a SpanStore object. Whenever a new span is created, a new + * SpanStore object is created and stored in the segment context. This parameter can never be + * null. + * @param tracer Reference to tracer. + */ + Span(SegmentContextSharedPtr segment_context, SpanStore* span_store, Tracer& tracer) + : segment_context_(std::move(segment_context)), span_store_(span_store), tracer_(tracer) {} + + // Tracing::Span + void setOperation(absl::string_view operation) override; + void setTag(absl::string_view name, absl::string_view value) override; + void log(SystemTime timestamp, const std::string& event) override; + void finishSpan() override; + void injectContext(Http::RequestHeaderMap& request_headers) override; + Tracing::SpanPtr spawnChild(const Tracing::Config& config, const std::string& name, + SystemTime start_time) override; + void setSampled(bool sampled) override; + std::string getBaggage(absl::string_view key) override; + void setBaggage(absl::string_view key, absl::string_view value) override; + + /* + * Get pointer to corresponding SpanStore object. This method is mainly used in testing. Used to + * check the internal data of the span. + */ + SpanStore* spanStore() const { return span_store_; } + SegmentContext* segmentContext() const { return segment_context_.get(); } + +private: + void tryToReportSpan() { + // If the current span is the root span of the entire segment and its sampling flag is not + // false, the data for the entire segment is reported. Please ensure that the root span is the + // last span to end in the entire segment. + if (span_store_->sampled() && span_store_->spanId() == 0) { + tracer_.report(*segment_context_); + } + } + + SegmentContextSharedPtr segment_context_; + SpanStore* span_store_; + + Tracer& tracer_; +}; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/BUILD b/test/extensions/tracers/skywalking/BUILD new file mode 100644 index 000000000000..b18f8cfe91dc --- /dev/null +++ b/test/extensions/tracers/skywalking/BUILD @@ -0,0 +1,113 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_name = "envoy.tracers.skywalking", + deps = [ + "//source/extensions/tracers/skywalking:config", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/mocks/server:tracer_factory_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "skywalking_client_config_test", + srcs = ["skywalking_client_config_test.cc"], + extension_name = "envoy.tracers.skywalking", + deps = [ + "//source/extensions/tracers/skywalking:skywalking_client_config_lib", + "//test/mocks:common_lib", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "skywalking_types_test", + srcs = ["skywalking_types_test.cc"], + extension_name = "envoy.tracers.skywalking", + deps = [ + ":skywalking_test_helper", + "//source/extensions/tracers/skywalking:skywalking_types_lib", + "//test/mocks:common_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "trace_segment_reporter_test", + srcs = ["trace_segment_reporter_test.cc"], + extension_name = "envoy.tracers.skywalking", + deps = [ + ":skywalking_test_helper", + "//source/extensions/tracers/skywalking:trace_segment_reporter_lib", + "//test/mocks:common_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "skywalking_test_helper", + srcs = ["skywalking_test_helper.h"], + extension_name = "envoy.tracers.skywalking", + deps = [ + "//source/common/common:base64_lib", + "//source/common/common:hex_lib", + "//source/extensions/tracers/skywalking:skywalking_types_lib", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "tracer_test", + srcs = ["tracer_test.cc"], + extension_name = "envoy.tracers.skywalking", + deps = [ + ":skywalking_test_helper", + "//source/extensions/tracers/skywalking:skywalking_tracer_lib", + "//test/mocks:common_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/mocks/stats:stats_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "skywalking_tracer_impl_test", + srcs = ["skywalking_tracer_impl_test.cc"], + extension_name = "envoy.tracers.skywalking", + deps = [ + ":skywalking_test_helper", + "//source/extensions/tracers/skywalking:skywalking_tracer_lib", + "//test/mocks:common_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/server:tracer_factory_context_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/tracers/skywalking/config_test.cc b/test/extensions/tracers/skywalking/config_test.cc new file mode 100644 index 000000000000..19c966cf7cb7 --- /dev/null +++ b/test/extensions/tracers/skywalking/config_test.cc @@ -0,0 +1,88 @@ +#include "envoy/config/trace/v3/http_tracer.pb.h" +#include "envoy/config/trace/v3/skywalking.pb.h" +#include "envoy/config/trace/v3/skywalking.pb.validate.h" + +#include "extensions/tracers/skywalking/config.h" + +#include "test/mocks/server/tracer_factory.h" +#include "test/mocks/server/tracer_factory_context.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Eq; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { +namespace { + +TEST(SkyWalkingTracerConfigTest, SkyWalkingHttpTracer) { + NiceMock context; + EXPECT_CALL(context.server_factory_context_.cluster_manager_, get(Eq("fake_cluster"))) + .WillRepeatedly( + Return(&context.server_factory_context_.cluster_manager_.thread_local_cluster_)); + ON_CALL(*context.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + features()) + .WillByDefault(Return(Upstream::ClusterInfo::Features::HTTP2)); + + const std::string yaml_string = R"EOF( + http: + name: skywalking + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + )EOF"; + envoy::config::trace::v3::Tracing configuration; + TestUtility::loadFromYaml(yaml_string, configuration); + + SkyWalkingTracerFactory factory; + auto message = Config::Utility::translateToFactoryConfig( + configuration.http(), ProtobufMessage::getStrictValidationVisitor(), factory); + Tracing::HttpTracerSharedPtr skywalking_tracer = factory.createHttpTracer(*message, context); + EXPECT_NE(nullptr, skywalking_tracer); +} + +TEST(SkyWalkingTracerConfigTest, SkyWalkingHttpTracerWithClientConfig) { + NiceMock context; + EXPECT_CALL(context.server_factory_context_.cluster_manager_, get(Eq("fake_cluster"))) + .WillRepeatedly( + Return(&context.server_factory_context_.cluster_manager_.thread_local_cluster_)); + ON_CALL(*context.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + features()) + .WillByDefault(Return(Upstream::ClusterInfo::Features::HTTP2)); + + const std::string yaml_string = R"EOF( + http: + name: skywalking + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + client_config: + backend_token: "A fake auth string for SkyWalking test" + service_name: "Test Service" + instance_name: "Test Instance" + max_cache_size: 2333 + )EOF"; + envoy::config::trace::v3::Tracing configuration; + TestUtility::loadFromYaml(yaml_string, configuration); + + SkyWalkingTracerFactory factory; + auto message = Config::Utility::translateToFactoryConfig( + configuration.http(), ProtobufMessage::getStrictValidationVisitor(), factory); + Tracing::HttpTracerSharedPtr skywalking_tracer = factory.createHttpTracer(*message, context); + EXPECT_NE(nullptr, skywalking_tracer); +} + +} // namespace +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/skywalking_client_config_test.cc b/test/extensions/tracers/skywalking/skywalking_client_config_test.cc new file mode 100644 index 000000000000..c0d4131a8e9d --- /dev/null +++ b/test/extensions/tracers/skywalking/skywalking_client_config_test.cc @@ -0,0 +1,100 @@ +#include "extensions/tracers/skywalking/skywalking_client_config.h" + +#include "test/mocks/common.h" +#include "test/mocks/server/tracer_factory_context.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { +namespace { + +class SkyWalkingClientConfigTest : public testing::Test { +public: + void setupSkyWalkingClientConfig(const std::string& yaml_string) { + auto& local_info = context_.server_factory_context_.local_info_; + + ON_CALL(local_info, clusterName()).WillByDefault(ReturnRef(test_string)); + ON_CALL(local_info, nodeName()).WillByDefault(ReturnRef(test_string)); + + envoy::config::trace::v3::SkyWalkingConfig proto_config; + TestUtility::loadFromYaml(yaml_string, proto_config); + + client_config_ = + std::make_unique(context_, proto_config.client_config()); + } + +protected: + NiceMock context_; + + std::string test_string = "ABCDEFGHIJKLMN"; + + SkyWalkingClientConfigPtr client_config_; +}; + +// Test whether the default value can be set correctly when there is no proto client config +// provided. +TEST_F(SkyWalkingClientConfigTest, NoProtoClientConfigTest) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + )EOF"; + + setupSkyWalkingClientConfig(yaml_string); + + EXPECT_EQ(client_config_->service(), test_string); + EXPECT_EQ(client_config_->serviceInstance(), test_string); + EXPECT_EQ(client_config_->maxCacheSize(), 1024); + EXPECT_EQ(client_config_->backendToken(), ""); +} + +// Test whether the client config can work correctly when the proto client config is provided. +TEST_F(SkyWalkingClientConfigTest, WithProtoClientConfigTest) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + client_config: + backend_token: "FAKE_FAKE_FAKE_FAKE_FAKE_FAKE" + service_name: "FAKE_FAKE_FAKE" + instance_name: "FAKE_FAKE_FAKE" + max_cache_size: 2333 + )EOF"; + + setupSkyWalkingClientConfig(yaml_string); + + EXPECT_EQ(client_config_->service(), "FAKE_FAKE_FAKE"); + EXPECT_EQ(client_config_->serviceInstance(), "FAKE_FAKE_FAKE"); + EXPECT_EQ(client_config_->maxCacheSize(), 2333); + EXPECT_EQ(client_config_->backendToken(), "FAKE_FAKE_FAKE_FAKE_FAKE_FAKE"); +} + +// Test whether the client config can get default value for service name and instance name. +TEST_F(SkyWalkingClientConfigTest, BothLocalInfoAndClientConfigEmptyTest) { + test_string = ""; + + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + )EOF"; + + setupSkyWalkingClientConfig(yaml_string); + + EXPECT_EQ(client_config_->service(), "EnvoyProxy"); + EXPECT_EQ(client_config_->serviceInstance(), "EnvoyProxy"); +} + +} // namespace +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/skywalking_test_helper.h b/test/extensions/tracers/skywalking/skywalking_test_helper.h new file mode 100644 index 000000000000..e158bb535da8 --- /dev/null +++ b/test/extensions/tracers/skywalking/skywalking_test_helper.h @@ -0,0 +1,77 @@ +#pragma once + +#include "common/common/base64.h" +#include "common/common/hex.h" + +#include "extensions/tracers/skywalking/skywalking_types.h" + +#include "test/test_common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { + +/* + * A simple helper class for auxiliary testing. Contains some simple static functions, such as + * encoding, generating random id, creating SpanContext, etc. + */ +class SkyWalkingTestHelper { +public: + static std::string generateId(Random::RandomGenerator& random) { + return absl::StrCat(Hex::uint64ToHex(random.random()), Hex::uint64ToHex(random.random())); + } + + static std::string base64Encode(absl::string_view input) { + return Base64::encode(input.data(), input.length()); + } + + static SegmentContextSharedPtr createSegmentContext(bool sampled, std::string seed, + std::string prev_seed, + Random::RandomGenerator& random) { + SpanContextPtr previous_span_context; + if (!prev_seed.empty()) { + std::string header_value = + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", sampled ? 1 : 0, base64Encode(generateId(random)), + base64Encode(generateId(random)), random.random(), + base64Encode(prev_seed + "#SERVICE"), base64Encode(prev_seed + "#INSTANCE"), + base64Encode(prev_seed + "#ENDPOINT"), base64Encode(prev_seed + "#ADDRESS")); + + Http::TestRequestHeaderMapImpl request_headers{{"sw8", header_value}}; + previous_span_context = SpanContext::spanContextFromRequest(request_headers); + ASSERT(previous_span_context); + } + Tracing::Decision decision; + decision.traced = sampled; + decision.reason = Tracing::Reason::Sampling; + + auto segment_context = + std::make_shared(std::move(previous_span_context), decision, random); + + segment_context->setService(seed + "#SERVICE"); + segment_context->setServiceInstance(seed + "#INSTANCE"); + + return segment_context; + } + + static SpanStore* createSpanStore(SegmentContext* segment_context, SpanStore* parent_span_store, + std::string seed) { + SpanStore* span_store = segment_context->createSpanStore(parent_span_store); + + span_store->setAsError(false); + span_store->setOperation(seed + "#OPERATION"); + span_store->setPeerAddress("0.0.0.0"); + span_store->setStartTime(22222222); + span_store->setEndTime(33333333); + + span_store->addTag(seed + "#TAG_KEY_A", seed + "#TAG_VALUE_A"); + span_store->addTag(seed + "#TAG_KEY_B", seed + "#TAG_VALUE_B"); + span_store->addTag(seed + "#TAG_KEY_C", seed + "#TAG_VALUE_C"); + return span_store; + } +}; + +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc b/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc new file mode 100644 index 000000000000..cb5075665dcc --- /dev/null +++ b/test/extensions/tracers/skywalking/skywalking_tracer_impl_test.cc @@ -0,0 +1,179 @@ +#include "extensions/tracers/skywalking/skywalking_tracer_impl.h" + +#include "test/extensions/tracers/skywalking/skywalking_test_helper.h" +#include "test/mocks/common.h" +#include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/tracing/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { +namespace { + +class SkyWalkingDriverTest : public testing::Test { +public: + void setupSkyWalkingDriver(const std::string& yaml_string) { + auto mock_client_factory = std::make_unique>(); + auto mock_client = std::make_unique>(); + mock_stream_ptr_ = std::make_unique>(); + + EXPECT_CALL(*mock_client, startRaw(_, _, _, _)).WillOnce(Return(mock_stream_ptr_.get())); + EXPECT_CALL(*mock_client_factory, create()).WillOnce(Return(ByMove(std::move(mock_client)))); + + auto& factory_context = context_.server_factory_context_; + + EXPECT_CALL(factory_context.cluster_manager_.async_client_manager_, + factoryForGrpcService(_, _, _)) + .WillOnce(Return(ByMove(std::move(mock_client_factory)))); + + EXPECT_CALL(factory_context.thread_local_.dispatcher_, createTimer_(_)) + .WillOnce(Invoke([](Event::TimerCb) { return new NiceMock(); })); + + ON_CALL(factory_context.local_info_, clusterName()).WillByDefault(ReturnRef(test_string)); + ON_CALL(factory_context.local_info_, nodeName()).WillByDefault(ReturnRef(test_string)); + + TestUtility::loadFromYaml(yaml_string, config_); + driver_ = std::make_unique(config_, context_); + } + +protected: + NiceMock context_; + NiceMock mock_tracing_config_; + Event::SimulatedTimeSystem time_system_; + + std::unique_ptr> mock_stream_ptr_{nullptr}; + + envoy::config::trace::v3::SkyWalkingConfig config_; + std::string test_string = "ABCDEFGHIJKLMN"; + + DriverPtr driver_; +}; + +TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestWithClientConfig) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + client_config: + backend_token: "FAKE_FAKE_FAKE_FAKE_FAKE_FAKE" + service_name: "FAKE_FAKE_FAKE" + instance_name: "FAKE_FAKE_FAKE" + max_cache_size: 2333 + )EOF"; + setupSkyWalkingDriver(yaml_string); + + std::string trace_id = + SkyWalkingTestHelper::generateId(context_.server_factory_context_.api_.random_); + std::string segment_id = + SkyWalkingTestHelper::generateId(context_.server_factory_context_.api_.random_); + + // Create new span segment with previous span context. + std::string previous_header_value = + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 0, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 233333, + SkyWalkingTestHelper::base64Encode("SERVICE"), + SkyWalkingTestHelper::base64Encode("INSTATNCE"), + SkyWalkingTestHelper::base64Encode("ENDPOINT"), + SkyWalkingTestHelper::base64Encode("ADDRESS")); + + Http::TestRequestHeaderMapImpl request_headers{{"sw8", previous_header_value}, + {":path", "/path"}, + {":method", "GET"}, + {":authority", "test.com"}}; + + ON_CALL(mock_tracing_config_, operationName()) + .WillByDefault(Return(Tracing::OperationName::Ingress)); + + Tracing::Decision decision; + decision.traced = true; + + Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, request_headers, "TEST_OP", + time_system_.systemTime(), decision); + EXPECT_NE(nullptr, org_span.get()); + + Span* span = dynamic_cast(org_span.get()); + ASSERT(span); + + EXPECT_NE(nullptr, span->segmentContext()->previousSpanContext()); + + EXPECT_EQ("FAKE_FAKE_FAKE", span->segmentContext()->service()); + EXPECT_EQ("FAKE_FAKE_FAKE", span->segmentContext()->serviceInstance()); + + // Tracing decision will be overwrite by sampling flag in propagation headers. + EXPECT_EQ(0, span->segmentContext()->sampled()); + + // Since the sampling flag is false, no segment data is reported. + span->finishSpan(); + + auto& factory_context = context_.server_factory_context_; + EXPECT_EQ(0U, factory_context.scope_.counter("tracing.skywalking.segments_sent").value()); + + // Create new span segment with no previous span context. + Http::TestRequestHeaderMapImpl new_request_headers{ + {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; + + Tracing::SpanPtr org_new_span = driver_->startSpan(mock_tracing_config_, new_request_headers, "", + time_system_.systemTime(), decision); + + Span* new_span = dynamic_cast(org_new_span.get()); + ASSERT(new_span); + + EXPECT_EQ(nullptr, new_span->segmentContext()->previousSpanContext()); + + EXPECT_EQ(true, new_span->segmentContext()->sampled()); + + EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)); + new_span->finishSpan(); + EXPECT_EQ(1U, factory_context.scope_.counter("tracing.skywalking.segments_sent").value()); + + // Create new span segment with error propagation header. + Http::TestRequestHeaderMapImpl error_request_headers{{":path", "/path"}, + {":method", "GET"}, + {":authority", "test.com"}, + {"sw8", "xxxxxx-error-propagation-header"}}; + Tracing::SpanPtr org_null_span = driver_->startSpan( + mock_tracing_config_, error_request_headers, "TEST_OP", time_system_.systemTime(), decision); + + EXPECT_EQ(nullptr, dynamic_cast(org_null_span.get())); + + auto& null_span = *org_null_span; + EXPECT_EQ(typeid(null_span).name(), typeid(Tracing::NullSpan).name()); +} + +TEST_F(SkyWalkingDriverTest, SkyWalkingDriverStartSpanTestNoClientConfig) { + const std::string yaml_string = R"EOF( + grpc_service: + envoy_grpc: + cluster_name: fake_cluster + )EOF"; + + setupSkyWalkingDriver(yaml_string); + + Http::TestRequestHeaderMapImpl request_headers{ + {":path", "/path"}, {":method", "GET"}, {":authority", "test.com"}}; + + Tracing::SpanPtr org_span = driver_->startSpan(mock_tracing_config_, request_headers, "TEST_OP", + time_system_.systemTime(), Tracing::Decision()); + EXPECT_NE(nullptr, org_span.get()); + + Span* span = dynamic_cast(org_span.get()); + ASSERT(span); + + EXPECT_EQ(test_string, span->segmentContext()->service()); + EXPECT_EQ(test_string, span->segmentContext()->serviceInstance()); +} + +} // namespace +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/skywalking_types_test.cc b/test/extensions/tracers/skywalking/skywalking_types_test.cc new file mode 100644 index 000000000000..eb1d3147558f --- /dev/null +++ b/test/extensions/tracers/skywalking/skywalking_types_test.cc @@ -0,0 +1,343 @@ +#include "common/common/base64.h" +#include "common/common/hex.h" + +#include "extensions/tracers/skywalking/skywalking_types.h" + +#include "test/extensions/tracers/skywalking/skywalking_test_helper.h" +#include "test/mocks/common.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { +namespace { + +// Some constant strings for testing. +constexpr absl::string_view TEST_SERVICE = "EnvoyIngressForTest"; +constexpr absl::string_view TEST_INSTANCE = "node-2.3.4.5~ingress"; +constexpr absl::string_view TEST_ADDRESS = "255.255.255.255"; +constexpr absl::string_view TEST_ENDPOINT = "/POST/path/for/test"; + +// Test whether SpanContext can correctly parse data from propagation headers and throw exceptions +// when errors occur. +TEST(SpanContextTest, SpanContextCommonTest) { + NiceMock mock_random_generator; + ON_CALL(mock_random_generator, random()).WillByDefault(Return(uint64_t(23333))); + + std::string trace_id = SkyWalkingTestHelper::generateId(mock_random_generator); + std::string segment_id = SkyWalkingTestHelper::generateId(mock_random_generator); + + // No propagation header then previous span context will be null. + Http::TestRequestHeaderMapImpl headers_no_propagation; + auto null_span_context = SpanContext::spanContextFromRequest(headers_no_propagation); + EXPECT_EQ(nullptr, null_span_context.get()); + + // Create properly formatted propagation headers and test whether the propagation headers can be + // parsed correctly. + std::string header_value_with_right_format = + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 0, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 233333, + SkyWalkingTestHelper::base64Encode(TEST_SERVICE), + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS)); + + Http::TestRequestHeaderMapImpl headers_with_right_format{{"sw8", header_value_with_right_format}}; + + auto previous_span_context = SpanContext::spanContextFromRequest(headers_with_right_format); + EXPECT_NE(nullptr, previous_span_context.get()); + + // Verify that each field parsed from the propagation headers is correct. + EXPECT_EQ(previous_span_context->sampled_, 0); + EXPECT_EQ(previous_span_context->trace_id_, trace_id); + EXPECT_EQ(previous_span_context->trace_segment_id_, segment_id); + EXPECT_EQ(previous_span_context->span_id_, 233333); + EXPECT_EQ(previous_span_context->service_, TEST_SERVICE); + EXPECT_EQ(previous_span_context->service_instance_, TEST_INSTANCE); + EXPECT_EQ(previous_span_context->endpoint_, TEST_ENDPOINT); + EXPECT_EQ(previous_span_context->target_address_, TEST_ADDRESS); + + std::string header_value_with_sampled = + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 1, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 233333, + SkyWalkingTestHelper::base64Encode(TEST_SERVICE), + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS)); + + Http::TestRequestHeaderMapImpl headers_with_sampled{{"sw8", header_value_with_sampled}}; + + auto previous_span_context_with_sampled = + SpanContext::spanContextFromRequest(headers_with_sampled); + EXPECT_EQ(previous_span_context_with_sampled->sampled_, 1); + + // Test whether an exception can be correctly thrown when some fields are missing. + std::string header_value_lost_some_parts = + fmt::format("{}-{}-{}-{}-{}-{}", 0, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 3, + SkyWalkingTestHelper::base64Encode(TEST_SERVICE), + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE)); + + Http::TestRequestHeaderMapImpl headers_lost_some_parts{{"sw8", header_value_lost_some_parts}}; + + EXPECT_THROW_WITH_MESSAGE( + SpanContext::spanContextFromRequest(headers_lost_some_parts), EnvoyException, + fmt::format("Invalid propagation header for SkyWalking: {}", header_value_lost_some_parts)); + + // Test whether an exception can be correctly thrown when the sampling flag is wrong. + Http::TestRequestHeaderMapImpl headers_with_error_sampled{ + {"sw8", + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 3, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 3, + SkyWalkingTestHelper::base64Encode(TEST_SERVICE), + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS))}}; + + EXPECT_THROW_WITH_MESSAGE(SpanContext::spanContextFromRequest(headers_with_error_sampled), + EnvoyException, + "Invalid propagation header for SkyWalking: sampling flag can only be " + "'0' or '1' but '3' was provided"); + + // Test whether an exception can be correctly thrown when the span id format is wrong. + Http::TestRequestHeaderMapImpl headers_with_error_span_id{ + {"sw8", + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 1, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), "abc", + SkyWalkingTestHelper::base64Encode(TEST_SERVICE), + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS))}}; + + EXPECT_THROW_WITH_MESSAGE( + SpanContext::spanContextFromRequest(headers_with_error_span_id), EnvoyException, + "Invalid propagation header for SkyWalking: connot convert 'abc' to valid span id"); + + // Test whether an exception can be correctly thrown when a field is empty. + std::string header_value_with_empty_field = + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 1, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 4, "", + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS)); + Http::TestRequestHeaderMapImpl headers_with_empty_field{{"sw8", header_value_with_empty_field}}; + + EXPECT_THROW_WITH_MESSAGE( + SpanContext::spanContextFromRequest(headers_with_empty_field), EnvoyException, + fmt::format("Invalid propagation header for SkyWalking: {}", header_value_with_empty_field)); + + // Test whether an exception can be correctly thrown when a string is not properly encoded. + Http::TestRequestHeaderMapImpl headers_with_error_field{ + {"sw8", + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 1, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 4, "hhhhhhh", + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS))}}; + + EXPECT_THROW_WITH_MESSAGE(SpanContext::spanContextFromRequest(headers_with_error_field), + EnvoyException, + "Invalid propagation header for SkyWalking: parse error"); +} + +// Test whether the SegmentContext works normally when Envoy is the root node (Propagation headers +// does not exist). +TEST(SegmentContextTest, SegmentContextTestWithEmptyPreviousSpanContext) { + NiceMock mock_random_generator; + + ON_CALL(mock_random_generator, random()).WillByDefault(Return(233333)); + + SegmentContextSharedPtr segment_context = + SkyWalkingTestHelper::createSegmentContext(true, "NEW", "", mock_random_generator); + + // When previous span context is null, the value of the sampling flag depends on the tracing + // decision + EXPECT_EQ(segment_context->sampled(), 1); + // The SegmentContext will use random generator to create new trace id and new trace segment id. + EXPECT_EQ(segment_context->traceId(), SkyWalkingTestHelper::generateId(mock_random_generator)); + EXPECT_EQ(segment_context->traceSegmentId(), + SkyWalkingTestHelper::generateId(mock_random_generator)); + + EXPECT_EQ(segment_context->previousSpanContext(), nullptr); + + // Test whether the value of the fields can be set correctly and the value of the fields can be + // obtained correctly. + EXPECT_EQ(segment_context->service(), "NEW#SERVICE"); + segment_context->setService(std::string(TEST_SERVICE)); + EXPECT_EQ(segment_context->service(), TEST_SERVICE); + + EXPECT_EQ(segment_context->serviceInstance(), "NEW#INSTANCE"); + segment_context->setServiceInstance(std::string(TEST_INSTANCE)); + EXPECT_EQ(segment_context->serviceInstance(), TEST_INSTANCE); + + EXPECT_EQ(segment_context->rootSpanStore(), nullptr); + + // Test whether SegmentContext can correctly create SpanStore object with null parent SpanStore. + SpanStore* root_span = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), nullptr, "PARENT"); + EXPECT_NE(nullptr, root_span); + + // The span id of the first SpanStore in each SegmentContext is 0. Its parent span id is -1. + EXPECT_EQ(root_span->spanId(), 0); + EXPECT_EQ(root_span->parentSpanId(), -1); + + // Root span of current segment should be Entry Span. + EXPECT_EQ(root_span->isEntrySpan(), true); + + // Verify that the SpanStore object is correctly stored in the SegmentContext. + EXPECT_EQ(segment_context->spanList().size(), 1); + EXPECT_EQ(segment_context->spanList()[0].get(), root_span); + + // Test whether SegmentContext can correctly create SpanStore object with a parent SpanStore. + SpanStore* child_span = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), root_span, "CHILD"); + + EXPECT_NE(nullptr, child_span); + + EXPECT_EQ(child_span->spanId(), 1); + EXPECT_EQ(child_span->parentSpanId(), 0); + + // All child spans of current segment should be Exit Span. + EXPECT_EQ(child_span->isEntrySpan(), false); + + EXPECT_EQ(segment_context->spanList().size(), 2); + EXPECT_EQ(segment_context->spanList()[1].get(), child_span); +} + +// Test whether the SegmentContext can work normally when a previous span context exists. +TEST(SegmentContextTest, SegmentContextTestWithPreviousSpanContext) { + NiceMock mock_random_generator; + + ON_CALL(mock_random_generator, random()).WillByDefault(Return(23333)); + + std::string trace_id = SkyWalkingTestHelper::generateId(mock_random_generator); + std::string segment_id = SkyWalkingTestHelper::generateId(mock_random_generator); + + std::string header_value_with_right_format = + fmt::format("{}-{}-{}-{}-{}-{}-{}-{}", 0, SkyWalkingTestHelper::base64Encode(trace_id), + SkyWalkingTestHelper::base64Encode(segment_id), 233333, + SkyWalkingTestHelper::base64Encode(TEST_SERVICE), + SkyWalkingTestHelper::base64Encode(TEST_INSTANCE), + SkyWalkingTestHelper::base64Encode(TEST_ENDPOINT), + SkyWalkingTestHelper::base64Encode(TEST_ADDRESS)); + + Http::TestRequestHeaderMapImpl headers_with_right_format{{"sw8", header_value_with_right_format}}; + + auto previous_span_context = SpanContext::spanContextFromRequest(headers_with_right_format); + SpanContext* previous_span_context_bk = previous_span_context.get(); + + Tracing::Decision decision; + decision.traced = true; + + EXPECT_CALL(mock_random_generator, random()).WillRepeatedly(Return(666666)); + + SegmentContext segment_context(std::move(previous_span_context), decision, mock_random_generator); + + // When a previous span context exists, the sampling flag of the SegmentContext depends on + // previous span context rather than tracing decision. + EXPECT_EQ(segment_context.sampled(), 0); + + // When previous span context exists, the trace id of SegmentContext remains the same as that of + // previous span context. + EXPECT_EQ(segment_context.traceId(), trace_id); + // SegmentContext will always create a new trace segment id. + EXPECT_NE(segment_context.traceSegmentId(), segment_id); + + EXPECT_EQ(segment_context.previousSpanContext(), previous_span_context_bk); +} + +// Test whether SpanStore can work properly. +TEST(SpanStoreTest, SpanStoreCommonTest) { + NiceMock mock_random_generator; + + Event::SimulatedTimeSystem time_system; + Envoy::SystemTime now = time_system.systemTime(); + + ON_CALL(mock_random_generator, random()).WillByDefault(Return(23333)); + + // Create segment context and first span store. + SegmentContextSharedPtr segment_context = + SkyWalkingTestHelper::createSegmentContext(true, "CURR", "PREV", mock_random_generator); + SpanStore* root_store = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), nullptr, "ROOT"); + EXPECT_NE(nullptr, root_store); + EXPECT_EQ(3, root_store->tags().size()); + + root_store->addLog(now, "TestLogStringAndNeverBeStored"); + EXPECT_EQ(0, root_store->logs().size()); + + // The span id of the first SpanStore in each SegmentContext is 0. Its parent span id is -1. + EXPECT_EQ(0, root_store->spanId()); + EXPECT_EQ(-1, root_store->parentSpanId()); + + root_store->setSpanId(123); + EXPECT_EQ(123, root_store->spanId()); + root_store->setParentSpanId(234); + EXPECT_EQ(234, root_store->parentSpanId()); + + EXPECT_EQ(1, root_store->sampled()); + root_store->setSampled(0); + EXPECT_EQ(0, root_store->sampled()); + + // Test whether the value of the fields can be set correctly and the value of the fields can be + // obtained correctly. + EXPECT_EQ(true, root_store->isEntrySpan()); + root_store->setAsEntrySpan(false); + EXPECT_EQ(false, root_store->isEntrySpan()); + + EXPECT_EQ(false, root_store->isError()); + root_store->setAsError(true); + EXPECT_EQ(true, root_store->isError()); + + EXPECT_EQ("ROOT#OPERATION", root_store->operation()); + root_store->setOperation(""); + EXPECT_EQ("", root_store->operation()); + root_store->setOperation("oooooop"); + EXPECT_EQ("oooooop", root_store->operation()); + + EXPECT_EQ("0.0.0.0", root_store->peerAddress()); + root_store->setPeerAddress(std::string(TEST_ADDRESS)); + EXPECT_EQ(TEST_ADDRESS, root_store->peerAddress()); + + EXPECT_EQ(22222222, root_store->startTime()); + root_store->setStartTime(23333); + EXPECT_EQ(23333, root_store->startTime()); + + EXPECT_EQ(33333333, root_store->endTime()); + root_store->setEndTime(25555); + EXPECT_EQ(25555, root_store->endTime()); + + SpanStore* child_store = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), root_store, "CHILD"); + + // Test whether SpanStore can correctly inject propagation headers to request headers. + Http::TestRequestHeaderMapImpl request_headers_no_upstream{{":authority", "test.com"}}; + // Only child span (Exit Span) can inject context header to request headers. + child_store->injectContext(request_headers_no_upstream); + std::string expected_header_value = fmt::format( + "{}-{}-{}-{}-{}-{}-{}-{}", child_store->sampled(), + SkyWalkingTestHelper::base64Encode(SkyWalkingTestHelper::generateId(mock_random_generator)), + SkyWalkingTestHelper::base64Encode(SkyWalkingTestHelper::generateId(mock_random_generator)), + child_store->spanId(), SkyWalkingTestHelper::base64Encode("CURR#SERVICE"), + SkyWalkingTestHelper::base64Encode("CURR#INSTANCE"), + SkyWalkingTestHelper::base64Encode("oooooop"), + SkyWalkingTestHelper::base64Encode("test.com")); + + EXPECT_EQ(child_store->peerAddress(), "test.com"); + + EXPECT_EQ(request_headers_no_upstream.get_("sw8"), expected_header_value); +} + +} // namespace +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/trace_segment_reporter_test.cc b/test/extensions/tracers/skywalking/trace_segment_reporter_test.cc new file mode 100644 index 000000000000..fa3f59effdb5 --- /dev/null +++ b/test/extensions/tracers/skywalking/trace_segment_reporter_test.cc @@ -0,0 +1,246 @@ +#include "extensions/tracers/skywalking/trace_segment_reporter.h" + +#include "test/extensions/tracers/skywalking/skywalking_test_helper.h" +#include "test/mocks/common.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { +namespace { + +class TraceSegmentReporterTest : public testing::Test { +public: + void setupTraceSegmentReporter(const std::string& yaml_string) { + EXPECT_CALL(mock_dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb timer_cb) { + timer_cb_ = timer_cb; + return timer_; + })); + timer_ = new NiceMock(); + + auto mock_client_factory = std::make_unique>(); + + auto mock_client = std::make_unique>(); + mock_client_ptr_ = mock_client.get(); + + mock_stream_ptr_ = std::make_unique>(); + + EXPECT_CALL(*mock_client_factory, create()).WillOnce(Return(ByMove(std::move(mock_client)))); + EXPECT_CALL(*mock_client_ptr_, startRaw(_, _, _, _)).WillOnce(Return(mock_stream_ptr_.get())); + + auto& local_info = context_.server_factory_context_.local_info_; + + ON_CALL(local_info, clusterName()).WillByDefault(ReturnRef(test_string)); + ON_CALL(local_info, nodeName()).WillByDefault(ReturnRef(test_string)); + + envoy::config::trace::v3::ClientConfig proto_client_config; + TestUtility::loadFromYaml(yaml_string, proto_client_config); + client_config_ = std::make_unique(context_, proto_client_config); + + reporter_ = std::make_unique(std::move(mock_client_factory), + mock_dispatcher_, mock_random_generator_, + tracing_stats_, *client_config_); + } + +protected: + NiceMock context_; + + NiceMock& mock_dispatcher_ = context_.server_factory_context_.dispatcher_; + NiceMock& mock_random_generator_ = + context_.server_factory_context_.api_.random_; + Event::GlobalTimeSystem& mock_time_source_ = context_.server_factory_context_.time_system_; + + NiceMock& mock_scope_ = context_.server_factory_context_.scope_; + + NiceMock* mock_client_ptr_{nullptr}; + + std::unique_ptr> mock_stream_ptr_{nullptr}; + + NiceMock* timer_; + Event::TimerCb timer_cb_; + + std::string test_string = "ABCDEFGHIJKLMN"; + + SkyWalkingClientConfigPtr client_config_; + + SkyWalkingTracerStats tracing_stats_{ + SKYWALKING_TRACER_STATS(POOL_COUNTER_PREFIX(mock_scope_, "tracing.skywalking."))}; + TraceSegmentReporterPtr reporter_; +}; + +// Test whether the reporter can correctly add metadata according to the configuration. +TEST_F(TraceSegmentReporterTest, TraceSegmentReporterInitialMetadata) { + const std::string yaml_string = R"EOF( + backend_token: "FakeStringForAuthenticaion" + )EOF"; + + setupTraceSegmentReporter(yaml_string); + Http::TestRequestHeaderMapImpl metadata; + reporter_->onCreateInitialMetadata(metadata); + + EXPECT_EQ("FakeStringForAuthenticaion", metadata.get_("authentication")); +} + +TEST_F(TraceSegmentReporterTest, TraceSegmentReporterNoMetadata) { + setupTraceSegmentReporter("{}"); + Http::TestRequestHeaderMapImpl metadata; + reporter_->onCreateInitialMetadata(metadata); + + EXPECT_EQ("", metadata.get_("authentication")); +} + +TEST_F(TraceSegmentReporterTest, TraceSegmentReporterReportTraceSegment) { + setupTraceSegmentReporter("{}"); + ON_CALL(mock_random_generator_, random()).WillByDefault(Return(23333)); + + SegmentContextSharedPtr segment_context = + SkyWalkingTestHelper::createSegmentContext(true, "NEW", "PRE", mock_random_generator_); + SpanStore* parent_store = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), nullptr, "PARENT"); + // Parent span store has peer address. + parent_store->setPeerAddress("0.0.0.0"); + + SpanStore* first_child_sptore = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), parent_store, "CHILD"); + // Skip reporting the first child span. + first_child_sptore->setSampled(0); + + // Create second child span. + SkyWalkingTestHelper::createSpanStore(segment_context.get(), parent_store, "CHILD"); + + EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)); + + reporter_->report(*segment_context); + + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); + + // Create a segment context with no previous span context. + SegmentContextSharedPtr second_segment_context = SkyWalkingTestHelper::createSegmentContext( + true, "SECOND_SEGMENT", "", mock_random_generator_); + SkyWalkingTestHelper::createSpanStore(second_segment_context.get(), nullptr, "PARENT"); + + EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)); + reporter_->report(*second_segment_context); + + EXPECT_EQ(2U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); +} + +TEST_F(TraceSegmentReporterTest, TraceSegmentReporterReportWithDefaultCache) { + setupTraceSegmentReporter("{}"); + ON_CALL(mock_random_generator_, random()).WillByDefault(Return(23333)); + + SegmentContextSharedPtr segment_context = + SkyWalkingTestHelper::createSegmentContext(true, "NEW", "PRE", mock_random_generator_); + SpanStore* parent_store = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), nullptr, "PARENT"); + SkyWalkingTestHelper::createSpanStore(segment_context.get(), parent_store, "CHILD"); + + EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)).Times(1025); + + reporter_->report(*segment_context); + + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); + + // Simulates a disconnected connection. + EXPECT_CALL(*timer_, enableTimer(_, _)); + reporter_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Unknown, ""); + + // Try to report 10 segments. Due to the disconnection, the cache size is only 3. So 7 of the + // segments will be discarded. + for (int i = 0; i < 2048; i++) { + reporter_->report(*segment_context); + } + + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(1024U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); + + // Simulate the situation where the connection is re-established. The remaining segments in the + // cache will be reported. + EXPECT_CALL(*mock_client_ptr_, startRaw(_, _, _, _)).WillOnce(Return(mock_stream_ptr_.get())); + timer_cb_(); + + EXPECT_EQ(1025U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(1024U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(1024U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); +} + +TEST_F(TraceSegmentReporterTest, TraceSegmentReporterReportWithCacheConfig) { + const std::string yaml_string = R"EOF( + max_cache_size: 3 + )EOF"; + + setupTraceSegmentReporter(yaml_string); + + ON_CALL(mock_random_generator_, random()).WillByDefault(Return(23333)); + + SegmentContextSharedPtr segment_context = + SkyWalkingTestHelper::createSegmentContext(true, "NEW", "PRE", mock_random_generator_); + SpanStore* parent_store = + SkyWalkingTestHelper::createSpanStore(segment_context.get(), nullptr, "PARENT"); + SkyWalkingTestHelper::createSpanStore(segment_context.get(), parent_store, "CHILD"); + + EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)).Times(4); + + reporter_->report(*segment_context); + + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); + + // Simulates a disconnected connection. + EXPECT_CALL(*timer_, enableTimer(_, _)); + reporter_->onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Unknown, ""); + + // Try to report 10 segments. Due to the disconnection, the cache size is only 3. So 7 of the + // segments will be discarded. + for (int i = 0; i < 10; i++) { + reporter_->report(*segment_context); + } + + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(7U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); + + // Simulate the situation where the connection is re-established. The remaining segments in the + // cache will be reported. + EXPECT_CALL(*mock_client_ptr_, startRaw(_, _, _, _)).WillOnce(Return(mock_stream_ptr_.get())); + timer_cb_(); + + EXPECT_EQ(4U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(7U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(3U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); +} + +} // namespace +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/tracers/skywalking/tracer_test.cc b/test/extensions/tracers/skywalking/tracer_test.cc new file mode 100644 index 000000000000..c0d9356f1d50 --- /dev/null +++ b/test/extensions/tracers/skywalking/tracer_test.cc @@ -0,0 +1,193 @@ +#include "extensions/tracers/skywalking/skywalking_client_config.h" +#include "extensions/tracers/skywalking/tracer.h" + +#include "test/extensions/tracers/skywalking/skywalking_test_helper.h" +#include "test/mocks/common.h" +#include "test/mocks/server/tracer_factory_context.h" +#include "test/mocks/stats/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace Tracers { +namespace SkyWalking { +namespace { + +class TracerTest : public testing::Test { +public: + void setupTracer(const std::string& yaml_string) { + EXPECT_CALL(mock_dispatcher_, createTimer_(_)).WillOnce(Invoke([](Event::TimerCb) { + return new NiceMock(); + })); + + auto mock_client_factory = std::make_unique>(); + + auto mock_client = std::make_unique>(); + + mock_stream_ptr_ = std::make_unique>(); + + EXPECT_CALL(*mock_client, startRaw(_, _, _, _)).WillOnce(Return(mock_stream_ptr_.get())); + EXPECT_CALL(*mock_client_factory, create()).WillOnce(Return(ByMove(std::move(mock_client)))); + + auto& local_info = context_.server_factory_context_.local_info_; + + ON_CALL(local_info, clusterName()).WillByDefault(ReturnRef(test_string)); + ON_CALL(local_info, nodeName()).WillByDefault(ReturnRef(test_string)); + + envoy::config::trace::v3::ClientConfig proto_client_config; + TestUtility::loadFromYaml(yaml_string, proto_client_config); + client_config_ = std::make_unique(context_, proto_client_config); + + tracer_ = std::make_unique(mock_time_source_); + tracer_->setReporter(std::make_unique( + std::move(mock_client_factory), mock_dispatcher_, mock_random_generator_, tracing_stats_, + *client_config_)); + } + +protected: + NiceMock mock_tracing_config_; + + NiceMock context_; + + NiceMock& mock_dispatcher_ = context_.server_factory_context_.dispatcher_; + NiceMock& mock_random_generator_ = + context_.server_factory_context_.api_.random_; + Event::GlobalTimeSystem& mock_time_source_ = context_.server_factory_context_.time_system_; + + NiceMock& mock_scope_ = context_.server_factory_context_.scope_; + + std::unique_ptr> mock_stream_ptr_{nullptr}; + + std::string test_string = "ABCDEFGHIJKLMN"; + + SkyWalkingClientConfigPtr client_config_; + + SkyWalkingTracerStats tracing_stats_{ + SKYWALKING_TRACER_STATS(POOL_COUNTER_PREFIX(mock_scope_, "tracing.skywalking."))}; + + TracerPtr tracer_; +}; + +// Test that the basic functionality of Tracer is working, including creating Span, using Span to +// create new child Spans. +TEST_F(TracerTest, TracerTestCreateNewSpanWithNoPropagationHeaders) { + setupTracer("{}"); + EXPECT_CALL(mock_random_generator_, random()).WillRepeatedly(Return(666666)); + + // Create a new SegmentContext. + SegmentContextSharedPtr segment_context = + SkyWalkingTestHelper::createSegmentContext(true, "CURR", "", mock_random_generator_); + + Envoy::Tracing::SpanPtr org_span = tracer_->startSpan( + mock_tracing_config_, mock_time_source_.systemTime(), "TEST_OP", segment_context, nullptr); + Span* span = dynamic_cast(org_span.get()); + + EXPECT_EQ(true, span->spanStore()->isEntrySpan()); + + EXPECT_EQ("", span->getBaggage("FakeStringAndNothingToDo")); + span->setBaggage("FakeStringAndNothingToDo", "FakeStringAndNothingToDo"); + + // Test whether the basic functions of Span are normal. + + span->setSampled(false); + EXPECT_EQ(false, span->spanStore()->sampled()); + + // The initial operation name is consistent with the 'operation' parameter in the 'startSpan' + // method call. + EXPECT_EQ("TEST_OP", span->spanStore()->operation()); + span->setOperation("op"); + EXPECT_EQ("op", span->spanStore()->operation()); + + // Test whether the tag can be set correctly. + span->setTag("TestTagKeyA", "TestTagValueA"); + span->setTag("TestTagKeyB", "TestTagValueB"); + EXPECT_EQ("TestTagValueA", span->spanStore()->tags().at(0).second); + EXPECT_EQ("TestTagValueB", span->spanStore()->tags().at(1).second); + + // When setting the status code tag, the corresponding tag name will be rewritten as + // 'status_code'. + span->setTag(Tracing::Tags::get().HttpStatusCode, "200"); + EXPECT_EQ("status_code", span->spanStore()->tags().at(2).first); + EXPECT_EQ("200", span->spanStore()->tags().at(2).second); + + // When setting the error tag, the SpanStore object will also mark itself as an error. + span->setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); + EXPECT_EQ(Tracing::Tags::get().Error, span->spanStore()->tags().at(3).first); + EXPECT_EQ(Tracing::Tags::get().True, span->spanStore()->tags().at(3).second); + EXPECT_EQ(true, span->spanStore()->isError()); + + // When setting http url tag, the corresponding tag name will be rewritten as 'url'. + span->setTag(Tracing::Tags::get().HttpUrl, "http://test.com/test/path"); + EXPECT_EQ("url", span->spanStore()->tags().at(4).first); + + Envoy::Tracing::SpanPtr org_first_child_span = + span->spawnChild(mock_tracing_config_, "TestChild", mock_time_source_.systemTime()); + Span* first_child_span = dynamic_cast(org_first_child_span.get()); + + EXPECT_EQ(false, first_child_span->spanStore()->isEntrySpan()); + + EXPECT_EQ(0, first_child_span->spanStore()->sampled()); + EXPECT_EQ(1, first_child_span->spanStore()->spanId()); + EXPECT_EQ(0, first_child_span->spanStore()->parentSpanId()); + + EXPECT_EQ("TestChild", first_child_span->spanStore()->operation()); + + Http::TestRequestHeaderMapImpl first_child_headers{{":authority", "test.com"}}; + std::string expected_header_value = fmt::format( + "{}-{}-{}-{}-{}-{}-{}-{}", 0, + SkyWalkingTestHelper::base64Encode(SkyWalkingTestHelper::generateId(mock_random_generator_)), + SkyWalkingTestHelper::base64Encode(SkyWalkingTestHelper::generateId(mock_random_generator_)), + 1, SkyWalkingTestHelper::base64Encode("CURR#SERVICE"), + SkyWalkingTestHelper::base64Encode("CURR#INSTANCE"), SkyWalkingTestHelper::base64Encode("op"), + SkyWalkingTestHelper::base64Encode("test.com")); + + first_child_span->injectContext(first_child_headers); + EXPECT_EQ(expected_header_value, first_child_headers.get_("sw8")); + + // Reset sampling flag to true. + span->setSampled(true); + Envoy::Tracing::SpanPtr org_second_child_span = + span->spawnChild(mock_tracing_config_, "TestChild", mock_time_source_.systemTime()); + Span* second_child_span = dynamic_cast(org_second_child_span.get()); + + EXPECT_EQ(1, second_child_span->spanStore()->sampled()); + EXPECT_EQ(2, second_child_span->spanStore()->spanId()); + EXPECT_EQ(0, second_child_span->spanStore()->parentSpanId()); + + EXPECT_CALL(*mock_stream_ptr_, sendMessageRaw_(_, _)).Times(1); + + // When the child span ends, the data is not reported immediately, but the end time is set. + first_child_span->finishSpan(); + second_child_span->finishSpan(); + EXPECT_NE(0, first_child_span->spanStore()->endTime()); + EXPECT_NE(0, second_child_span->spanStore()->endTime()); + + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); + + // When the first span in the current segment ends, the entire segment is reported. + span->finishSpan(); + + EXPECT_EQ(1U, mock_scope_.counter("tracing.skywalking.segments_sent").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_dropped").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.cache_flushed").value()); + EXPECT_EQ(0U, mock_scope_.counter("tracing.skywalking.segments_flushed").value()); +} + +} // namespace +} // namespace SkyWalking +} // namespace Tracers +} // namespace Extensions +} // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 1a0529d16319..c8cc46423a2f 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -33,6 +33,7 @@ HEXDIG HEXDIGIT LTT OWS +SkyWalking TIDs ceil CHACHA