diff --git a/cmd/layotto/main.go b/cmd/layotto/main.go index 5d148bc473..68cf14cbf5 100644 --- a/cmd/layotto/main.go +++ b/cmd/layotto/main.go @@ -32,6 +32,7 @@ import ( "mosn.io/layotto/diagnostics" "mosn.io/layotto/pkg/grpc/default_api" secretstores_loader "mosn.io/layotto/pkg/runtime/secretstores" + "mosn.io/mosn/pkg/trace/skywalking" "os" "strconv" "time" @@ -162,6 +163,8 @@ import ( _ "mosn.io/pkg/buffer" _ "mosn.io/layotto/diagnostics/exporter_iml" + lprotocol "mosn.io/layotto/diagnostics/protocol" + lsky "mosn.io/layotto/diagnostics/skywalking" ) // loggerForDaprComp is constructed for reusing dapr's components. @@ -498,7 +501,8 @@ func ExtensionsRegister(c *cli.Context) { // 4. register tracer xtrace.RegisterDelegate(bolt.ProtocolName, tracebolt.Boltv1Delegate) trace.RegisterTracerBuilder("SOFATracer", protocol.HTTP1, tracehttp.NewTracer) - trace.RegisterTracerBuilder("SOFATracer", "layotto", diagnostics.NewTracer) + trace.RegisterTracerBuilder("SOFATracer", lprotocol.Layotto, diagnostics.NewTracer) + trace.RegisterTracerBuilder(skywalking.SkyDriverName, lprotocol.Layotto, lsky.NewGrpcSkyTracer) } diff --git a/cmd/layotto_multiple_api/main.go b/cmd/layotto_multiple_api/main.go index c5d5656e72..b2b4d4d408 100644 --- a/cmd/layotto_multiple_api/main.go +++ b/cmd/layotto_multiple_api/main.go @@ -24,6 +24,8 @@ import ( l8_grpc "mosn.io/layotto/pkg/grpc" "mosn.io/layotto/pkg/grpc/dapr" "mosn.io/layotto/pkg/grpc/default_api" + _ "mosn.io/mosn/pkg/filter/stream/grpcmetric" + "mosn.io/mosn/pkg/trace/skywalking" "os" "strconv" "time" @@ -152,6 +154,8 @@ import ( _ "mosn.io/pkg/buffer" _ "mosn.io/layotto/diagnostics/exporter_iml" + lprotocol "mosn.io/layotto/diagnostics/protocol" + lsky "mosn.io/layotto/diagnostics/skywalking" ) // loggerForDaprComp is constructed for reusing dapr's components. @@ -464,6 +468,7 @@ func ExtensionsRegister(c *cli.Context) { xtrace.RegisterDelegate(bolt.ProtocolName, tracebolt.Boltv1Delegate) trace.RegisterTracerBuilder("SOFATracer", protocol.HTTP1, tracehttp.NewTracer) trace.RegisterTracerBuilder("SOFATracer", "layotto", diagnostics.NewTracer) + trace.RegisterTracerBuilder(skywalking.SkyDriverName, lprotocol.Layotto, lsky.NewGrpcSkyTracer) } diff --git a/codecov.yml b/codecov.yml index bbb533543d..0547c6c5c6 100644 --- a/codecov.yml +++ b/codecov.yml @@ -27,4 +27,5 @@ ignore: - "cmd/**/*" - "components/lock/consul/consul_lock_task.go" - "components/rpc/invoker/mosn/channel/xchannel.go" + - "diagnostics/**/*" - "components/pkg/mock/**/*" diff --git a/components/trace/span.go b/components/trace/span.go index afc745d6df..47171cb7dc 100644 --- a/components/trace/span.go +++ b/components/trace/span.go @@ -51,6 +51,7 @@ func (span *Span) SpanId() string { func (span *Span) SetParentSpanId(id string) { span.parentSpanId = id } + func (span *Span) ParentSpanId() string { return span.parentSpanId } diff --git a/configs/config_trace_skywalking.json b/configs/config_trace_skywalking.json new file mode 100644 index 0000000000..458d185f8f --- /dev/null +++ b/configs/config_trace_skywalking.json @@ -0,0 +1,81 @@ +{ + "servers": [ + { + "default_log_path": "stdout", + "default_log_level": "INFO", + "listeners": [ + { + "name": "grpc", + "address": "0.0.0.0:34904", + "bind_port": true, + "filter_chains": [ + { + "filters": [ + { + "type": "grpc", + "config": { + "server_name": "runtime", + "grpc_config": { + "hellos": { + "helloworld": { + "hello": "greeting" + } + }, + "config_store": { + "etcd": { + "address": [ + "127.0.0.1:2379" + ], + "timeout": "10" + } + } + } + } + } + ] + } + ], + "stream_filters": [ + { + "type": "flowControlFilter", + "config": { + "global_switch": true, + "limit_key_type": "PATH", + "rules": [ + { + "resource": "/spec.proto.runtime.v1.Runtime/SayHello", + "grade": 1, + "threshold": 5 + } + ] + } + }, + { + "type": "grpc_metric" + } + ] + } + ] + } + ], + "tracing": { + "enable": true, + "driver": "SkyWalking", + "config": { + "reporter": "gRPC", + "backend_service": "127.0.0.1:11800", + "service_name": "layotto" + } + }, + "metrics": { + "sinks": [ + { + "type": "prometheus", + "config": { + "port": 34903 + } + } + ] + } +} + diff --git a/diagnostics/grpc/grpc_request.go b/diagnostics/grpc/grpc_request.go new file mode 100644 index 0000000000..4520c6b186 --- /dev/null +++ b/diagnostics/grpc/grpc_request.go @@ -0,0 +1,5 @@ +package grpc + +type RequestInfo struct { + FullMethod string +} diff --git a/diagnostics/grpc_tracing.go b/diagnostics/grpc_tracing.go index cf1812eb3e..19670e5cd1 100644 --- a/diagnostics/grpc_tracing.go +++ b/diagnostics/grpc_tracing.go @@ -8,6 +8,8 @@ import ( "google.golang.org/grpc" ltrace "mosn.io/layotto/components/trace" + lgrpc "mosn.io/layotto/diagnostics/grpc" + "mosn.io/layotto/diagnostics/protocol" "mosn.io/mosn/pkg/trace" ) @@ -18,14 +20,19 @@ func UnaryInterceptorFilter(ctx context.Context, req interface{}, info *grpc.Una return resp, err } // get tracer - tracer := trace.Tracer("layotto") + tracer := trace.Tracer(protocol.Layotto) // start a span - span := tracer.Start(ctx, req, time.Now()) + span := tracer.Start(ctx, &lgrpc.RequestInfo{ + FullMethod: info.FullMethod, + }, time.Now()) defer span.FinishSpan() + span.SetTag(ltrace.LAYOTTO_METHOD_NAME, info.FullMethod) span.SetTag(ltrace.LAYOTTO_REQUEST_RESULT, "0") + // construct a new context which contains the span ctx = GetNewContext(ctx, span) + // handle request resp, err = handler(ctx, req) if err != nil { @@ -40,13 +47,17 @@ func StreamInterceptorFilter(srv interface{}, ss grpc.ServerStream, info *grpc.S return err } // get tracer - tracer := trace.Tracer("layotto") + tracer := trace.Tracer(protocol.Layotto) ctx := ss.Context() // start a span - span := tracer.Start(ctx, nil, time.Now()) + span := tracer.Start(ctx, &lgrpc.RequestInfo{ + FullMethod: info.FullMethod, + }, time.Now()) defer span.FinishSpan() + span.SetTag(ltrace.LAYOTTO_METHOD_NAME, info.FullMethod) span.SetTag(ltrace.LAYOTTO_REQUEST_RESULT, "0") + // construct a new context which contains the span wrapped := grpc_middleware.WrapServerStream(ss) ctx = GetNewContext(ctx, span) @@ -56,5 +67,6 @@ func StreamInterceptorFilter(srv interface{}, ss grpc.ServerStream, info *grpc.S if err != nil { span.SetTag(ltrace.LAYOTTO_REQUEST_RESULT, "1") } + return err } diff --git a/diagnostics/protocol/protocol.go b/diagnostics/protocol/protocol.go new file mode 100644 index 0000000000..febc98659f --- /dev/null +++ b/diagnostics/protocol/protocol.go @@ -0,0 +1,5 @@ +package protocol + +const ( + Layotto = "layotto" +) diff --git a/diagnostics/skywalking/grpc_tracer.go b/diagnostics/skywalking/grpc_tracer.go new file mode 100644 index 0000000000..1b710f0f94 --- /dev/null +++ b/diagnostics/skywalking/grpc_tracer.go @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Layotto Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package skywalking + +import ( + "context" + "mosn.io/layotto/diagnostics/grpc" + "time" + + "github.com/SkyAPM/go2sky" + language_agent "github.com/SkyAPM/go2sky/reporter/grpc/language-agent" + "mosn.io/api" + ltrace "mosn.io/layotto/components/trace" + "mosn.io/layotto/diagnostics/protocol" + "mosn.io/mosn/pkg/log" + "mosn.io/mosn/pkg/trace" + "mosn.io/mosn/pkg/trace/skywalking" + "mosn.io/mosn/pkg/types" +) + +func init() { + trace.RegisterTracerBuilder(skywalking.SkyDriverName, protocol.Layotto, NewGrpcSkyTracer) +} + +func NewGrpcSkyTracer(_ map[string]interface{}) (api.Tracer, error) { + return &grpcSkyTracer{}, nil +} + +type grpcSkyTracer struct { + *go2sky.Tracer +} + +func (tracer *grpcSkyTracer) SetGO2SkyTracer(t *go2sky.Tracer) { + tracer.Tracer = t +} + +func (tracer *grpcSkyTracer) Start(ctx context.Context, request interface{}, _ time.Time) api.Span { + info, ok := request.(*grpc.RequestInfo) + if !ok { + log.DefaultLogger.Debugf("[SkyWalking] [tracer] [layotto] unable to get request header, downstream trace ignored") + return skywalking.NoopSpan + } + + // create entry span (downstream) + entry, nCtx, err := tracer.CreateEntrySpan(ctx, info.FullMethod, func() (sw8 string, err error) { + return + }) + + if err != nil { + log.DefaultLogger.Errorf("[SkyWalking] [tracer] [http1] create entry span error, err: %v", err) + return skywalking.NoopSpan + } + entry.Tag(go2sky.TagHTTPMethod, "POST") + entry.Tag(go2sky.TagURL, info.FullMethod) + entry.SetComponent(skywalking.MOSNComponentID) + entry.SetSpanLayer(language_agent.SpanLayer_Http) + + return &grpcSkySpan{ + tracer: tracer, + ctx: nCtx, + carrier: &skywalking.SpanCarrier{ + EntrySpan: entry, + }, + Span: <race.Span{}, + } +} + +type grpcSkySpan struct { + *ltrace.Span + tracer *grpcSkyTracer + ctx context.Context + carrier *skywalking.SpanCarrier +} + +func (h *grpcSkySpan) TraceId() string { + return go2sky.TraceID(h.ctx) +} + +func (h *grpcSkySpan) InjectContext(requestHeaders types.HeaderMap, requestInfo api.RequestInfo) { +} + +func (h *grpcSkySpan) SetRequestInfo(requestInfo api.RequestInfo) { +} + +func (h *grpcSkySpan) FinishSpan() { + entry := h.carrier.EntrySpan + if h.Tag(ltrace.LAYOTTO_REQUEST_RESULT) == "1" { + entry.Error(time.Now(), skywalking.ErrorLog) + entry.Tag(go2sky.TagStatusCode, "500") + } else { + entry.Tag(go2sky.TagStatusCode, "200") + } + + entry.End() +} diff --git a/diagnostics/skywalking/skywalking-docker-compose.yaml b/diagnostics/skywalking/skywalking-docker-compose.yaml new file mode 100644 index 0000000000..ad656f4b94 --- /dev/null +++ b/diagnostics/skywalking/skywalking-docker-compose.yaml @@ -0,0 +1,38 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more +#contributor license agreements. See the NOTICE file distributed with +#this work for additional information regarding copyright ownership. +#The ASF licenses this file to You under the Apache License, Version 2.0 +#(the "License"); you may not use this file except in compliance with +#the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +version: '3.3' +services: + skywalking-oap: + image: apache/skywalking-oap-server:8.0.1-es7 + container_name: skywalking-oap + restart: always + ports: + - 11800:11800 + - 12800:12800 + environment: + SW_STORAGE: h2 + skywalking-ui: + image: apache/skywalking-ui:8.0.1 + container_name: skywalking-ui + depends_on: + - skywalking-oap + links: + - skywalking-oap + restart: always + ports: + - 8080:8080 + environment: + SW_OAP_ADDRESS: skywalking-oap:12800 diff --git a/diagnostics/tracing.go b/diagnostics/tracing.go index b91bdf1f03..e1864bef98 100644 --- a/diagnostics/tracing.go +++ b/diagnostics/tracing.go @@ -45,6 +45,7 @@ func getActiveExportersFromConfig(config map[string]interface{}) []string { func (tracer *grpcTracer) Start(ctx context.Context, request interface{}, startTime time.Time) api.Span { span := NewSpan(ctx, startTime, tracer.config) + return span } diff --git a/docs/img/trace/sky.png b/docs/img/trace/sky.png new file mode 100644 index 0000000000..21383b53b4 Binary files /dev/null and b/docs/img/trace/sky.png differ diff --git a/docs/zh/README.md b/docs/zh/README.md index b491066c21..02e5d84ece 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -71,6 +71,14 @@ Layotto 提供了多种语言版本的 SDK，SDK 通过 gRPC 与 Layotto 进行 | -------- | :----: | :--------------------------------------------------------------: | --------------------------- | | API 插件 | ✅ | [demo](https://mosn.io/layotto/#/zh/start/api_plugin/helloworld) | 为 Layotto 添加您自己的 API | +### 可观测性 + + +| feature | status | quick start | desc | +|------------| :----: |:-----------------------------------------------------------:|-----------------------| +| Skywalking | ✅ | [demo](https://mosn.io/layotto/#/zh/start/trace/skywalking) | Layotto 接入 Skywalking | + + ### Actuator | feature | status | quick start | desc | diff --git a/docs/zh/_sidebar.md b/docs/zh/_sidebar.md index 8ab4a7c06b..1020d7eed0 100644 --- a/docs/zh/_sidebar.md +++ b/docs/zh/_sidebar.md @@ -20,7 +20,9 @@ - 在七层网络进行流量干预 - [方法级别限流](zh/start/stream_filter/flow_control.md) - [健康检查、查询运行时元数据](zh/start/actuator/start.md) - - [可观测性(trace,metric)](zh/start/trace/trace.md) + - 可观测性 + - [Trace, Metrics](zh/start/trace/trace.md) + - [Trace 接入 Skywalking](zh/start/trace/skywalking.md) - [将业务逻辑通过 WASM 下沉进sidecar](zh/start/wasm/start.md) - [基于 WASM 跟 Runtime 实现的 Faas 模型](zh/start/faas/start.md) - [用户手册](zh/building_blocks/) diff --git a/docs/zh/start/trace/skywalking.md b/docs/zh/start/trace/skywalking.md new file mode 100644 index 0000000000..53597580ef --- /dev/null +++ b/docs/zh/start/trace/skywalking.md @@ -0,0 +1,60 @@ +# Skywalking trace 接入 + +## 配置 + +示例：configs/config_trace_skywalking.json + +````json +{ + "tracing": { + "enable": true, + "driver": "SkyWalking", + "config": { + "reporter": "gRPC", + "backend_service": "127.0.0.1:11800", + "service_name": "layotto" + } + } +} +```` + +| 字段 | 必填 | 说明 | +| --- |-----|--------------------------| +| reporter | Y | 上报方式 grpc | +| backend_service | Y | skywalking oap server 地址 | +| service_name | Y | 服务名称 | + +## 运行 skywalking + +````shell +cd diagnostics/skywalking + +docker-compose -f skywalking-docker-compose.yaml up -d +```` + +## 运行 layotto + +````shell +cd cmd/layotto_multiple_api/ +go build -o layotto +./layotto start -c ../../configs/config_trace_skywalking.json +```` + +## 运行 Demo + +````shell +cd demo/flowControl +go run client.go +```` + +访问 http://127.0.0.1:8080 + +![](../../../img/trace/sky.png) + +## 清理资源 + +````shell +cd diagnostics/skywalking + +docker-compose -f skywalking-docker-compose.yaml down +```` \ No newline at end of file diff --git a/go.mod b/go.mod index 2a0523afc6..74fb3d825e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/SkyAPM/go2sky v0.5.0 github.com/agrea/ptr v0.0.0-20180711073057-77a518d99b7b github.com/alicebob/miniredis/v2 v2.16.0 github.com/dapr/components-contrib v1.5.1-rc.1 diff --git a/go.sum b/go.sum index 397b13129b..67ea42e0da 100644 --- a/go.sum +++ b/go.sum @@ -141,6 +141,7 @@ github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/SkyAPM/go2sky v0.5.0 h1:9RDBQviaeazG7PJMLLnMcU4U++PORbqEls4ix4OEgQw= github.com/SkyAPM/go2sky v0.5.0/go.mod h1:TANzYw5EvIlTidGWvQxtvO87rM6C746HkM0xkWqnPQw= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY=