From 7a46b5428f239871993d66be2c7c667121f60a6f Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 10 Nov 2023 12:01:16 -0700 Subject: [PATCH] feat(internal/trace): add OpenTelemetry support (#8655) Add GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING env var flag for opt-in OpenTelemetry tracing. refs: #2205 --- go.mod | 7 +- go.sum | 18 ++- go.work.sum | 7 - internal/generated/snippets/go.sum | 8 + internal/testutil/trace_otel.go | 55 +++++++ internal/trace/trace.go | 146 +++++++++++++++-- internal/trace/trace_test.go | 249 +++++++++++++++++++++++++++++ 7 files changed, 466 insertions(+), 24 deletions(-) create mode 100644 internal/testutil/trace_otel.go diff --git a/go.mod b/go.mod index 67e5b6783101..dc275f672e48 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,10 @@ require ( github.com/google/martian/v3 v3.3.2 github.com/googleapis/gax-go/v2 v2.12.0 go.opencensus.io v0.24.0 + go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/trace v1.19.0 golang.org/x/oauth2 v0.13.0 - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 google.golang.org/api v0.149.0 google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b @@ -23,11 +25,14 @@ require ( cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.4.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.4.0 // indirect diff --git a/go.sum b/go.sum index 82388fa6e149..b937b0a266f2 100644 --- a/go.sum +++ b/go.sum @@ -13,12 +13,18 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -62,6 +68,7 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -71,8 +78,17 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= @@ -117,7 +133,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -158,6 +173,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/go.work.sum b/go.work.sum index c910f6e91137..fd1856f5b4b0 100644 --- a/go.work.sum +++ b/go.work.sum @@ -14,16 +14,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1: github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405/go.mod h1:oT32Z4o8Zv2xPQTg0pbVaPr0MPOH6f14RgXt7zfIpwg= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230629202037-9506855d4529/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230911183012-2d3300fd4832/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:qDbnxtViX5J6CvFbxeNUSzKgVlDLJ/6L+caxye9+Flo= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:itlFWGBbEyD32PUeJsTG8h8Wz7iJXfVK4gt1EJ+pAG0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= diff --git a/internal/generated/snippets/go.sum b/internal/generated/snippets/go.sum index bcb0dbccd598..f892abdce620 100644 --- a/internal/generated/snippets/go.sum +++ b/internal/generated/snippets/go.sum @@ -86,6 +86,9 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -235,6 +238,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -246,6 +250,10 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= diff --git a/internal/testutil/trace_otel.go b/internal/testutil/trace_otel.go new file mode 100644 index 000000000000..52fa50e80f72 --- /dev/null +++ b/internal/testutil/trace_otel.go @@ -0,0 +1,55 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "context" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +// OpenTelemetryTestExporter is a test utility exporter. It should be created +// with NewOpenTelemetryTestExporter. +type OpenTelemetryTestExporter struct { + exporter *tracetest.InMemoryExporter + tp *sdktrace.TracerProvider +} + +// NewOpenTelemetryTestExporter creates a OpenTelemetryTestExporter with +// underlying InMemoryExporter and TracerProvider from OpenTelemetry. +func NewOpenTelemetryTestExporter() *OpenTelemetryTestExporter { + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider( + sdktrace.WithSyncer(exporter), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + otel.SetTracerProvider(tp) + return &OpenTelemetryTestExporter{ + exporter: exporter, + tp: tp, + } +} + +// Spans returns the current in-memory stored spans. +func (te *OpenTelemetryTestExporter) Spans() tracetest.SpanStubs { + return te.exporter.GetSpans() +} + +// Unregister shuts down the underlying OpenTelemetry TracerProvider. +func (te *OpenTelemetryTestExporter) Unregister(ctx context.Context) { + te.tp.Shutdown(ctx) +} diff --git a/internal/trace/trace.go b/internal/trace/trace.go index c201d343e989..f6b88253b4a3 100644 --- a/internal/trace/trace.go +++ b/internal/trace/trace.go @@ -16,35 +16,94 @@ package trace import ( "context" + "errors" "fmt" + "os" + "strings" "go.opencensus.io/trace" - "golang.org/x/xerrors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + ottrace "go.opentelemetry.io/otel/trace" "google.golang.org/api/googleapi" "google.golang.org/genproto/googleapis/rpc/code" "google.golang.org/grpc/status" ) -// StartSpan adds a span to the trace with the given name. +const ( + telemetryPlatformTracingOpenCensus = "opencensus" + telemetryPlatformTracingOpenTelemetry = "opentelemetry" + telemetryPlatformTracingVar = "GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING" +) + +var ( + // TODO(chrisdsmith): Should the name of the OpenTelemetry tracer be public and mutable? + openTelemetryTracerName string = "cloud.google.com/go" + openTelemetryTracingEnabled bool = strings.EqualFold(strings.TrimSpace( + os.Getenv(telemetryPlatformTracingVar)), telemetryPlatformTracingOpenTelemetry) +) + +// IsOpenCensusTracingEnabled returns true if the environment variable +// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING is NOT set to the +// case-insensitive value "opentelemetry". +func IsOpenCensusTracingEnabled() bool { + return !IsOpenTelemetryTracingEnabled() +} + +// IsOpenTelemetryTracingEnabled returns true if the environment variable +// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING is set to the +// case-insensitive value "opentelemetry". +func IsOpenTelemetryTracingEnabled() bool { + return openTelemetryTracingEnabled +} + +// StartSpan adds a span to the trace with the given name. If IsOpenCensusTracingEnabled +// returns true, the span will be an OpenCensus span. If IsOpenTelemetryTracingEnabled +// returns true, the span will be an OpenTelemetry span. Set the environment variable +// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING to the case-insensitive +// value "opentelemetry" before loading the package to use OpenTelemetry tracing. +// The default will remain OpenCensus until [TBD], at which time the default will +// switch to "opentelemetry" and explicitly setting the environment variable to +// "opencensus" will be required to continue using OpenCensus tracing. func StartSpan(ctx context.Context, name string) context.Context { - ctx, _ = trace.StartSpan(ctx, name) + if IsOpenTelemetryTracingEnabled() { + ctx, _ = otel.GetTracerProvider().Tracer(openTelemetryTracerName).Start(ctx, name) + } else { + ctx, _ = trace.StartSpan(ctx, name) + } return ctx } -// EndSpan ends a span with the given error. +// EndSpan ends a span with the given error. If IsOpenCensusTracingEnabled +// returns true, the span will be an OpenCensus span. If IsOpenTelemetryTracingEnabled +// returns true, the span will be an OpenTelemetry span. Set the environment variable +// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING to the case-insensitive +// value "opentelemetry" before loading the package to use OpenTelemetry tracing. +// The default will remain OpenCensus until [TBD], at which time the default will +// switch to "opentelemetry" and explicitly setting the environment variable to +// "opencensus" will be required to continue using OpenCensus tracing. func EndSpan(ctx context.Context, err error) { - span := trace.FromContext(ctx) - if err != nil { - span.SetStatus(toStatus(err)) + if IsOpenTelemetryTracingEnabled() { + span := ottrace.SpanFromContext(ctx) + if err != nil { + span.SetStatus(codes.Error, toOpenTelemetryStatusDescription(err)) + span.RecordError(err) + } + span.End() + } else { + span := trace.FromContext(ctx) + if err != nil { + span.SetStatus(toStatus(err)) + } + span.End() } - span.End() } -// toStatus interrogates an error and converts it to an appropriate -// OpenCensus status. +// toStatus converts an error to an equivalent OpenCensus status. func toStatus(err error) trace.Status { var err2 *googleapi.Error - if ok := xerrors.As(err, &err2); ok { + if ok := errors.As(err, &err2); ok { return trace.Status{Code: httpStatusCodeToOCCode(err2.Code), Message: err2.Message} } else if s, ok := status.FromError(err); ok { return trace.Status{Code: int32(s.Code()), Message: s.Message()} @@ -53,6 +112,18 @@ func toStatus(err error) trace.Status { } } +// toOpenTelemetryStatus converts an error to an equivalent OpenTelemetry status description. +func toOpenTelemetryStatusDescription(err error) string { + var err2 *googleapi.Error + if ok := errors.As(err, &err2); ok { + return err2.Message + } else if s, ok := status.FromError(err); ok { + return s.Message() + } else { + return err.Error() + } +} + // TODO(deklerk): switch to using OpenCensus function when it becomes available. // Reference: https://github.com/googleapis/googleapis/blob/26b634d2724ac5dd30ae0b0cbfb01f07f2e4050e/google/rpc/code.proto func httpStatusCodeToOCCode(httpStatusCode int) int32 { @@ -86,10 +157,33 @@ func httpStatusCodeToOCCode(httpStatusCode int) int32 { } } -// TODO: (odeke-em): perhaps just pass around spans due to the cost -// incurred from using trace.FromContext(ctx) yet we could avoid -// throwing away the work done by ctx, span := trace.StartSpan. +// TracePrintf retrieves the current OpenCensus or OpenTelemetry span from context, then: +// * calls Span.Annotatef if OpenCensus is enabled; or +// * calls Span.AddEvent if OpenTelemetry is enabled. +// +// If IsOpenCensusTracingEnabled returns true, the expected span must be an +// OpenCensus span. If IsOpenTelemetryTracingEnabled returns true, the expected +// span must be an OpenTelemetry span. Set the environment variable +// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING to the case-insensitive +// value "opentelemetry" before loading the package to use OpenTelemetry tracing. +// The default will remain OpenCensus until [TBD], at which time the default will +// switch to "opentelemetry" and explicitly setting the environment variable to +// "opencensus" will be required to continue using OpenCensus tracing. func TracePrintf(ctx context.Context, attrMap map[string]interface{}, format string, args ...interface{}) { + if IsOpenTelemetryTracingEnabled() { + attrs := otAttrs(attrMap) + ottrace.SpanFromContext(ctx).AddEvent(fmt.Sprintf(format, args...), ottrace.WithAttributes(attrs...)) + } else { + attrs := ocAttrs(attrMap) + // TODO: (odeke-em): perhaps just pass around spans due to the cost + // incurred from using trace.FromContext(ctx) yet we could avoid + // throwing away the work done by ctx, span := trace.StartSpan. + trace.FromContext(ctx).Annotatef(attrs, format, args...) + } +} + +// ocAttrs converts a generic map to OpenCensus attributes. +func ocAttrs(attrMap map[string]interface{}) []trace.Attribute { var attrs []trace.Attribute for k, v := range attrMap { var a trace.Attribute @@ -107,5 +201,27 @@ func TracePrintf(ctx context.Context, attrMap map[string]interface{}, format str } attrs = append(attrs, a) } - trace.FromContext(ctx).Annotatef(attrs, format, args...) + return attrs +} + +// otAttrs converts a generic map to OpenTelemetry attributes. +func otAttrs(attrMap map[string]interface{}) []attribute.KeyValue { + var attrs []attribute.KeyValue + for k, v := range attrMap { + var a attribute.KeyValue + switch v := v.(type) { + case string: + a = attribute.Key(k).String(v) + case bool: + a = attribute.Key(k).Bool(v) + case int: + a = attribute.Key(k).Int(v) + case int64: + a = attribute.Key(k).Int64(v) + default: + a = attribute.Key(k).String(fmt.Sprintf("%#v", v)) + } + attrs = append(attrs, a) + } + return attrs } diff --git a/internal/trace/trace_test.go b/internal/trace/trace_test.go index 2af9fb8be781..9b5d70c83792 100644 --- a/internal/trace/trace_test.go +++ b/internal/trace/trace_test.go @@ -15,18 +15,151 @@ package trace import ( + "context" "errors" "net/http" + "sort" "testing" "cloud.google.com/go/internal/testutil" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/googleapis/gax-go/v2/apierror" octrace "go.opencensus.io/trace" + "go.opentelemetry.io/otel/attribute" + otcodes "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" "google.golang.org/api/googleapi" "google.golang.org/genproto/googleapis/rpc/code" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +var ( + ignoreEventFields = cmpopts.IgnoreFields(sdktrace.Event{}, "Time") + ignoreValueFields = cmpopts.IgnoreFields(attribute.Value{}, "vtype", "numeric", "stringly", "slice") +) + +func TestStartSpan_OpenCensus(t *testing.T) { + old := openTelemetryTracingEnabled + openTelemetryTracingEnabled = false + te := testutil.NewTestExporter() + t.Cleanup(func() { + openTelemetryTracingEnabled = old + te.Unregister() + }) + + ctx := context.Background() + ctx = StartSpan(ctx, "test-span") + + TracePrintf(ctx, annotationData(), "Add my annotations") + + err := &googleapi.Error{Code: http.StatusBadRequest, Message: "INVALID ARGUMENT"} + EndSpan(ctx, err) + + if !IsOpenCensusTracingEnabled() { + t.Errorf("got false, want true") + } + if IsOpenTelemetryTracingEnabled() { + t.Errorf("got true, want false") + } + spans := te.Spans + if len(spans) != 1 { + t.Fatalf("got %d, want 1", len(spans)) + } + if got, want := spans[0].Name, "test-span"; got != want { + t.Fatalf("got %s, want %s", got, want) + } + if want := int32(3); spans[0].Status.Code != want { + t.Errorf("got %v, want %v", spans[0].Status.Code, want) + } + if want := "INVALID ARGUMENT"; spans[0].Status.Message != want { + t.Errorf("got %v, want %v", spans[0].Status.Message, want) + } + if len(spans[0].Annotations) != 1 { + t.Fatalf("got %d, want 1", len(spans[0].Annotations)) + } + got := spans[0].Annotations[0].Attributes + want := make(map[string]interface{}) + want["my_bool"] = true + want["my_float"] = "0.9" + want["my_int"] = int64(123) + want["my_int64"] = int64(456) + want["my_string"] = "my string" + opt := cmpopts.SortMaps(func(a, b int) bool { + return a < b + }) + if !cmp.Equal(got, want, opt) { + t.Errorf("got(-), want(+),: \n%s", cmp.Diff(got, want, opt)) + } +} + +func TestStartSpan_OpenTelemetry(t *testing.T) { + old := openTelemetryTracingEnabled + openTelemetryTracingEnabled = true + ctx := context.Background() + te := testutil.NewOpenTelemetryTestExporter() + t.Cleanup(func() { + openTelemetryTracingEnabled = old + te.Unregister(ctx) + }) + + ctx = StartSpan(ctx, "test-span") + + TracePrintf(ctx, annotationData(), "Add my annotations") + + err := &googleapi.Error{Code: http.StatusBadRequest, Message: "INVALID ARGUMENT"} + EndSpan(ctx, err) + + if IsOpenCensusTracingEnabled() { + t.Errorf("got true, want false") + } + if !IsOpenTelemetryTracingEnabled() { + t.Errorf("got false, want true") + } + spans := te.Spans() + if len(spans) != 1 { + t.Fatalf("got %d, want 1", len(spans)) + } + if got, want := spans[0].Name, "test-span"; got != want { + t.Fatalf("got %s, want %s", got, want) + } + if want := otcodes.Error; spans[0].Status.Code != want { + t.Errorf("got %v, want %v", spans[0].Status.Code, want) + } + if want := "INVALID ARGUMENT"; spans[0].Status.Description != want { + t.Errorf("got %v, want %v", spans[0].Status.Description, want) + } + + want := []attribute.KeyValue{ + attribute.Key("my_bool").Bool(true), + attribute.Key("my_float").String("0.9"), + attribute.Key("my_int").Int(123), + attribute.Key("my_int64").Int64(int64(456)), + attribute.Key("my_string").String("my string"), + } + got := spans[0].Events[0].Attributes + // Sorting is required since the TracePrintf parameter is a map. + sort.Slice(got, func(i, j int) bool { + return got[i].Key < got[j].Key + }) + if !cmp.Equal(got, want, ignoreEventFields, ignoreValueFields) { + t.Errorf("got %v, want %v", got, want) + } + wantEvent := sdktrace.Event{ + Name: "exception", + Attributes: []attribute.KeyValue{ + // KeyValues are NOT sorted by key, but the sort is deterministic, + // since this Event was created by Span.RecordError. + attribute.Key("exception.type").String("*googleapi.Error"), + attribute.Key("exception.message").String("googleapi: Error 400: INVALID ARGUMENT"), + }, + } + if !cmp.Equal(spans[0].Events[1], wantEvent, ignoreEventFields, ignoreValueFields) { + t.Errorf("got %v, want %v", spans[0].Events[1], want) + } +} + func TestToStatus(t *testing.T) { for _, testcase := range []struct { input error @@ -51,3 +184,119 @@ func TestToStatus(t *testing.T) { } } } + +func TestToOpenTelemetryStatusDescription(t *testing.T) { + for _, testcase := range []struct { + input error + want string + }{ + { + errors.New("some random error"), + "some random error", + }, + { + &googleapi.Error{Code: http.StatusConflict, Message: "some specific googleapi http error"}, + "some specific googleapi http error", + }, + { + status.Error(codes.DataLoss, "some specific grpc error"), + "some specific grpc error", + }, + } { + // Wrap supported types in apierror.APIError as GAPIC clients + // do, but fall back to the unwrapped error if not supported. + // https://github.com/googleapis/gax-go/blob/v2.12.0/v2/invoke.go#L95 + var err error + err, ok := apierror.FromError(testcase.input) + if !ok { + err = testcase.input + } + + got := toOpenTelemetryStatusDescription(err) + if got != testcase.want { + t.Errorf("got %s, want %s", got, testcase.want) + } + } +} + +func TestToStatus_APIError(t *testing.T) { + for _, testcase := range []struct { + input error + want octrace.Status + }{ + { + // Apparently nonsensical error, but this is supported by the implementation. + &googleapi.Error{Code: 200, Message: "OK"}, + octrace.Status{Code: int32(code.Code_OK), Message: "OK"}, + }, + { + &googleapi.Error{Code: 499, Message: "error 499"}, + octrace.Status{Code: int32(code.Code_CANCELLED), Message: "error 499"}, + }, + { + &googleapi.Error{Code: http.StatusInternalServerError, Message: "error 500"}, + octrace.Status{Code: int32(code.Code_UNKNOWN), Message: "error 500"}, + }, + { + &googleapi.Error{Code: http.StatusBadRequest, Message: "error 400"}, + octrace.Status{Code: int32(code.Code_INVALID_ARGUMENT), Message: "error 400"}, + }, + { + &googleapi.Error{Code: http.StatusGatewayTimeout, Message: "error 504"}, + octrace.Status{Code: int32(code.Code_DEADLINE_EXCEEDED), Message: "error 504"}, + }, + { + &googleapi.Error{Code: http.StatusNotFound, Message: "error 404"}, + octrace.Status{Code: int32(code.Code_NOT_FOUND), Message: "error 404"}, + }, + { + &googleapi.Error{Code: http.StatusConflict, Message: "error 409"}, + octrace.Status{Code: int32(code.Code_ALREADY_EXISTS), Message: "error 409"}, + }, + { + &googleapi.Error{Code: http.StatusForbidden, Message: "error 403"}, + octrace.Status{Code: int32(code.Code_PERMISSION_DENIED), Message: "error 403"}, + }, + { + &googleapi.Error{Code: http.StatusUnauthorized, Message: "error 401"}, + octrace.Status{Code: int32(code.Code_UNAUTHENTICATED), Message: "error 401"}, + }, + { + &googleapi.Error{Code: http.StatusTooManyRequests, Message: "error 429"}, + octrace.Status{Code: int32(code.Code_RESOURCE_EXHAUSTED), Message: "error 429"}, + }, + { + &googleapi.Error{Code: http.StatusNotImplemented, Message: "error 501"}, + octrace.Status{Code: int32(code.Code_UNIMPLEMENTED), Message: "error 501"}, + }, + { + &googleapi.Error{Code: http.StatusServiceUnavailable, Message: "error 503"}, + octrace.Status{Code: int32(code.Code_UNAVAILABLE), Message: "error 503"}, + }, + { + &googleapi.Error{Code: http.StatusMovedPermanently, Message: "error 301"}, + octrace.Status{Code: int32(code.Code_UNKNOWN), Message: "error 301"}, + }, + } { + // Wrap googleapi.Error in apierror.APIError as GAPIC clients do. + // https://github.com/googleapis/gax-go/blob/v2.12.0/v2/invoke.go#L95 + err, ok := apierror.FromError(testcase.input) + if !ok { + t.Fatalf("apierror.FromError failed to parse %v", testcase.input) + } + got := toStatus(err) + if r := testutil.Diff(got, testcase.want); r != "" { + t.Errorf("got -, want +:\n%s", r) + } + } +} + +func annotationData() map[string]interface{} { + attrMap := make(map[string]interface{}) + attrMap["my_string"] = "my string" + attrMap["my_bool"] = true + attrMap["my_int"] = 123 + attrMap["my_int64"] = int64(456) + attrMap["my_float"] = 0.9 + return attrMap +}