From 525b608484c81e16b626cae0804bea1abf30233a Mon Sep 17 00:00:00 2001 From: Matt Whelan Date: Mon, 10 Aug 2020 14:35:57 -0700 Subject: [PATCH 1/3] Modify Serverless to use a named pipe for telemetry when present (#201) * Update protobuf file to latest. f13a86b4b013d2ae07e78c7c27359d0a158be118 * Log gRPC version used. * Replace usage of deprecated grpc.WithDialer. (#942) * Prevent debug messages unless debug level set. (#943) * Include request_headers_map from connect reply in RecordSpan metadata. (#940) * Fix code strings to match spec. (#945) * Add parent.* intrinsics from transaction to root span. (#946) * Update span GUID propagation (#944) * Pull in new cross agent tests (not yet passing) * Code changes for span GUID creation * Get tests passing * t.Helper() was added in Go 1.9, so it can't be used here. :( * PR feedback * Update commit of cross-agent tests now that PR is merged * Never use the default transport. * When a timeout error occurs save instead of drop data. * Refactor trace_observer.go * Move defer statement to follow conventions * Refactor grpc.Dial logic. (#949) * Enable GitHub Actions for CI testing. * Add changelog entry. * Update version for v3.7.0 (#956) * Update version for v3.7.0 * Fix headings * Modify Serverless to use a named pipe for telemetry when present Serverless telemetry is normally sent to stdout. This is an alternate path designed for use with an externally managed named pipe at a fixed location. * Changelog entry * Formatting * Fix failing go1.9 tests. * Update CHANGELOG.md Co-authored-by: Rey Abolofia * Address PR feedback Co-authored-by: Rey Abolofia Co-authored-by: Elizabeth Heinlein Co-authored-by: Elizabeth Heinlein <52758187+newrelic-eheinlein@users.noreply.github.com> Co-authored-by: Rey Abolofia --- CHANGELOG.md | 5 +++ build-script.sh | 11 +++++ v3/integrations/nrlambda/handler.go | 36 +++++++++++++--- v3/integrations/nrlambda/handler_test.go | 54 ++++++++++++++++++++---- 4 files changed, 92 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72008d3ad..ccc11c2b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # ChangeLog +## Next + +### Changes +* When sending Serverless telemetry using the `nrlambda` integration, support an externally-managed named pipe. + ## 3.8.1 ### Bug Fixes diff --git a/build-script.sh b/build-script.sh index fd242f8a6..2f91ef34f 100755 --- a/build-script.sh +++ b/build-script.sh @@ -40,8 +40,19 @@ for dir in $DIRS; do VERSION=$(go version) V17="1.7" V18="1.8" + V19="1.9" if [[ "$VERSION" =~ .*"$V17".* || "$VERSION" =~ .*"$V18".* ]]; then echo "Not installing GRPC for old versions" + elif [[ "$VERSION" =~ .*"$V19" ]]; then + # install v3 dependencies that support this go version + set +e + go get -u google.golang.org/grpc # this go get will fail to build + set -e + cd $GOPATH/src/google.golang.org/grpc + git checkout v1.31.0 + cd - + + go get -u github.com/golang/protobuf/protoc-gen-go else go get -u github.com/golang/protobuf/protoc-gen-go go get -u google.golang.org/grpc diff --git a/v3/integrations/nrlambda/handler.go b/v3/integrations/nrlambda/handler.go index ef30eacbd..4f5cbbaf4 100644 --- a/v3/integrations/nrlambda/handler.go +++ b/v3/integrations/nrlambda/handler.go @@ -26,7 +26,7 @@ import ( "github.com/aws/aws-lambda-go/lambdacontext" "github.com/newrelic/go-agent/v3/internal" "github.com/newrelic/go-agent/v3/internal/integrationsupport" - newrelic "github.com/newrelic/go-agent/v3/newrelic" + "github.com/newrelic/go-agent/v3/newrelic" ) type response struct { @@ -67,6 +67,27 @@ func responseEvent(ctx context.Context, event interface{}) { } } +type writerProvider interface { + borrowWriter(needsWriter func(writer io.Writer)) +} + +type defaultWriterProvider struct { +} + +const telemetryNamedPipe = "/tmp/newrelic-telemetry" + +func (wp *defaultWriterProvider) borrowWriter(needsWriter func(io.Writer)) { + // If the telemetry named pipe exists and is writable, use it instead of stdout + pipeFile, err := os.OpenFile(telemetryNamedPipe, os.O_WRONLY, 0) + if err != nil { + needsWriter(os.Stdout) + return + } + //We need to close the pipe; of course we don't close stdout + defer pipeFile.Close() + needsWriter(pipeFile) +} + func (h *wrappedHandler) Invoke(ctx context.Context, payload []byte) ([]byte, error) { var arn, requestID string if lctx, ok := lambdacontext.FromContext(ctx); ok { @@ -74,7 +95,9 @@ func (h *wrappedHandler) Invoke(ctx context.Context, payload []byte) ([]byte, er requestID = lctx.AwsRequestID } - defer internal.ServerlessWrite(h.app.Private, arn, h.writer) + defer h.hasWriter.borrowWriter(func(writer io.Writer) { + internal.ServerlessWrite(h.app.Private, arn, writer) + }) txn := h.app.StartTransaction(h.functionName) defer txn.End() @@ -110,9 +133,10 @@ type wrappedHandler struct { // a time, we use a synchronization primitive to determine if this is // the first transaction for defensiveness in case of future changes. firstTransaction sync.Once - // writer is used to log the data JSON at the end of each transaction. - // This field exists (rather than hardcoded os.Stdout) for testing. - writer io.Writer + // hasWriter is used to log the data JSON at the end of each transaction. + // The writerProvider manages the lifecycle of the file handle being written + // to, similar to the Loan pattern. This field exists mostly for testing. + hasWriter writerProvider } // WrapHandler wraps the provided handler and returns a new handler with @@ -127,7 +151,7 @@ func WrapHandler(handler lambda.Handler, app *newrelic.Application) lambda.Handl original: handler, app: app, functionName: lambdacontext.FunctionName, - writer: os.Stdout, + hasWriter: &defaultWriterProvider{}, } } diff --git a/v3/integrations/nrlambda/handler_test.go b/v3/integrations/nrlambda/handler_test.go index 3dc705642..1b7afff12 100644 --- a/v3/integrations/nrlambda/handler_test.go +++ b/v3/integrations/nrlambda/handler_test.go @@ -8,7 +8,9 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" + "os" "strings" "testing" @@ -45,6 +47,15 @@ func distributedTracingEnabled(key string) string { } } +// bufWriterProvider is a testing implementation of writerProvider +type bufWriterProvider struct { + buf io.Writer +} + +func (bw bufWriterProvider) borrowWriter(needsWriter func(writer io.Writer)) { + needsWriter(bw.buf) +} + func TestColdStart(t *testing.T) { originalHandler := func(c context.Context) {} app := testApp(nil, t) @@ -52,7 +63,7 @@ func TestColdStart(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} ctx := context.Background() lctx := &lambdacontext.LambdaContext{ @@ -104,7 +115,7 @@ func TestColdStart(t *testing.T) { // Invoke the handler again to test the cold-start attribute absence. buf = &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} internal.HarvestTesting(app.Private, nil) resp, err = wrapped.Invoke(ctx, nil) if nil != err || string(resp) != "null" { @@ -154,7 +165,7 @@ func TestErrorCapture(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} resp, err := wrapped.Invoke(context.Background(), nil) if err != returnError || string(resp) != "" { @@ -229,7 +240,7 @@ func TestSetWebRequest(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} req := events.APIGatewayProxyRequest{ Headers: map[string]string{ @@ -301,7 +312,7 @@ func TestDistributedTracing(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} dtHdr := http.Header{} app.StartTransaction("hello").InsertDistributedTraceHeaders(dtHdr) @@ -396,7 +407,7 @@ func TestEventARN(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} req := events.DynamoDBEvent{ Records: []events.DynamoDBEventRecord{{ @@ -473,7 +484,7 @@ func TestAPIGatewayProxyResponse(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} resp, err := wrapped.Invoke(context.Background(), nil) if nil != err { @@ -535,7 +546,7 @@ func TestCustomEvent(t *testing.T) { w := wrapped.(*wrappedHandler) w.functionName = "functionName" buf := &bytes.Buffer{} - w.writer = buf + w.hasWriter = bufWriterProvider{buf} resp, err := wrapped.Invoke(context.Background(), nil) if nil != err || string(resp) != "null" { @@ -555,3 +566,30 @@ func TestCustomEvent(t *testing.T) { t.Error("no output written") } } + +func TestDefaultWriterProvider(t *testing.T) { + dwp := defaultWriterProvider{} + dwp.borrowWriter(func(writer io.Writer) { + if writer != os.Stdout { + t.Error("Expected stdout") + } + }) + + const telemetryFile = "/tmp/newrelic-telemetry" + defer os.Remove(telemetryFile) + file, err := os.Create(telemetryFile) + if err != nil { + t.Error("Unexpected error creating telemetry file", err) + } + + err = file.Close() + if err != nil { + t.Error("Error closing telemetry file", err) + } + + dwp.borrowWriter(func(writer io.Writer) { + if writer == os.Stdout { + t.Error("Expected telemetry file, got stdout") + } + }) +} From 8710b5775c789213911586d32f54978be454c1a1 Mon Sep 17 00:00:00 2001 From: eheinlein Date: Tue, 18 Aug 2020 16:09:03 -0700 Subject: [PATCH 2/3] Test on Go version 1.15 now that it has been released --- .github/workflows/ci.yaml | 66 ++++++++++++++++++++------------------- build-script.sh | 2 +- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6cdcdf94b..d73a5bbb6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,70 +103,72 @@ jobs: - go-version: 1.13.x dirs: v3/newrelic,v3/internal,v3/examples - go-version: 1.14.x + dirs: v3/newrelic,v3/internal,v3/examples + - go-version: 1.15.x dirs: v3/newrelic,v3/internal,v3/examples,v3/integrations/logcontext # v3 integrations - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/logcontext/nrlogrusplugin extratesting: go get -u github.com/sirupsen/logrus@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrawssdk-v1 extratesting: go get -u github.com/aws/aws-sdk-go@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrawssdk-v2 extratesting: go get -u github.com/aws/aws-sdk-go-v2@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrecho-v3 # Test against the latest v3 Echo: extratesting: go get -u github.com/labstack/echo@v3 - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrecho-v4 extratesting: go get -u github.com/labstack/echo/v4@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrelasticsearch-v7 extratesting: go get -u github.com/elastic/go-elasticsearch/v7@7.x - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrgin extratesting: go get -u github.com/gin-gonic/gin@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrgorilla extratesting: go get -u github.com/gorilla/mux@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrgraphgophers extratesting: go get -u github.com/graph-gophers/graphql-go@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrlogrus extratesting: go get -u github.com/sirupsen/logrus@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrlogxi extratesting: go get -u github.com/mgutz/logxi@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrpkgerrors extratesting: go get -u github.com/pkg/errors@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrlambda extratesting: go get -u github.com/aws/aws-lambda-go@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrmysql extratesting: go get -u github.com/go-sql-driver/mysql@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrpq extratesting: go get -u github.com/lib/pq@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrpq/example/sqlx - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrredis-v7 extratesting: go get -u github.com/go-redis/redis/v7@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrsqlite3 extratesting: go get -u github.com/mattn/go-sqlite3@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrsnowflake extratesting: go get -u github.com/snowflakedb/gosnowflake@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrgrpc extratesting: go get -u google.golang.org/grpc@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrmicro # As of Dec 2019, there is a race condition in when using go-micro@master # in their logging system. Instead, we'll test against the latest @@ -174,36 +176,36 @@ jobs: # As of Jan 2019, it is impossible to go get the latest micro version. # As of June 2020, confirmed errors still result # extratesting: go get -u github.com/micro/go-micro@latest - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrnats extratesting: go get -u github.com/nats-io/nats.go/@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrnats/test extratesting: go get -u github.com/nats-io/nats.go/@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrstan extratesting: go get -u github.com/nats-io/stan.go/@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrstan/test extratesting: go get -u github.com/nats-io/stan.go/@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrstan/examples extratesting: go get -u github.com/nats-io/stan.go/@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/logcontext extratesting: go get -u github.com/sirupsen/logrus@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrzap extratesting: go get -u go.uber.org/zap@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrhttprouter extratesting: go get -u github.com/julienschmidt/httprouter@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrb3 - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrmongo extratesting: go get -u go.mongodb.org/mongo-driver@master - - go-version: 1.14.x + - go-version: 1.15.x dirs: v3/integrations/nrgraphqlgo,v3/integrations/nrgraphqlgo/example extratesting: go get -u github.com/graphql-go/graphql@master diff --git a/build-script.sh b/build-script.sh index 2f91ef34f..22b6b0c07 100755 --- a/build-script.sh +++ b/build-script.sh @@ -4,7 +4,7 @@ set -x set -e -LATEST_VERSION="go1.14" +LATEST_VERSION="go1.15" # NOTE: Once we get rid of travis for good, this whole section can be removed # along with the .travis.yml file. From 61767628be93a09d57e49e22aee7361c0df19e8d Mon Sep 17 00:00:00 2001 From: eheinlein Date: Wed, 2 Sep 2020 11:50:01 -0700 Subject: [PATCH 3/3] Updating version # --- CHANGELOG.md | 2 +- v3/newrelic/version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc11c2b0..c732e93f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # ChangeLog -## Next +## 3.9.0 ### Changes * When sending Serverless telemetry using the `nrlambda` integration, support an externally-managed named pipe. diff --git a/v3/newrelic/version.go b/v3/newrelic/version.go index ead3e07f3..c8a9dc39f 100644 --- a/v3/newrelic/version.go +++ b/v3/newrelic/version.go @@ -11,7 +11,7 @@ import ( const ( // Version is the full string version of this Go Agent. - Version = "3.8.1" + Version = "3.9.0" ) var (