Fix offset tracker to build binaries for go stdlib as well (#256)

* Fix offset tracker to build binaries for go stdlib as well

* Update README.md

* Update make docker-offsets, changelog and lint

* Update offsets-tracker/README.md

Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com>

* Update changelog

* Update CHANGELOG.md

Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com>

* Fix changelog

---------

Co-authored-by: Eden Federman <eden@keyval.dev>
Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com>

CHANGELOG.md

@@ -8,9 +8,17 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http
 
 ## [Unreleased]
 
+### Changed
+
+- The function signature of `"go.opentelemetry.io/auto/offsets-tracker/downloader".DownloadBinary` has changed.
+  It now has an additional flag indicating whether it'll build a dummy app for Go stdlib packages or not. ([#256]https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/256)
+- The function signature of `"go.opentelemetry.io/auto/offsets-tracker/target".New` has changed.
+  It now accepts a flag to determine if the returned `Data` is from the Go stdlib or not. ([#256]https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/256)
+
 ### Fixed
 
 - Fix context propagation across different goroutines. ([#118](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/118))
+- The offset tracker can once again build binaries for the Go stdlib. ([#256]https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/256)
 
 ## [v0.2.2-alpha] - 2023-07-12
 

Makefile

@@ -87,7 +87,7 @@ offsets:
 
 .PHONY: docker-offsets
 docker-offsets:
-	docker run --rm -v $(shell pwd):/app golang:1.20 /bin/sh -c "cd ../app && make offsets"
+	docker run --rm -v $(shell pwd):/app golang:1.21 /bin/sh -c "cd ../app && make offsets"
 
 .PHONY: update-licenses
 update-licenses: generate $(GOLICENSES)

offsets-tracker/README.md

@@ -57,18 +57,24 @@ For example, here is the tracking of `method` field inside `transport.Stream` st
 
 ## Versions Discovery
 
-By default, offsets-tracker finds availble versions by executing `go list -versions <target-name>`.
+By default, offsets-tracker finds available versions by executing `go list -versions <target-name>`.
 
 Unfortunately, Go standard library versions are not discoverable via `go list`. 
-In order to discover Go versions, offsets-tracker can fetch the versions published at `https://go.dev/dl`.
+In order to discover Go versions, the offsets-tracker fetch the versions published at `https://go.dev/dl`.
 Fetching `go.dev` for discovering versions can be enabled by setting`.FindVersionsBy(target.GoDevFileVersionsStrategy)` when registering a new target.
 
 ## Download Strategy
 
 offsets-tracker wraps every Go module version as a Go application that depends on that module.
 Those applications are the result of [generating template files](./downloader/wrapper) with the appropriate version.
 
-In the case of the Go standard library, offsets-tracker downloads the published binary for the specified version. 
+In the case of the Go standard library, the offsets-tracker creates a dummy application that depends on the specific library. For example
+
+```go
+import (
+  _ "net/http"
+)
+```
 
 ## Version Constraints
 

offsets-tracker/downloader/go_wrapper.go

@@ -18,7 +18,7 @@ import (
 	_ "embed"
 	"fmt"
 	"io/fs"
-	"io/ioutil"
+	"os"
 	"path"
 
 	"go.opentelemetry.io/auto/offsets-tracker/utils"
@@ -30,25 +30,35 @@ var (
 	//go:embed wrapper/go.mod.txt
 	goMod string
 
+	//go:embed wrapper/go.modstd.txt
+	goModStdLib string
+
 	//go:embed wrapper/main.go.txt
 	goMain string
 )
 
 // DownloadBinary downloads the module with modName at version.
-func DownloadBinary(modName string, version string) (string, string, error) {
-	dir, err := ioutil.TempDir("", appName)
+// revive:disable-next-line:flag-parameter
+func DownloadBinary(modName string, version string, isGoStandartLib bool) (string, string, error) {
+	dir, err := os.MkdirTemp("", appName)
 	if err != nil {
 		return "", "", err
 	}
 
-	goModContent := fmt.Sprintf(goMod, modName, version)
-	err = ioutil.WriteFile(path.Join(dir, "go.mod"), []byte(goModContent), fs.ModePerm)
+	var goModContent string
+	if isGoStandartLib {
+		goModContent = fmt.Sprintf(goModStdLib, version)
+	} else {
+		goModContent = fmt.Sprintf(goMod, modName, version)
+	}
+
+	err = os.WriteFile(path.Join(dir, "go.mod"), []byte(goModContent), fs.ModePerm)
 	if err != nil {
 		return "", "", err
 	}
 
 	goMainContent := fmt.Sprintf(goMain, modName)
-	err = ioutil.WriteFile(path.Join(dir, "main.go"), []byte(goMainContent), fs.ModePerm)
+	err = os.WriteFile(path.Join(dir, "main.go"), []byte(goMainContent), fs.ModePerm)
 	if err != nil {
 		return "", "", err
 	}
@@ -58,7 +68,7 @@ func DownloadBinary(modName string, version string) (string, string, error) {
 		return "", "", err
 	}
 
-	_, _, err = utils.RunCommand("GOOS=linux GOARCH=amd64 go build", dir)
+	_, _, err = utils.RunCommand(fmt.Sprintf("GOOS=linux GOARCH=amd64 go build -o %s", appName), dir)
 	if err != nil {
 		return "", "", err
 	}

offsets-tracker/downloader/wrapper/go.modstd.txt

@@ -0,0 +1,3 @@
+module testapp
+
+go %s

offsets-tracker/main.go

@@ -44,15 +44,26 @@ func main() {
 		log.Fatalf("error in parsing version constraint: %v\n", err)
 	}
 
-	stdLibOffsets, err := target.New("go", *outputFile).
+	stdLibRuntimeOffsets, err := target.New("runtime", *outputFile, true).
 		FindVersionsBy(target.GoDevFileVersionsStrategy).
-		DownloadBinaryBy(target.DownloadPreCompiledBinaryFetchStrategy).
+		DownloadBinaryBy(target.WrapAsGoAppBinaryFetchStrategy).
 		VersionConstraint(&minimunGoVersion).
 		FindOffsets([]*binary.DataMember{
 			{
 				StructName: "runtime.g",
 				Field:      "goid",
 			},
+		})
+
+	if err != nil {
+		log.Fatalf("error while fetching offsets for \"runtime\": %v\n", err)
+	}
+
+	stdLibNetHTTPOffsets, err := target.New("net/http", *outputFile, true).
+		FindVersionsBy(target.GoDevFileVersionsStrategy).
+		DownloadBinaryBy(target.WrapAsGoAppBinaryFetchStrategy).
+		VersionConstraint(&minimunGoVersion).
+		FindOffsets([]*binary.DataMember{
 			{
 				StructName: "net/http.Request",
 				Field:      "Method",
@@ -73,17 +84,28 @@ func main() {
 				StructName: "net/http.Request",
 				Field:      "ctx",
 			},
+		})
+
+	if err != nil {
+		log.Fatalf("error while fetching offsets for \"net/http\": %v\n", err)
+	}
+
+	stdLibNetURLOffsets, err := target.New("net/url", *outputFile, true).
+		FindVersionsBy(target.GoDevFileVersionsStrategy).
+		DownloadBinaryBy(target.WrapAsGoAppBinaryFetchStrategy).
+		VersionConstraint(&minimunGoVersion).
+		FindOffsets([]*binary.DataMember{
 			{
 				StructName: "net/url.URL",
 				Field:      "Path",
 			},
 		})
 
 	if err != nil {
-		log.Fatalf("error while fetching offsets: %v\n", err)
+		log.Fatalf("error while fetching offsets for \"net/url\": %v\n", err)
 	}
 
-	grpcOffsets, err := target.New("google.golang.org/grpc", *outputFile).
+	grpcOffsets, err := target.New("google.golang.org/grpc", *outputFile, false).
 		FindOffsets([]*binary.DataMember{
 			{
 				StructName: "google.golang.org/grpc/internal/transport.Stream",
@@ -128,7 +150,7 @@ func main() {
 	}
 
 	fmt.Println("Done collecting offsets, writing results to file ...")
-	err = writer.WriteResults(*outputFile, stdLibOffsets, grpcOffsets)
+	err = writer.WriteResults(*outputFile, stdLibRuntimeOffsets, stdLibNetHTTPOffsets, stdLibNetURLOffsets, grpcOffsets)
 	if err != nil {
 		log.Fatalf("error while writing results to file: %v\n", err)
 	}

offsets-tracker/target/target.go

@@ -57,19 +57,21 @@ type VersionedResult struct {
 // Data represents the target Go module data.
 type Data struct {
 	name                string
+	isGoStdlib          bool
 	versionsStrategy    VersionsStrategy
 	binaryFetchStrategy BinaryFetchStrategy
 	versionConstraint   *version.Constraints
 	cache               *cache.Cache
 }
 
 // New returns a new Data.
-func New(name string, fileName string) *Data {
+func New(name string, fileName string, isStdlib bool) *Data {
 	return &Data{
 		name:                name,
 		versionsStrategy:    GoListVersionsStrategy,
 		binaryFetchStrategy: WrapAsGoAppBinaryFetchStrategy,
 		cache:               cache.NewCache(fileName),
+		isGoStdlib:          isStdlib,
 	}
 }
 
@@ -197,7 +199,7 @@ func (t *Data) findVersions() ([]string, error) {
 func (t *Data) downloadBinary(modName string, version string) (string, string, error) {
 	switch t.binaryFetchStrategy {
 	case WrapAsGoAppBinaryFetchStrategy:
-		return downloader.DownloadBinary(modName, version)
+		return downloader.DownloadBinary(modName, version, t.isGoStdlib)
 	case DownloadPreCompiledBinaryFetchStrategy:
 		return downloader.DownloadBinaryFromRemote(modName, version)
 	}

pkg/inject/offset_results.json

@@ -168,7 +168,7 @@
       "Header": {
         "versions": {
           "oldest": "1.12.0",
-          "newest": "1.20.7"
+          "newest": "1.21.0"
         },
         "offsets": [
           {
@@ -180,7 +180,7 @@
       "Method": {
         "versions": {
           "oldest": "1.12.0",
-          "newest": "1.20.7"
+          "newest": "1.21.0"
         },
         "offsets": [
           {
@@ -192,7 +192,7 @@
       "RemoteAddr": {
         "versions": {
           "oldest": "1.12.0",
-          "newest": "1.20.7"
+          "newest": "1.21.0"
         },
         "offsets": [
           {
@@ -204,7 +204,7 @@
       "URL": {
         "versions": {
           "oldest": "1.12.0",
-          "newest": "1.20.7"
+          "newest": "1.21.0"
         },
         "offsets": [
           {
@@ -216,7 +216,7 @@
       "ctx": {
         "versions": {
           "oldest": "1.12.0",
-          "newest": "1.20.7"
+          "newest": "1.21.0"
         },
         "offsets": [
           {
@@ -244,7 +244,7 @@
       "goid": {
         "versions": {
           "oldest": "1.12.0",
-          "newest": "1.20.7"
+          "newest": "1.21.0"
         },
         "offsets": [
           {