Go 1.21 more fixes

.github/workflows/go.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Patch gc
       shell: sh
       run:
-        go run github.com/eh-steve/goloader/jit/patchgc@2aedbae4eb66fb4d0a4f24efbeb1873007792bca
+        go run -a github.com/eh-steve/goloader/jit/patchgc@2aedbae4eb66fb4d0a4f24efbeb1873007792bca
 
     - name: Checkout code
       uses: actions/checkout@v3
@@ -46,4 +46,4 @@ jobs:
     - name: Test
       shell: sh
       run:
-        cd $GOPATH/src/github.com/eh-steve/goloader/jit && export ${{ matrix.cgo-enabled }} ${{ matrix.dynlink }} && go test -c . && ./jit.test -test.v
+        cd $GOPATH/src/github.com/eh-steve/goloader/jit && export ${{ matrix.cgo-enabled }} ${{ matrix.dynlink }} && go test -a -c . && ./jit.test -test.v

convert.go

@@ -16,14 +16,14 @@ import (
 	"unsafe"
 )
 
-func CanAttemptConversion(oldValue, newValue interface{}) bool {
+func CanAttemptConversion(oldValue interface{}, newType reflect.Type) bool {
 	oldT := efaceOf(&oldValue)._type
-	newT := efaceOf(&newValue)._type
+	newT := fromRType(newType)
 	seen := map[_typePair]struct{}{}
 	return typesEqual(oldT, newT, seen)
 }
 
-func ConvertTypesAcrossModules(oldModule, newModule *CodeModule, oldValue, newValue interface{}) (res interface{}, err error) {
+func ConvertTypesAcrossModules(oldModule, newModule *CodeModule, oldValue interface{}, newType reflect.Type) (res interface{}, err error) {
 	defer func() {
 		if v := recover(); v != nil {
 			err = fmt.Errorf("unexpected panic (this is a bug): %v\n stack trace: %s", v, debug.Stack())
@@ -35,25 +35,24 @@ func ConvertTypesAcrossModules(oldModule, newModule *CodeModule, oldValue, newVa
 	// So we need to recurse over the entire structure, and find any itabs and replace them with the equivalent from the new module
 
 	oldT := efaceOf(&oldValue)._type
-	newT := efaceOf(&newValue)._type
+	newT := fromRType(newType)
 	seen := map[_typePair]struct{}{}
 	if !typesEqual(oldT, newT, seen) {
-		return nil, fmt.Errorf("old type %T and new type %T are not equal", oldValue, newValue)
+		return nil, fmt.Errorf("old type %T and new type %s are not equal", oldValue, newType)
 	}
 
 	// Need to take data in old value and copy into new value one field at a time, but check that
 	// the type is either shared (first module) or translated from the old to the new modules
-	newV := Indirect(ValueOf(&newValue)).Elem()
 	oldV := Indirect(ValueOf(&oldValue)).Elem()
 
 	cycleDetector := map[uintptr]*Value{}
 	typeHash := make(map[uint32][]*_type, len(newModule.module.typelinks))
 	buildModuleTypeHash(activeModules()[0], typeHash)
 	buildModuleTypeHash(newModule.module, typeHash)
 
-	cvt(oldModule, newModule, Value{oldV}, newV.Type(), nil, cycleDetector, typeHash)
+	cvt(oldModule, newModule, Value{oldV}, AsType(newT), nil, cycleDetector, typeHash)
 
-	return oldV.ConvertWithInterface(newV.Type()).Interface(), err
+	return oldV.ConvertWithInterface(AsType(newT)).Interface(), err
 }
 
 func toType(t Type) *_type {

init.1.13.go

@@ -30,7 +30,7 @@ type initTask struct {
 
 func (linker *Linker) doInitialize(codeModule *CodeModule, symbolMap map[string]uintptr) error {
 	for _, name := range linker.initFuncs {
-		if taskPtr, ok := symbolMap[name]; ok {
+		if taskPtr, ok := symbolMap[name]; ok && taskPtr != 0 { // taskPtr may be nil if the inittask wasn't seen in the host symtab (probably a no-op and therefore eliminated)
 			shouldSkipDedup := false
 			for _, pkgPath := range linker.options.SkipTypeDeduplicationForPackages {
 				if strings.HasPrefix(name, pkgPath) {

init.1.21.go

@@ -42,7 +42,7 @@ func (linker *Linker) doInitialize(codeModule *CodeModule, symbolMap map[string]
 		return slices.Index(autolibOrder, strings.TrimSuffix(linker.initFuncs[i], _InitTaskSuffix)) < slices.Index(autolibOrder, strings.TrimSuffix(linker.initFuncs[j], _InitTaskSuffix))
 	})
 	for _, name := range linker.initFuncs {
-		if taskPtr, ok := symbolMap[name]; ok {
+		if taskPtr, ok := symbolMap[name]; ok && taskPtr != 0 { // taskPtr may be nil if the inittask wasn't seen in the host symtab (probably a no-op and therefore eliminated)
 			shouldSkipDedup := false
 			for _, pkgPath := range linker.options.SkipTypeDeduplicationForPackages {
 				if strings.HasPrefix(name, pkgPath) {

itab.go

@@ -199,7 +199,7 @@ func (cm *CodeModule) patchTypeMethodOffsets(t *_type, u, prevU *uncommonType, p
 							for _, tl := range firstmoduledata.typelinks {
 								firstmoduledata.typemap[typeOff(tl)] = (*_type)(unsafe.Pointer(firstmoduledata.types + uintptr(tl)))
 							}
-							pinnedTypemaps = append(pinnedTypemaps, firstmoduledata.typemap)
+							*pinnedTypemapsTyped = append(*pinnedTypemapsTyped, firstmoduledata.typemap)
 						}
 						firstmoduledata.typemap[firstModuleTypemapCounter] = methodType
 						firstModuleTypemapEntries[methodType] = firstModuleTypemapCounter

jit/gc_patch.go

@@ -165,7 +165,7 @@ func PatchGC(goBinary string, debugLog bool) error {
 		err = os.WriteFile(flagPath, newFlagFile, flagFileStat.Mode())
 		if err != nil {
 			if strings.Contains(err.Error(), "permission denied") || strings.Contains(err.Error(), "not permitted") {
-				return fmt.Errorf("could not write patched '%s': %w\nTry changing $GOROOT's owner to current user, or run patch with sudo\ngo install github.com/eh-steve/goloader/jit/patchgc@latest && sudo $GOPATH/bin/patchgc", flagPath, err)
+				return fmt.Errorf("could not write patched '%s': %w\nTry changing $GOROOT's owner to current user with: \n\nsudo chown -R $USER:$USER $GOROOT\n\n or run patch with sudo:\ngo install github.com/eh-steve/goloader/jit/patchgc@latest && sudo $GOPATH/bin/patchgc", flagPath, err)
 			}
 			return fmt.Errorf("could not write patched '%s': %w", flagPath, err)
 		}

jit/go.mod

@@ -3,12 +3,12 @@ module github.com/eh-steve/goloader/jit
 go 1.18
 
 require (
-	github.com/eh-steve/goloader v0.0.0-20230730231803-5c95d7a5f4e2
-	github.com/eh-steve/goloader/jit/testdata v0.0.0-20230730231803-5c95d7a5f4e2
-	github.com/eh-steve/goloader/unload v0.0.0-20230730231803-5c95d7a5f4e2
+	github.com/eh-steve/goloader v0.0.0-20240102093554-682fa8e6d525
+	github.com/eh-steve/goloader/jit/testdata v0.0.0-20240102093554-682fa8e6d525
+	github.com/eh-steve/goloader/unload v0.0.0-20240102093554-682fa8e6d525
 )
 
 require github.com/opentracing/opentracing-go v1.2.0 // indirect
 
-replace github.com/eh-steve/goloader => ../
+//replace github.com/eh-steve/goloader => ../
 //replace github.com/eh-steve/goloader/jit/testdata => ./testdata

jit/go.sum

@@ -56,12 +56,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
-github.com/eh-steve/goloader v0.0.0-20230730231803-5c95d7a5f4e2 h1:0mZ2Y8x4dhPfm9eOlbzBx31YSLaECdPvKEBrL5Hc0YE=
-github.com/eh-steve/goloader v0.0.0-20230730231803-5c95d7a5f4e2/go.mod h1:k7xs3CUwCvOU9aw851I6AEb6ZzZJ3nos5dZ6A/2ewM0=
-github.com/eh-steve/goloader/jit/testdata v0.0.0-20230730231803-5c95d7a5f4e2 h1:pulprUBu3Rncam2xFKzaUTf8SlbaoOHtNocYBSbvprI=
-github.com/eh-steve/goloader/jit/testdata v0.0.0-20230730231803-5c95d7a5f4e2/go.mod h1:GyUY1mfY7NCdoVRo5H8+kmhLa/xE418ZcjfDfgwicXw=
-github.com/eh-steve/goloader/unload v0.0.0-20230730231803-5c95d7a5f4e2 h1:XBYe9gRPYpfxBRO9kBeep/FZrbeIuxPn5d6dun2oLQU=
-github.com/eh-steve/goloader/unload v0.0.0-20230730231803-5c95d7a5f4e2/go.mod h1:bA/aiceMvuZh45HAA/xzWJKT4E9EXqzmuPsB9yT/D/M=
+github.com/eh-steve/goloader v0.0.0-20240102093554-682fa8e6d525 h1:mLQlxgHqb8/JXICqYpEs84/Fd35Bn4+wCVqNCqVgKxg=
+github.com/eh-steve/goloader v0.0.0-20240102093554-682fa8e6d525/go.mod h1:k7xs3CUwCvOU9aw851I6AEb6ZzZJ3nos5dZ6A/2ewM0=
+github.com/eh-steve/goloader/jit/testdata v0.0.0-20240102093554-682fa8e6d525 h1:Bxd+hlL9gNnKxqvCxxXbKrQA93btAWQ0QUuWEO8NzQk=
+github.com/eh-steve/goloader/jit/testdata v0.0.0-20240102093554-682fa8e6d525/go.mod h1:GyUY1mfY7NCdoVRo5H8+kmhLa/xE418ZcjfDfgwicXw=
+github.com/eh-steve/goloader/unload v0.0.0-20240102093554-682fa8e6d525 h1:zWUW0O4I5OQju1znmqqvrh6GQu0kVN06iMMg/OtMI1o=
+github.com/eh-steve/goloader/unload v0.0.0-20240102093554-682fa8e6d525/go.mod h1:bA/aiceMvuZh45HAA/xzWJKT4E9EXqzmuPsB9yT/D/M=
 github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

jit/go_cmd.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -111,25 +110,37 @@ type PackageError struct {
 	Err         string   // the error itself
 }
 
-func GoModDownload(goCmd, workDir string, args ...string) error {
+func GoModDownload(goCmd, workDir string, verbose bool, args ...string) error {
 	dlCmd := exec.Command(goCmd, append([]string{"mod", "download"}, args...)...)
+	if verbose {
+		dlCmd.Stdout = os.Stdout
+		dlCmd.Stderr = os.Stderr
+	}
 	dlCmd.Dir = workDir
-	output, err := dlCmd.CombinedOutput()
+	err := dlCmd.Run()
 	if err != nil {
-		return fmt.Errorf("failed to go mod download %s: %s", args, output)
+		return fmt.Errorf("failed to go mod download %s", args)
 	}
 
 	tidyCmd := exec.Command(goCmd, "mod", "tidy")
-	output, err = tidyCmd.CombinedOutput()
+	if verbose {
+		tidyCmd.Stdout = os.Stdout
+		tidyCmd.Stderr = os.Stderr
+	}
+	err = tidyCmd.Run()
 	if err != nil {
-		return fmt.Errorf("failed to go mod tidy: %s", output)
+		return fmt.Errorf("failed to go mod tidy: %s", err)
 	}
 	return nil
 }
 
-func GoGet(goCmd, packagePath, workDir string) error {
+func GoGet(goCmd, packagePath, workDir string, verbose bool) error {
 	goGetCmd := exec.Command(goCmd, "get", packagePath)
 	goGetCmd.Dir = workDir
+	if verbose {
+		goGetCmd.Stderr = os.Stderr
+		goGetCmd.Stdout = os.Stdout
+	}
 	output, err := goGetCmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("failed to go get %s: %s", packagePath, output)
@@ -158,30 +169,24 @@ func GoListStd(goCmd string) map[string]struct{} {
 	return stdLibPkgs
 }
 
-func GoList(goCmd, absPath, workDir string) (*Package, error) {
+func GoList(goCmd, absPath, workDir string, verbose bool) (*Package, error) {
 	golistCmd := exec.Command(goCmd, "list", "-json", absPath)
 	golistCmd.Dir = workDir
-	output, err := golistCmd.StdoutPipe()
-	stdErrBuf := &bytes.Buffer{}
-	golistCmd.Stderr = io.MultiWriter(os.Stderr, stdErrBuf)
-	if err != nil {
-		panic(err)
-	}
-	listDec := json.NewDecoder(output)
-	err = golistCmd.Start()
-	if err != nil {
-		return nil, fmt.Errorf("failed to start 'go list -json %s': %w\nstderr:\n%s", absPath, err, stdErrBuf.String())
-	}
+
+	stdoutBuf, stdErrBuf := &bytes.Buffer{}, &bytes.Buffer{}
+
+	golistCmd.Stdout = stdoutBuf
+	golistCmd.Stderr = stdErrBuf
 	pkg := Package{}
-	err = listDec.Decode(&pkg)
+
+	err := golistCmd.Run()
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode response of 'go list -json %s': %w\nstderr:\n%s", absPath, err, stdErrBuf.String())
+		return nil, fmt.Errorf("failed to run 'go list -json %s': %w\nstderr:\n%s", absPath, err, stdErrBuf.String())
 	}
-	err = golistCmd.Wait()
+	err = json.Unmarshal(stdoutBuf.Bytes(), &pkg)
 	if err != nil {
-		return nil, fmt.Errorf("failed to wait for 'go list -json %s': %w\nstderr:\n%s", absPath, err, stdErrBuf.String())
+		return nil, fmt.Errorf("failed to decode response of 'go list -json %s': %w\nstderr:\n%s", absPath, err, stdErrBuf.String())
 	}
-
 	if len(pkg.GoFiles)+len(pkg.CgoFiles) == 0 {
 		return nil, fmt.Errorf("no Go files found in directory %s", absPath)
 	}

jit/jit.go

@@ -488,21 +488,21 @@ func BuildGoFiles(config BuildConfig, pathToGoFile string, extraFiles ...string)
 	if err != nil {
 		return nil, fmt.Errorf("failed to patch gc: %w", err)
 	}
-	pkg, err := GoList(config.GoBinary, absPath, workDir)
+	pkg, err := GoList(config.GoBinary, absPath, workDir, config.DebugLog)
 	if err != nil {
 		return nil, err
 	}
 
 	if len(pkg.DepsErrors) > 0 {
-		err = GoModDownload(config.GoBinary, workDir)
+		err = GoModDownload(config.GoBinary, workDir, config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
-		err = GoGet(config.GoBinary, workDir, workDir)
+		err = GoGet(config.GoBinary, workDir, workDir, config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
-		pkg, err = GoList(config.GoBinary, absPath, "")
+		pkg, err = GoList(config.GoBinary, absPath, "", config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
@@ -596,13 +596,13 @@ func BuildGoText(config BuildConfig, goText string) (*LoadableUnit, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to patch gc: %w", err)
 	}
-	pkg, err := GoList(config.GoBinary, tmpFilePath, "")
+	pkg, err := GoList(config.GoBinary, tmpFilePath, "", config.DebugLog)
 	if err != nil {
 		return nil, err
 	}
 
 	if len(pkg.DepsErrors) > 0 {
-		err = GoModDownload(config.GoBinary, buildDir)
+		err = GoModDownload(config.GoBinary, buildDir, config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
@@ -611,11 +611,11 @@ func BuildGoText(config BuildConfig, goText string) (*LoadableUnit, error) {
 			return nil, fmt.Errorf("could not get absolute path of directory containing file %s: %w", tmpFilePath, err)
 		}
 
-		err = GoGet(config.GoBinary, absPackagePath, "")
+		err = GoGet(config.GoBinary, absPackagePath, "", config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
-		pkg, err = GoList(config.GoBinary, tmpFilePath, "")
+		pkg, err = GoList(config.GoBinary, tmpFilePath, "", config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
@@ -666,7 +666,7 @@ func BuildGoPackage(config BuildConfig, pathToGoPackage string) (*LoadableUnit,
 		return nil, fmt.Errorf("failed to patch gc: %w", err)
 	}
 	// Execute list from within the package folder so that go list resolves the module correctly from that path
-	pkg, err := GoList(config.GoBinary, absPath, absPath)
+	pkg, err := GoList(config.GoBinary, absPath, absPath, config.DebugLog)
 	if err != nil {
 		return nil, err
 	}
@@ -676,15 +676,15 @@ func BuildGoPackage(config BuildConfig, pathToGoPackage string) (*LoadableUnit,
 	}
 
 	if len(pkg.DepsErrors) > 0 {
-		err = GoModDownload(config.GoBinary, absPath)
+		err = GoModDownload(config.GoBinary, absPath, config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
-		err = GoGet(config.GoBinary, absPath, absPath)
+		err = GoGet(config.GoBinary, absPath, absPath, config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
-		pkg, err = GoList(config.GoBinary, absPath, "")
+		pkg, err = GoList(config.GoBinary, absPath, "", config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
@@ -774,12 +774,12 @@ func BuildGoPackageRemote(config BuildConfig, goPackage string, version string)
 		versionSuffix = "@" + version
 	}
 
-	err = GoGet(config.GoBinary, goPackage+versionSuffix, workDir)
+	err = GoGet(config.GoBinary, goPackage+versionSuffix, workDir, config.DebugLog)
 	if err != nil {
 		return nil, err
 	}
 
-	pkg, err := GoList(config.GoBinary, goPackage, workDir)
+	pkg, err := GoList(config.GoBinary, goPackage, workDir, config.DebugLog)
 	if err != nil {
 		return nil, err
 	}
@@ -789,15 +789,15 @@ func BuildGoPackageRemote(config BuildConfig, goPackage string, version string)
 	}
 
 	if len(pkg.DepsErrors) > 0 {
-		err = GoModDownload(config.GoBinary, workDir, pkg.Module.Path)
+		err = GoModDownload(config.GoBinary, workDir, config.DebugLog, pkg.Module.Path)
 		if err != nil {
 			return nil, err
 		}
-		err = GoGet(config.GoBinary, goPackage, workDir)
+		err = GoGet(config.GoBinary, goPackage, workDir, config.DebugLog)
 		if err != nil {
 			return nil, err
 		}
-		pkg, err = GoList(config.GoBinary, goPackage, "")
+		pkg, err = GoList(config.GoBinary, goPackage, "", config.DebugLog)
 		if err != nil {
 			return nil, err
 		}

jit/jit_test.go

@@ -1053,7 +1053,7 @@ func TestConvertOldAndNewTypes(t *testing.T) {
 				t.Fatalf("expected current to be the same as input: %d  %d", current, input)
 			}
 
-			newThing2, err := goloader.ConvertTypesAcrossModules(module1, module2, thing1, thing2)
+			newThing2, err := goloader.ConvertTypesAcrossModules(module1, module2, thing1, reflect.TypeOf(thing2))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1072,7 +1072,7 @@ func TestConvertOldAndNewTypes(t *testing.T) {
 			ifaceCurrentComplex12 := ifaceOut12.Val2["complex"].(map[interface{}]interface{})
 			_ = thingIface1.Method2(nil)
 
-			newThingIface2, err := goloader.ConvertTypesAcrossModules(module1, module2, thingIface1, thingIface2)
+			newThingIface2, err := goloader.ConvertTypesAcrossModules(module1, module2, thingIface1, reflect.TypeOf(thingIface2))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1192,7 +1192,7 @@ func TestStatefulHttpServer(t *testing.T) {
 			makeHandler2 := symbols2["MakeServer"].(func() http.Handler)
 			handler2 := makeHandler2()
 
-			newHandler, err := goloader.ConvertTypesAcrossModules(module, module2, handler, handler2)
+			newHandler, err := goloader.ConvertTypesAcrossModules(module, module2, handler, reflect.TypeOf(handler2))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1287,7 +1287,7 @@ func TestCloneConnection(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			newDialer2, err := goloader.ConvertTypesAcrossModules(module1, module2, dialer1, dialer2)
+			newDialer2, err := goloader.ConvertTypesAcrossModules(module1, module2, dialer1, reflect.TypeOf(dialer2))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1478,6 +1478,7 @@ func TestK8s(t *testing.T) {
 	}
 
 	conf := baseConfig
+	conf.DebugLog = true
 	conf.UnsafeBlindlyUseFirstmoduleTypes = true
 	data := testData{
 		files: []string{"./testdata/test_k8s/test.go"},