Fix issues related to Go 1.18 (ko-build#657)

* Fix issues related to Go 1.18

Update our internal fork for 1.18's ParseBuildInfo for use by pre-1.18
build versions to exactly the code used in the Go 1.18 release branch.
This affects users who `go install` ko running Go <1.18, since that code
was old and incompatible with 1.18-produced output of `go version -m`.

Add a workflow to test all combinations of pre- and post-1.18 setups for
both how ko was built, and what version of Go is installed by the user.

Update our release workflow to build using Go 1.18, so users who
download built binaries don't depend on our forked code at all.

* Only consider supported Go versions

* Massage output of go version -m so it can be parsed

* disable SBOM in unit test

* do the massaging inside internal/

* boilerplate come on

* proceed even when go mod version doesn't give us anything

* undo unit test change

.github/workflows/go-1.18.yaml

@@ -0,0 +1,33 @@
+name: Go 1.18 compat test
+
+on:
+  pull_request:
+    branches: ['main']
+
+jobs:
+  go118:
+    strategy:
+      fail-fast: false
+      matrix:
+        ko-go-version: ['1.17.x', '1.18.x']
+        user-go-version: ['1.17.x', '1.18.x']
+    name: Go 1.18 compat (ko=${{ matrix.ko-go-version }} / user=${{ matrix.user-go-version }})
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    # Build ko using ko-go-version
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.ko-go-version }}
+    - run: go install ./
+
+    # Run ko using user-go-version
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.user-go-version }}
+    - run: |
+        go install github.com/google/go-containerregistry/cmd/registry@latest
+        registry &
+        KO_DOCKER_REPO=localhost:1338 ko build ./test/

.github/workflows/modules-integration-test.yaml

@@ -9,7 +9,7 @@ jobs:
     name: Module Tests
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x, 1.18.x]
+        go-version: [1.17.x, 1.18.x]
     runs-on: 'ubuntu-latest'
     steps:
       - uses: actions/setup-go@v2

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
       - run: git fetch --prune --unshallow
       - uses: actions/setup-go@v1
         with:
-          go-version: 1.17.x
+          go-version: 1.18.x
       - uses: goreleaser/goreleaser-action@v2.9.1
         with:
           version: latest

internal/sbom/cyclonedx.go

@@ -39,8 +39,14 @@ func h1ToSHA256(s string) string {
 }
 
 func GenerateCycloneDX(mod []byte) ([]byte, error) {
-	bi := &BuildInfo{}
-	if err := bi.UnmarshalText(mod); err != nil {
+	var err error
+	mod, err = massageGoVersionM(mod)
+	if err != nil {
+		return nil, err
+	}
+
+	bi, err := ParseBuildInfo(string(mod))
+	if err != nil {
 		return nil, err
 	}
 

internal/sbom/mod.go

@@ -23,11 +23,13 @@
 package sbom
 
 import (
-	"bytes"
 	"fmt"
+	"strconv"
+	"strings"
 )
 
 // BuildInfo represents the build information read from a Go binary.
+// https://cs.opensource.google/go/go/+/release-branch.go1.18:src/runtime/debug/mod.go;drc=release-branch.go1.18;l=41
 type BuildInfo struct {
 	GoVersion string         // Version of Go that produced this binary.
 	Path      string         // The main package path
@@ -47,13 +49,32 @@ type Module struct {
 // BuildSetting describes a setting that may be used to understand how the
 // binary was built. For example, VCS commit and dirty status is stored here.
 type BuildSetting struct {
-	// Key and Value describe the build setting. They must not contain tabs
-	// or newlines.
+	// Key and Value describe the build setting.
+	// Key must not contain an equals sign, space, tab, or newline.
+	// Value must not contain newlines ('\n').
 	Key, Value string
 }
 
-func (bi *BuildInfo) UnmarshalText(data []byte) (err error) {
-	*bi = BuildInfo{}
+// https://cs.opensource.google/go/go/+/release-branch.go1.18:src/strings/strings.go;drc=release-branch.go1.18;l=1181
+func stringsCut(s, sep string) (before, after string, found bool) {
+	if i := strings.Index(s, sep); i >= 0 {
+		return s[:i], s[i+len(sep):], true
+	}
+	return s, "", false
+}
+
+// quoteKey reports whether key is required to be quoted.
+func quoteKey(key string) bool {
+	return len(key) == 0 || strings.ContainsAny(key, "= \t\r\n\"`")
+}
+
+// quoteValue reports whether value is required to be quoted.
+func quoteValue(value string) bool {
+	return strings.ContainsAny(value, " \t\r\n\"`")
+}
+
+// https://cs.opensource.google/go/go/+/release-branch.go1.18:src/runtime/debug/mod.go;drc=release-branch.go1.18;l=121
+func ParseBuildInfo(data string) (bi *BuildInfo, err error) {
 	lineNum := 1
 	defer func() {
 		if err != nil {
@@ -62,99 +83,133 @@ func (bi *BuildInfo) UnmarshalText(data []byte) (err error) {
 	}()
 
 	var (
-		pathLine  = []byte("path\t")
-		modLine   = []byte("mod\t")
-		depLine   = []byte("dep\t")
-		repLine   = []byte("=>\t")
-		buildLine = []byte("build\t")
-		newline   = []byte("\n")
-		tab       = []byte("\t")
+		pathLine  = "path\t"
+		modLine   = "mod\t"
+		depLine   = "dep\t"
+		repLine   = "=>\t"
+		buildLine = "build\t"
+		newline   = "\n"
+		tab       = "\t"
 	)
 
-	readModuleLine := func(elem [][]byte) (Module, error) {
+	readModuleLine := func(elem []string) (Module, error) {
 		if len(elem) != 2 && len(elem) != 3 {
 			return Module{}, fmt.Errorf("expected 2 or 3 columns; got %d", len(elem))
 		}
+		version := elem[1]
 		sum := ""
 		if len(elem) == 3 {
-			sum = string(elem[2])
+			sum = elem[2]
 		}
 		return Module{
-			Path:    string(elem[0]),
-			Version: string(elem[1]),
+			Path:    elem[0],
+			Version: version,
 			Sum:     sum,
 		}, nil
 	}
 
+	bi = new(BuildInfo)
 	var (
 		last *Module
-		line []byte
+		line string
 		ok   bool
 	)
 	// Reverse of BuildInfo.String(), except for go version.
 	for len(data) > 0 {
-		line, data, ok = bytesCut(data, newline)
+		line, data, ok = stringsCut(data, newline)
 		if !ok {
 			break
 		}
-		line = bytes.TrimPrefix(line, []byte("\t"))
 		switch {
-		case bytes.HasPrefix(line, pathLine):
+		case strings.HasPrefix(line, pathLine):
 			elem := line[len(pathLine):]
 			bi.Path = string(elem)
-		case bytes.HasPrefix(line, modLine):
-			elem := bytes.Split(line[len(modLine):], tab)
+		case strings.HasPrefix(line, modLine):
+			elem := strings.Split(line[len(modLine):], tab)
 			last = &bi.Main
 			*last, err = readModuleLine(elem)
 			if err != nil {
-				return err
+				return nil, err
 			}
-		case bytes.HasPrefix(line, depLine):
-			elem := bytes.Split(line[len(depLine):], tab)
+		case strings.HasPrefix(line, depLine):
+			elem := strings.Split(line[len(depLine):], tab)
 			last = new(Module)
 			bi.Deps = append(bi.Deps, last)
 			*last, err = readModuleLine(elem)
 			if err != nil {
-				return err
+				return nil, err
 			}
-		case bytes.HasPrefix(line, repLine):
-			elem := bytes.Split(line[len(repLine):], tab)
+		case strings.HasPrefix(line, repLine):
+			elem := strings.Split(line[len(repLine):], tab)
 			if len(elem) != 3 {
-				return fmt.Errorf("expected 3 columns for replacement; got %d", len(elem))
+				return nil, fmt.Errorf("expected 3 columns for replacement; got %d", len(elem))
 			}
 			if last == nil {
-				return fmt.Errorf("replacement with no module on previous line")
+				return nil, fmt.Errorf("replacement with no module on previous line")
 			}
 			last.Replace = &Module{
 				Path:    string(elem[0]),
 				Version: string(elem[1]),
 				Sum:     string(elem[2]),
 			}
 			last = nil
-		case bytes.HasPrefix(line, buildLine):
-			elem := bytes.Split(line[len(buildLine):], tab)
-			if len(elem) != 2 {
-				return fmt.Errorf("expected 2 columns for build setting; got %d", len(elem))
+		case strings.HasPrefix(line, buildLine):
+			kv := line[len(buildLine):]
+			if len(kv) < 1 {
+				return nil, fmt.Errorf("build line missing '='")
+			}
+
+			var key, rawValue string
+			switch kv[0] {
+			case '=':
+				return nil, fmt.Errorf("build line with missing key")
+
+			case '`', '"':
+				rawKey, err := strconv.QuotedPrefix(kv)
+				if err != nil {
+					return nil, fmt.Errorf("invalid quoted key in build line")
+				}
+				if len(kv) == len(rawKey) {
+					return nil, fmt.Errorf("build line missing '=' after quoted key")
+				}
+				if c := kv[len(rawKey)]; c != '=' {
+					return nil, fmt.Errorf("unexpected character after quoted key: %q", c)
+				}
+				key, _ = strconv.Unquote(rawKey)
+				rawValue = kv[len(rawKey)+1:]
+
+			default:
+				var ok bool
+				key, rawValue, ok = stringsCut(kv, "=")
+				if !ok {
+					return nil, fmt.Errorf("build line missing '=' after key")
+				}
+				if quoteKey(key) {
+					return nil, fmt.Errorf("unquoted key %q must be quoted", key)
+				}
 			}
-			if len(elem[0]) == 0 {
-				return fmt.Errorf("empty key")
+
+			var value string
+			if len(rawValue) > 0 {
+				switch rawValue[0] {
+				case '`', '"':
+					var err error
+					value, err = strconv.Unquote(rawValue)
+					if err != nil {
+						return nil, fmt.Errorf("invalid quoted value in build line")
+					}
+
+				default:
+					value = rawValue
+					if quoteValue(value) {
+						return nil, fmt.Errorf("unquoted value %q must be quoted", value)
+					}
+				}
 			}
-			bi.Settings = append(bi.Settings, BuildSetting{Key: string(elem[0]), Value: string(elem[1])})
+
+			bi.Settings = append(bi.Settings, BuildSetting{Key: key, Value: value})
 		}
 		lineNum++
 	}
-	return nil
-}
-
-// bytesCut slices s around the first instance of sep,
-// returning the text before and after sep.
-// The found result reports whether sep appears in s.
-// If sep does not appear in s, cut returns s, nil, false.
-//
-// bytesCut returns slices of the original slice s, not copies.
-func bytesCut(s, sep []byte) (before, after []byte, found bool) {
-	if i := bytes.Index(s, sep); i >= 0 {
-		return s[:i], s[i+len(sep):], true
-	}
-	return s, nil, false
+	return bi, nil
 }

internal/sbom/mod_1.18.go

@@ -24,11 +24,11 @@ import (
 
 type BuildInfo debug.BuildInfo
 
-func (bi *BuildInfo) UnmarshalText(data []byte) error {
+func ParseBuildInfo(data string) (*BuildInfo, error) {
 	dbi, err := debug.ParseBuildInfo(string(data))
 	if err != nil {
-		return fmt.Errorf("parsing build info: %w", err)
+		return nil, fmt.Errorf("parsing build info: %w", err)
 	}
-	*bi = BuildInfo(*dbi)
-	return nil
+	bi := BuildInfo(*dbi)
+	return &bi, nil
 }

internal/sbom/sbom.go

@@ -0,0 +1,51 @@
+// Copyright 2022 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sbom
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+// massageGoModVersion massages the output of `go version -m` into a form that
+// can be consumed by ParseBuildInfo.
+//
+// `go version -m` adds a line at the beginning of its output, and tabs at the
+// beginning of every line, that ParseBuildInfo doesn't like.
+func massageGoVersionM(b []byte) ([]byte, error) {
+	var out bytes.Buffer
+	scanner := bufio.NewScanner(bytes.NewReader(b))
+	if !scanner.Scan() {
+		// Input was malformed, and doesn't contain any newlines (it
+		// may even be empty). This seems to happen on Windows
+		// (https://github.com/google/ko/issues/535) and in unit tests.
+		// Just proceed with an empty output for now, and SBOMs will be empty.
+		// TODO: This should be an error.
+		return nil, nil
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("malformed input: %w", err)
+	}
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		fmt.Fprintln(&out, line)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}

internal/sbom/spdx.go

@@ -27,8 +27,14 @@ import (
 const dateFormat = "2006-01-02T15:04:05Z"
 
 func GenerateSPDX(koVersion string, date time.Time, mod []byte) ([]byte, error) {
-	bi := &BuildInfo{}
-	if err := bi.UnmarshalText(mod); err != nil {
+	var err error
+	mod, err = massageGoVersionM(mod)
+	if err != nil {
+		return nil, err
+	}
+
+	bi, err := ParseBuildInfo(string(mod))
+	if err != nil {
 		return nil, err
 	}