[TUF autoupdater] Include arch in release file path and download file path

pkg/autoupdate/tuf/autoupdate.go

@@ -328,7 +328,7 @@ func (ta *TufAutoupdater) downloadUpdate(binary autoupdatableBinary, targets dat
 func findRelease(binary autoupdatableBinary, targets data.TargetFiles, channel string) (string, data.TargetFileMeta, error) {
 	// First, find the target that the channel release file is pointing to
 	var releaseTarget string
-	targetReleaseFile := path.Join(string(binary), runtime.GOOS, channel, "release.json")
+	targetReleaseFile := path.Join(string(binary), runtime.GOOS, PlatformArch(), channel, "release.json")
 	for targetName, target := range targets {
 		if targetName != targetReleaseFile {
 			continue
@@ -361,6 +361,16 @@ func findRelease(binary autoupdatableBinary, targets data.TargetFiles, channel s
 	return "", data.TargetFileMeta{}, fmt.Errorf("could not find metadata for release target %s for binary %s", releaseTarget, binary)
 }
 
+// PlatformArch returns the correct arch for the runtime OS. For now, since osquery doesn't publish an arm64 release,
+// we use the universal binaries for darwin.
+func PlatformArch() string {
+	if runtime.GOOS == "darwin" {
+		return "universal"
+	}
+
+	return runtime.GOARCH
+}
+
 // storeError saves errors that occur during the periodic check for updates, so that they
 // can be queryable via the `kolide_tuf_autoupdater_errors` table.
 func (ta *TufAutoupdater) storeError(autoupdateErr error) {

pkg/autoupdate/tuf/autoupdate_test.go

@@ -97,9 +97,9 @@ func TestExecute(t *testing.T) {
 	// Get metadata for each release
 	_, err = autoupdater.metadataClient.Update()
 	require.NoError(t, err, "could not update metadata client to fetch target metadata")
-	osquerydMetadata, err := autoupdater.metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryOsqueryd, runtime.GOOS, binaryOsqueryd, testReleaseVersion))
+	osquerydMetadata, err := autoupdater.metadataClient.Target(fmt.Sprintf("%s/%s/%s/%s-%s.tar.gz", binaryOsqueryd, runtime.GOOS, PlatformArch(), binaryOsqueryd, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for osqueryd")
-	launcherMetadata, err := autoupdater.metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, binaryLauncher, testReleaseVersion))
+	launcherMetadata, err := autoupdater.metadataClient.Target(fmt.Sprintf("%s/%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, PlatformArch(), binaryLauncher, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for launcher")
 
 	// Expect that we attempt to tidy the library first before running execute loop

pkg/autoupdate/tuf/ci/tuf_server.go

@@ -41,6 +41,11 @@ func InitRemoteTufServer(t *testing.T, testReleaseVersion string) (tufServerURL
 	// Sign the root metadata file
 	require.NoError(t, repo.Sign("root.json"), "could not sign root metadata file")
 
+	arch := runtime.GOARCH
+	if runtime.GOOS == "darwin" {
+		arch = "universal"
+	}
+
 	// Create test binaries and release files per binary and per release channel
 	for _, b := range []string{"osqueryd", "launcher"} {
 		for _, v := range []string{"0.1.1", "0.12.3-deadbeef", testReleaseVersion} {
@@ -52,7 +57,7 @@ func InitRemoteTufServer(t *testing.T, testReleaseVersion string) (tufServerURL
 			// and evaluated.
 			if v == testReleaseVersion {
 				// Create test binary and copy it to the staged targets directory
-				stagedTargetsDir := filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS)
+				stagedTargetsDir := filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, arch)
 				executablePath := executableLocation(stagedTargetsDir, b)
 				require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0777), "could not make staging directory")
 				CopyBinary(t, executablePath)
@@ -62,13 +67,13 @@ func InitRemoteTufServer(t *testing.T, testReleaseVersion string) (tufServerURL
 				compress(t, binaryFileName, stagedTargetsDir, stagedTargetsDir, b)
 			} else {
 				// Create and commit a test binary
-				require.NoError(t, os.MkdirAll(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS), 0777), "could not make staging directory")
-				err = os.WriteFile(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, binaryFileName), []byte("I am a test target"), 0777)
+				require.NoError(t, os.MkdirAll(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, arch), 0777), "could not make staging directory")
+				err = os.WriteFile(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, arch, binaryFileName), []byte("I am a test target"), 0777)
 				require.NoError(t, err, "could not write test target binary to temp dir")
 			}
 
 			// Add the target
-			require.NoError(t, repo.AddTarget(fmt.Sprintf("%s/%s/%s", b, runtime.GOOS, binaryFileName), nil), "could not add test target binary to tuf")
+			require.NoError(t, repo.AddTarget(fmt.Sprintf("%s/%s/%s/%s", b, runtime.GOOS, arch, binaryFileName), nil), "could not add test target binary to tuf")
 
 			// Commit
 			require.NoError(t, repo.Snapshot(), "could not take snapshot")
@@ -81,11 +86,11 @@ func InitRemoteTufServer(t *testing.T, testReleaseVersion string) (tufServerURL
 
 			// If this is our release version, also create and commit a test release file
 			for _, c := range []string{"stable", "beta", "nightly"} {
-				require.NoError(t, os.MkdirAll(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, c), 0777), "could not make staging directory")
-				err = os.WriteFile(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, c, "release.json"), []byte("{}"), 0777)
+				require.NoError(t, os.MkdirAll(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, arch, c), 0777), "could not make staging directory")
+				err = os.WriteFile(filepath.Join(tufDir, "staged", "targets", b, runtime.GOOS, arch, c, "release.json"), []byte("{}"), 0777)
 				require.NoError(t, err, "could not write test target release file to temp dir")
-				customMetadata := fmt.Sprintf("{\"target\":\"%s/%s/%s\"}", b, runtime.GOOS, binaryFileName)
-				require.NoError(t, repo.AddTarget(fmt.Sprintf("%s/%s/%s/release.json", b, runtime.GOOS, c), []byte(customMetadata)), "could not add test target release file to tuf")
+				customMetadata := fmt.Sprintf("{\"target\":\"%s/%s/%s/%s\"}", b, runtime.GOOS, arch, binaryFileName)
+				require.NoError(t, repo.AddTarget(fmt.Sprintf("%s/%s/%s/%s/release.json", b, runtime.GOOS, arch, c), []byte(customMetadata)), "could not add test target release file to tuf")
 
 				// Commit
 				require.NoError(t, repo.Snapshot(), "could not take snapshot")
@@ -106,10 +111,10 @@ func InitRemoteTufServer(t *testing.T, testReleaseVersion string) (tufServerURL
 	require.FileExists(t, filepath.Join(tufDir, "repository", "snapshot.json"))
 	require.FileExists(t, filepath.Join(tufDir, "repository", "timestamp.json"))
 	require.FileExists(t, filepath.Join(tufDir, "repository", "targets.json"))
-	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "launcher", runtime.GOOS, "stable", "release.json"))
-	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "launcher", runtime.GOOS, fmt.Sprintf("launcher-%s.tar.gz", testReleaseVersion)))
-	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "osqueryd", runtime.GOOS, "stable", "release.json"))
-	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "osqueryd", runtime.GOOS, fmt.Sprintf("osqueryd-%s.tar.gz", testReleaseVersion)))
+	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "launcher", runtime.GOOS, arch, "stable", "release.json"))
+	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "launcher", runtime.GOOS, arch, fmt.Sprintf("launcher-%s.tar.gz", testReleaseVersion)))
+	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "osqueryd", runtime.GOOS, arch, "stable", "release.json"))
+	require.FileExists(t, filepath.Join(tufDir, "repository", "targets", "osqueryd", runtime.GOOS, arch, fmt.Sprintf("osqueryd-%s.tar.gz", testReleaseVersion)))
 
 	// Set up a test server to serve these files
 	testMetadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

pkg/autoupdate/tuf/library_manager.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -121,7 +122,8 @@ func (ulm *updateLibraryManager) stageAndVerifyUpdate(binary autoupdatableBinary
 	stagedUpdatePath := filepath.Join(ulm.stagingDir, targetFilename)
 
 	// Request download from mirror
-	resp, err := ulm.mirrorClient.Get(ulm.mirrorUrl + fmt.Sprintf("/kolide/%s/%s/%s", binary, runtime.GOOS, targetFilename))
+	downloadPath := path.Join("/", "kolide", string(binary), runtime.GOOS, PlatformArch(), targetFilename)
+	resp, err := ulm.mirrorClient.Get(ulm.mirrorUrl + downloadPath)
 	if err != nil {
 		return stagedUpdatePath, fmt.Errorf("could not make request to download target %s: %w", targetFilename, err)
 	}

pkg/autoupdate/tuf/library_manager_test.go

@@ -102,9 +102,9 @@ func TestAddToLibrary(t *testing.T) {
 	require.NoError(t, err, "could not update metadata client")
 
 	// Get the target metadata
-	launcherTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, binaryLauncher, testReleaseVersion))
+	launcherTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, PlatformArch(), binaryLauncher, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for launcher target")
-	osquerydTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryOsqueryd, runtime.GOOS, binaryOsqueryd, testReleaseVersion))
+	osquerydTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s/%s-%s.tar.gz", binaryOsqueryd, runtime.GOOS, PlatformArch(), binaryOsqueryd, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for launcher target")
 
 	testCases := []struct {
@@ -263,9 +263,9 @@ func TestAddToLibrary_verifyStagedUpdate_handlesInvalidFiles(t *testing.T) {
 	require.NoError(t, err, "could not update metadata client")
 
 	// Get the target metadata
-	launcherTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, binaryLauncher, testReleaseVersion))
+	launcherTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s/%s-%s.tar.gz", binaryLauncher, runtime.GOOS, PlatformArch(), binaryLauncher, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for launcher target")
-	osquerydTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s-%s.tar.gz", binaryOsqueryd, runtime.GOOS, binaryOsqueryd, testReleaseVersion))
+	osquerydTargetMeta, err := metadataClient.Target(fmt.Sprintf("%s/%s/%s/%s-%s.tar.gz", binaryOsqueryd, runtime.GOOS, PlatformArch(), binaryOsqueryd, testReleaseVersion))
 	require.NoError(t, err, "could not get test metadata for launcher target")
 
 	testCases := []struct {

pkg/osquery/tables/tufinfo/release_version.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -21,6 +22,7 @@ func TufReleaseVersionTable(flags types.Flags) *table.Plugin {
 	columns := []table.ColumnDefinition{
 		table.TextColumn("binary"),
 		table.TextColumn("operating_system"),
+		table.TextColumn("architecture"),
 		table.TextColumn("channel"),
 		table.TextColumn("target"),
 	}
@@ -57,7 +59,7 @@ func generateTufReleaseVersionTable(flags types.Flags) table.GenerateFunc {
 				}
 
 				parts := strings.Split(targetFileName, "/")
-				if len(parts) != 4 {
+				if len(parts) != 5 {
 					// Shouldn't happen given the check above, but just in case
 					continue
 				}
@@ -70,7 +72,8 @@ func generateTufReleaseVersionTable(flags types.Flags) table.GenerateFunc {
 				results = append(results, map[string]string{
 					"binary":           binary,
 					"operating_system": parts[1],
-					"channel":          parts[2],
+					"architecture":     parts[2],
+					"channel":          parts[3],
 					"target":           metadata.Target,
 				})
 			}
@@ -83,9 +86,15 @@ func generateTufReleaseVersionTable(flags types.Flags) table.GenerateFunc {
 func expectedReleaseTargets(binary string) map[string]bool {
 	targets := make(map[string]bool, 0)
 	for _, operatingSystem := range []string{"darwin", "windows", "linux"} {
-		for _, channel := range []string{"stable", "beta", "nightly"} {
-			targets[fmt.Sprintf("%s/%s/%s/release.json", binary, operatingSystem, channel)] = true
+		for _, arch := range []string{"universal", "arm64", "amd64"} {
+			if operatingSystem != "darwin" && arch == "universal" {
+				continue
+			}
+			for _, channel := range []string{"stable", "beta", "alpha", "nightly"} {
+				targets[path.Join(binary, operatingSystem, arch, channel, "release.json")] = true
+			}
 		}
+
 	}
 
 	return targets

pkg/osquery/tables/tufinfo/release_version_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/kolide/launcher/pkg/agent/types/mocks"
+	"github.com/kolide/launcher/pkg/autoupdate/tuf"
 	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
 	"github.com/osquery/osquery-go/gen/osquery"
 
@@ -26,8 +27,8 @@ func TestTufReleaseVersionTable(t *testing.T) {
 
 	testRootDir := t.TempDir()
 	v := randomSemver()
-	expectedResults["launcher"] = fmt.Sprintf("launcher/%s/launcher-%s.tar.gz", runtime.GOOS, v)
-	expectedResults["osqueryd"] = fmt.Sprintf("osqueryd/%s/osqueryd-%s.tar.gz", runtime.GOOS, v)
+	expectedResults["launcher"] = fmt.Sprintf("launcher/%s/%s/launcher-%s.tar.gz", runtime.GOOS, tuf.PlatformArch(), v)
+	expectedResults["osqueryd"] = fmt.Sprintf("osqueryd/%s/%s/osqueryd-%s.tar.gz", runtime.GOOS, tuf.PlatformArch(), v)
 	tufci.SeedLocalTufRepo(t, v, testRootDir)
 
 	mockFlags := mocks.NewFlags(t)