From 04414e26c0d8cf0795d02c364637c3bb3f218bd4 Mon Sep 17 00:00:00 2001 From: MatteoPologruto <109663225+MatteoPologruto@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:31:09 +0200 Subject: [PATCH] Check if a signed URL is specified and use it download tools (#953) * Wrap v2 tools install function inside tools.Download * Download tools defaulting to the replace behaviour * Improve archive renamer and fix failing tests * Find the correct tool and system when `version=latest` is specified * Reintroduce caching option when downloading tools --- tools/download.go | 170 ++++------------------------------------- tools/download_test.go | 24 +++--- tools/tools.go | 12 --- v2/http.go | 2 +- v2/pkgs/tools.go | 137 ++++++++++++++++++++++++++------- v2/pkgs/tools_test.go | 8 +- 6 files changed, 142 insertions(+), 211 deletions(-) diff --git a/tools/download.go b/tools/download.go index 360d6e4c3..6e5fa8b7f 100644 --- a/tools/download.go +++ b/tools/download.go @@ -16,43 +16,18 @@ package tools import ( - "bytes" "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" - "fmt" - "io" - "net/http" "os" "os/exec" "path/filepath" "runtime" + "github.com/arduino/arduino-create-agent/gen/tools" + "github.com/arduino/arduino-create-agent/utilities" "github.com/arduino/arduino-create-agent/v2/pkgs" - "github.com/arduino/go-paths-helper" - "github.com/blang/semver" - "github.com/codeclysm/extract/v3" ) -// public vars to allow override in the tests -var ( - OS = runtime.GOOS - Arch = runtime.GOARCH -) - -func pathExists(path string) bool { - _, err := os.Stat(path) - if err == nil { - return true - } - if os.IsNotExist(err) { - return false - } - return true -} - // Download will parse the index at the indexURL for the tool to download. // It will extract it in a folder in .arduino-create, and it will update the // Installed map. @@ -70,97 +45,21 @@ func pathExists(path string) bool { // if it already exists. func (t *Tools) Download(pack, name, version, behaviour string) error { - body, err := t.index.Read() - if err != nil { - return err - } - - var data pkgs.Index - json.Unmarshal(body, &data) - - // Find the tool by name - correctTool, correctSystem := findTool(pack, name, version, data) - - if correctTool.Name == "" || correctSystem.URL == "" { - t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack) - return nil - } - - key := correctTool.Name + "-" + correctTool.Version - - // Check if it already exists - if behaviour == "keep" { - location, ok := t.getMapValue(key) - if ok && pathExists(location) { - // overwrite the default tool with this one - t.setMapValue(correctTool.Name, location) - t.logger("The tool is already present on the system") - return t.writeMap() - } - } - - // Download the tool - t.logger("Downloading tool " + name + " from " + correctSystem.URL) - resp, err := http.Get(correctSystem.URL) + tool := pkgs.New(t.index, t.directory.String(), behaviour) + _, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack}) if err != nil { return err } - defer resp.Body.Close() - - // Read the body - body, err = io.ReadAll(resp.Body) - if err != nil { - return err - } - - // Checksum - checksum := sha256.Sum256(body) - checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size]) - - if checkSumString != correctSystem.Checksum { - return errors.New("checksum doesn't match") - } - - tempPath := paths.TempDir() - // Create a temporary dir to extract package - if err := tempPath.MkdirAll(); err != nil { - return fmt.Errorf("creating temp dir for extraction: %s", err) - } - tempDir, err := tempPath.MkTempDir("package-") - if err != nil { - return fmt.Errorf("creating temp dir for extraction: %s", err) - } - defer tempDir.RemoveAll() - t.logger("Unpacking tool " + name) - ctx := context.Background() - reader := bytes.NewReader(body) - // Extract into temp directory - if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil { - return fmt.Errorf("extracting archive: %s", err) - } - - location := t.directory.Join(pack, correctTool.Name, correctTool.Version) - err = location.RemoveAll() + path := filepath.Join(pack, name, version) + safePath, err := utilities.SafeJoin(t.directory.String(), path) if err != nil { return err } - // Check package content and find package root dir - root, err := findPackageRoot(tempDir) - if err != nil { - return fmt.Errorf("searching package root dir: %s", err) - } - - if err := root.Rename(location); err != nil { - if err := root.CopyDirTo(location); err != nil { - return fmt.Errorf("moving extracted archive to destination dir: %s", err) - } - } - // if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers // AFAIK this is only the case for the windows-driver tool - err = t.installDrivers(location.String()) + err = t.installDrivers(safePath) if err != nil { return err } @@ -169,55 +68,12 @@ func (t *Tools) Download(pack, name, version, behaviour string) error { t.logger("Ensure that the files are executable") // Update the tool map - t.logger("Updating map with location " + location.String()) - - t.setMapValue(name, location.String()) - t.setMapValue(name+"-"+correctTool.Version, location.String()) - return t.writeMap() -} + t.logger("Updating map with location " + safePath) -func findPackageRoot(parent *paths.Path) (*paths.Path, error) { - files, err := parent.ReadDir() - if err != nil { - return nil, fmt.Errorf("reading package root dir: %s", err) - } - files.FilterOutPrefix("__MACOSX") + t.setMapValue(name, safePath) + t.setMapValue(name+"-"+version, safePath) - // if there is only one dir, it is the root dir - if len(files) == 1 && files[0].IsDir() { - return files[0], nil - } - return parent, nil -} - -func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) { - var correctTool pkgs.Tool - correctTool.Version = "0.0" - - for _, p := range data.Packages { - if p.Name != pack { - continue - } - for _, t := range p.Tools { - if version != "latest" { - if t.Name == name && t.Version == version { - correctTool = t - } - } else { - // Find latest - v1, _ := semver.Make(t.Version) - v2, _ := semver.Make(correctTool.Version) - if t.Name == name && v1.Compare(v2) > 0 { - correctTool = t - } - } - } - } - - // Find the url based on system - correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch) - - return correctTool, correctSystem + return nil } func (t *Tools) installDrivers(location string) error { @@ -225,7 +81,7 @@ func (t *Tools) installDrivers(location string) error { extension := ".bat" // add .\ to force locality preamble := ".\\" - if OS != "windows" { + if runtime.GOOS != "windows" { extension = ".sh" // add ./ to force locality preamble = "./" @@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error { os.Chdir(location) t.logger(preamble + "post_install" + extension) oscmd := exec.Command(preamble + "post_install" + extension) - if OS != "linux" { + if runtime.GOOS != "linux" { // spawning a shell could be the only way to let the user type his password TellCommandNotToSpawnShell(oscmd) } diff --git a/tools/download_test.go b/tools/download_test.go index c45914b51..1e958de91 100644 --- a/tools/download_test.go +++ b/tools/download_test.go @@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) { {"linux", "arm", "arm-linux-gnueabihf"}, } defer func() { - OS = runtime.GOOS // restore `runtime.OS` - Arch = runtime.GOARCH // restore `runtime.ARCH` + pkgs.OS = runtime.GOOS // restore `runtime.OS` + pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH` }() testIndex := paths.New("testdata", "test_tool_index.json") buf, err := testIndex.ReadFile() @@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) { require.NoError(t, err) for _, tc := range testCases { t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) { - OS = tc.hostOS // override `runtime.OS` for testing purposes - Arch = tc.hostArch // override `runtime.ARCH` for testing purposes + pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes + pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes // Find the tool by name - correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data) + correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data) + require.True(t, found) require.NotNil(t, correctTool) require.NotNil(t, correctSystem) require.Equal(t, correctTool.Name, "arduino-fwuploader") @@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) { {"windows", "amd64", "i686-mingw32"}, } defer func() { - OS = runtime.GOOS // restore `runtime.OS` - Arch = runtime.GOARCH // restore `runtime.ARCH` + pkgs.OS = runtime.GOOS // restore `runtime.OS` + pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH` }() testIndex := paths.New("testdata", "test_tool_index.json") buf, err := testIndex.ReadFile() @@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) { require.NoError(t, err) for _, tc := range testCases { t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) { - OS = tc.hostOS // override `runtime.OS` for testing purposes - Arch = tc.hostArch // override `runtime.ARCH` for testing purposes + pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes + pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes // Find the tool by name - correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data) + correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data) + require.True(t, found) require.NotNil(t, correctTool) require.NotNil(t, correctSystem) require.Equal(t, correctTool.Name, "arduino-fwuploader") @@ -145,7 +147,7 @@ func TestDownload(t *testing.T) { if filePath.IsDir() { require.DirExists(t, filePath.String()) } else { - if OS == "windows" { + if runtime.GOOS == "windows" { require.FileExists(t, filePath.String()+".exe") } else { require.FileExists(t, filePath.String()) diff --git a/tools/tools.go b/tools/tools.go index e641db351..cb9efc787 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) { return value, ok } -// writeMap() writes installed map to the json file "installed.json" -func (t *Tools) writeMap() error { - t.mutex.RLock() - defer t.mutex.RUnlock() - b, err := json.Marshal(t.installed) - if err != nil { - return err - } - filePath := t.directory.Join("installed.json") - return filePath.WriteFile(b) -} - // readMap() reads the installed map from json file "installed.json" func (t *Tools) readMap() error { t.mutex.Lock() diff --git a/v2/http.go b/v2/http.go index bcfbc82aa..390ec3989 100644 --- a/v2/http.go +++ b/v2/http.go @@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler { logAdapter := LogAdapter{Logger: logger} // Mount tools - toolsSvc := pkgs.New(index, directory) + toolsSvc := pkgs.New(index, directory, "replace") toolsEndpoints := toolssvc.NewEndpoints(toolsSvc) toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil) toolssvr.Mount(mux, toolsServer) diff --git a/v2/pkgs/tools.go b/v2/pkgs/tools.go index 55ff6c2e4..b0daaaaef 100644 --- a/v2/pkgs/tools.go +++ b/v2/pkgs/tools.go @@ -33,9 +33,16 @@ import ( "github.com/arduino/arduino-create-agent/gen/tools" "github.com/arduino/arduino-create-agent/index" "github.com/arduino/arduino-create-agent/utilities" + "github.com/blang/semver" "github.com/codeclysm/extract/v3" ) +// public vars to allow override in the tests +var ( + OS = runtime.GOOS + Arch = runtime.GOARCH +) + // Tools is a client that implements github.com/arduino/arduino-create-agent/gen/tools.Service interface. // It saves tools in a specified folder with this structure: packager/name/version // For example: @@ -50,17 +57,19 @@ import ( // // It requires an Index Resource to search for tools type Tools struct { - index *index.Resource - folder string + index *index.Resource + folder string + behaviour string } // New will return a Tool object, allowing the caller to execute operations on it. // The New function will accept an index as parameter (used to download the indexes) // and a folder used to download the indexes -func New(index *index.Resource, folder string) *Tools { +func New(index *index.Resource, folder, behaviour string) *Tools { return &Tools{ - index: index, - folder: folder, + index: index, + folder: folder, + behaviour: behaviour, } } @@ -166,21 +175,28 @@ func (t *Tools) Install(ctx context.Context, payload *tools.ToolPayload) (*tools var index Index json.Unmarshal(body, &index) - for _, packager := range index.Packages { - if packager.Name != payload.Packager { - continue - } - - for _, tool := range packager.Tools { - if tool.Name == payload.Name && - tool.Version == payload.Version { - - sys := tool.GetFlavourCompatibleWith(runtime.GOOS, runtime.GOARCH) + correctTool, correctSystem, found := FindTool(payload.Packager, payload.Name, payload.Version, index) + path = filepath.Join(payload.Packager, correctTool.Name, correctTool.Version) - return t.install(ctx, path, sys.URL, sys.Checksum) + key := correctTool.Name + "-" + correctTool.Version + // Check if it already exists + if t.behaviour == "keep" && pathExists(t.folder) { + location, ok, err := checkInstalled(t.folder, key) + if err != nil { + return nil, err + } + if ok && pathExists(location) { + // overwrite the default tool with this one + err := writeInstalled(t.folder, path) + if err != nil { + return nil, err } + return &tools.Operation{Status: "ok"}, nil } } + if found { + return t.install(ctx, path, correctSystem.URL, correctSystem.Checksum) + } return nil, tools.MakeNotFound( fmt.Errorf("tool not found with packager '%s', name '%s', version '%s'", @@ -256,27 +272,51 @@ func (t *Tools) Remove(ctx context.Context, payload *tools.ToolPayload) (*tools. func rename(base string) extract.Renamer { return func(path string) string { parts := strings.Split(filepath.ToSlash(path), "/") - path = strings.Join(parts[1:], "/") - path = filepath.Join(base, path) + newPath := strings.Join(parts[1:], "/") + if newPath == "" { + newPath = filepath.Join(newPath, path) + } + path = filepath.Join(base, newPath) return path } } -func writeInstalled(folder, path string) error { +func readInstalled(installedFile string) (map[string]string, error) { // read installed.json installed := map[string]string{} - - installedFile, err := utilities.SafeJoin(folder, "installed.json") - if err != nil { - return err - } data, err := os.ReadFile(installedFile) if err == nil { err = json.Unmarshal(data, &installed) if err != nil { - return err + return nil, err } } + return installed, nil +} + +func checkInstalled(folder, key string) (string, bool, error) { + installedFile, err := utilities.SafeJoin(folder, "installed.json") + if err != nil { + return "", false, err + } + installed, err := readInstalled(installedFile) + if err != nil { + return "", false, err + } + location, ok := installed[key] + return location, ok, err +} + +func writeInstalled(folder, path string) error { + // read installed.json + installedFile, err := utilities.SafeJoin(folder, "installed.json") + if err != nil { + return err + } + installed, err := readInstalled(installedFile) + if err != nil { + return err + } parts := strings.Split(path, string(filepath.Separator)) tool := parts[len(parts)-2] @@ -288,10 +328,55 @@ func writeInstalled(folder, path string) error { installed[tool] = toolFile installed[toolWithVersion] = toolFile - data, err = json.Marshal(installed) + data, err := json.Marshal(installed) if err != nil { return err } return os.WriteFile(installedFile, data, 0644) } + +func pathExists(path string) bool { + _, err := os.Stat(path) + if err == nil { + return true + } + if os.IsNotExist(err) { + return false + } + return true +} + +// FindTool searches the index for the correct tool and system that match the specified tool name and version +func FindTool(pack, name, version string, data Index) (Tool, System, bool) { + var correctTool Tool + correctTool.Version = "0.0" + found := false + + for _, p := range data.Packages { + if p.Name != pack { + continue + } + for _, t := range p.Tools { + if version != "latest" { + if t.Name == name && t.Version == version { + correctTool = t + found = true + } + } else { + // Find latest + v1, _ := semver.Make(t.Version) + v2, _ := semver.Make(correctTool.Version) + if t.Name == name && v1.Compare(v2) > 0 { + correctTool = t + found = true + } + } + } + } + + // Find the url based on system + correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch) + + return correctTool, correctSystem, found +} diff --git a/v2/pkgs/tools_test.go b/v2/pkgs/tools_test.go index 78c56398f..be4d5e4d1 100644 --- a/v2/pkgs/tools_test.go +++ b/v2/pkgs/tools_test.go @@ -45,7 +45,7 @@ func TestTools(t *testing.T) { // Instantiate Index Index := index.Init(indexURL, config.GetDataDir()) - service := pkgs.New(Index, tmp) + service := pkgs.New(Index, tmp, "replace") ctx := context.Background() @@ -126,7 +126,7 @@ func TestEvilFilename(t *testing.T) { // Instantiate Index Index := index.Init(indexURL, config.GetDataDir()) - service := pkgs.New(Index, tmp) + service := pkgs.New(Index, tmp, "replace") ctx := context.Background() @@ -195,7 +195,7 @@ func TestInstalledHead(t *testing.T) { // Instantiate Index Index := index.Init(indexURL, config.GetDataDir()) - service := pkgs.New(Index, tmp) + service := pkgs.New(Index, tmp, "replace") ctx := context.Background() @@ -216,7 +216,7 @@ func TestInstall(t *testing.T) { LastRefresh: time.Now(), } - tool := pkgs.New(testIndex, tmp) + tool := pkgs.New(testIndex, tmp, "replace") ctx := context.Background()