Skip to content

Commit

Permalink
Check if a signed URL is specified and use it download tools (#953)
Browse files Browse the repository at this point in the history 
* Wrap v2 tools install function inside tools.Download

* Download tools defaulting to the replace behaviour

* Improve archive renamer and fix failing tests

* Find the correct tool and system when `version=latest` is specified

* Reintroduce caching option when downloading tools
  • Loading branch information
@MatteoPologruto
MatteoPologruto committed Jul 2, 2024
1 parent b02967e commit 04414e2
Show file tree
Hide file tree
Showing 6 changed files with 142 additions and 211 deletions.
170 changes: 13 additions & 157 deletions tools/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,43 +16,18 @@
package tools

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"

"github.com/arduino/arduino-create-agent/gen/tools"
"github.com/arduino/arduino-create-agent/utilities"
"github.com/arduino/arduino-create-agent/v2/pkgs"
"github.com/arduino/go-paths-helper"
"github.com/blang/semver"
"github.com/codeclysm/extract/v3"
)

// public vars to allow override in the tests
var (
OS = runtime.GOOS
Arch = runtime.GOARCH
)

func pathExists(path string) bool {
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
return true
}

// Download will parse the index at the indexURL for the tool to download.
// It will extract it in a folder in .arduino-create, and it will update the
// Installed map.
Expand All @@ -70,97 +45,21 @@ func pathExists(path string) bool {
// if it already exists.
func (t *Tools) Download(pack, name, version, behaviour string) error {

body, err := t.index.Read()
if err != nil {
return err
}

var data pkgs.Index
json.Unmarshal(body, &data)

// Find the tool by name
correctTool, correctSystem := findTool(pack, name, version, data)

if correctTool.Name == "" || correctSystem.URL == "" {
t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack)
return nil
}

key := correctTool.Name + "-" + correctTool.Version

// Check if it already exists
if behaviour == "keep" {
location, ok := t.getMapValue(key)
if ok && pathExists(location) {
// overwrite the default tool with this one
t.setMapValue(correctTool.Name, location)
t.logger("The tool is already present on the system")
return t.writeMap()
}
}

// Download the tool
t.logger("Downloading tool " + name + " from " + correctSystem.URL)
resp, err := http.Get(correctSystem.URL)
tool := pkgs.New(t.index, t.directory.String(), behaviour)
_, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack})
if err != nil {
return err
}
defer resp.Body.Close()

// Read the body
body, err = io.ReadAll(resp.Body)
if err != nil {
return err
}

// Checksum
checksum := sha256.Sum256(body)
checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])

if checkSumString != correctSystem.Checksum {
return errors.New("checksum doesn't match")
}

tempPath := paths.TempDir()
// Create a temporary dir to extract package
if err := tempPath.MkdirAll(); err != nil {
return fmt.Errorf("creating temp dir for extraction: %s", err)
}
tempDir, err := tempPath.MkTempDir("package-")
if err != nil {
return fmt.Errorf("creating temp dir for extraction: %s", err)
}
defer tempDir.RemoveAll()

t.logger("Unpacking tool " + name)
ctx := context.Background()
reader := bytes.NewReader(body)
// Extract into temp directory
if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
return fmt.Errorf("extracting archive: %s", err)
}

location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
err = location.RemoveAll()
path := filepath.Join(pack, name, version)
safePath, err := utilities.SafeJoin(t.directory.String(), path)
if err != nil {
return err
}

// Check package content and find package root dir
root, err := findPackageRoot(tempDir)
if err != nil {
return fmt.Errorf("searching package root dir: %s", err)
}

if err := root.Rename(location); err != nil {
if err := root.CopyDirTo(location); err != nil {
return fmt.Errorf("moving extracted archive to destination dir: %s", err)
}
}

// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
// AFAIK this is only the case for the windows-driver tool
err = t.installDrivers(location.String())
err = t.installDrivers(safePath)
if err != nil {
return err
}
Expand All @@ -169,63 +68,20 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
t.logger("Ensure that the files are executable")

// Update the tool map
t.logger("Updating map with location " + location.String())

t.setMapValue(name, location.String())
t.setMapValue(name+"-"+correctTool.Version, location.String())
return t.writeMap()
}
t.logger("Updating map with location " + safePath)

func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
files, err := parent.ReadDir()
if err != nil {
return nil, fmt.Errorf("reading package root dir: %s", err)
}
files.FilterOutPrefix("__MACOSX")
t.setMapValue(name, safePath)
t.setMapValue(name+"-"+version, safePath)

// if there is only one dir, it is the root dir
if len(files) == 1 && files[0].IsDir() {
return files[0], nil
}
return parent, nil
}

func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
var correctTool pkgs.Tool
correctTool.Version = "0.0"

for _, p := range data.Packages {
if p.Name != pack {
continue
}
for _, t := range p.Tools {
if version != "latest" {
if t.Name == name && t.Version == version {
correctTool = t
}
} else {
// Find latest
v1, _ := semver.Make(t.Version)
v2, _ := semver.Make(correctTool.Version)
if t.Name == name && v1.Compare(v2) > 0 {
correctTool = t
}
}
}
}

// Find the url based on system
correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)

return correctTool, correctSystem
return nil
}

func (t *Tools) installDrivers(location string) error {
OkPressed := 6
extension := ".bat"
// add .\ to force locality
preamble := ".\\"
if OS != "windows" {
if runtime.GOOS != "windows" {
extension = ".sh"
// add ./ to force locality
preamble = "./"
Expand All @@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error {
os.Chdir(location)
t.logger(preamble + "post_install" + extension)
oscmd := exec.Command(preamble + "post_install" + extension)
if OS != "linux" {
if runtime.GOOS != "linux" {
// spawning a shell could be the only way to let the user type his password
TellCommandNotToSpawnShell(oscmd)
}
Expand Down
24 changes: 13 additions & 11 deletions tools/download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) {
{"linux", "arm", "arm-linux-gnueabihf"},
}
defer func() {
OS = runtime.GOOS // restore `runtime.OS`
Arch = runtime.GOARCH // restore `runtime.ARCH`
pkgs.OS = runtime.GOOS // restore `runtime.OS`
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
}()
testIndex := paths.New("testdata", "test_tool_index.json")
buf, err := testIndex.ReadFile()
Expand All @@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) {
require.NoError(t, err)
for _, tc := range testCases {
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
OS = tc.hostOS // override `runtime.OS` for testing purposes
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
// Find the tool by name
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
require.True(t, found)
require.NotNil(t, correctTool)
require.NotNil(t, correctSystem)
require.Equal(t, correctTool.Name, "arduino-fwuploader")
Expand All @@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) {
{"windows", "amd64", "i686-mingw32"},
}
defer func() {
OS = runtime.GOOS // restore `runtime.OS`
Arch = runtime.GOARCH // restore `runtime.ARCH`
pkgs.OS = runtime.GOOS // restore `runtime.OS`
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
}()
testIndex := paths.New("testdata", "test_tool_index.json")
buf, err := testIndex.ReadFile()
Expand All @@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) {
require.NoError(t, err)
for _, tc := range testCases {
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
OS = tc.hostOS // override `runtime.OS` for testing purposes
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
// Find the tool by name
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
require.True(t, found)
require.NotNil(t, correctTool)
require.NotNil(t, correctSystem)
require.Equal(t, correctTool.Name, "arduino-fwuploader")
Expand Down Expand Up @@ -145,7 +147,7 @@ func TestDownload(t *testing.T) {
if filePath.IsDir() {
require.DirExists(t, filePath.String())
} else {
if OS == "windows" {
if runtime.GOOS == "windows" {
require.FileExists(t, filePath.String()+".exe")
} else {
require.FileExists(t, filePath.String())
Expand Down
12 changes: 0 additions & 12 deletions tools/tools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) {
return value, ok
}

// writeMap() writes installed map to the json file "installed.json"
func (t *Tools) writeMap() error {
t.mutex.RLock()
defer t.mutex.RUnlock()
b, err := json.Marshal(t.installed)
if err != nil {
return err
}
filePath := t.directory.Join("installed.json")
return filePath.WriteFile(b)
}

// readMap() reads the installed map from json file "installed.json"
func (t *Tools) readMap() error {
t.mutex.Lock()
Expand Down
2 changes: 1 addition & 1 deletion v2/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler {
logAdapter := LogAdapter{Logger: logger}

// Mount tools
toolsSvc := pkgs.New(index, directory)
toolsSvc := pkgs.New(index, directory, "replace")
toolsEndpoints := toolssvc.NewEndpoints(toolsSvc)
toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil)
toolssvr.Mount(mux, toolsServer)
Expand Down
Loading

0 comments on commit 04414e2

Please sign in to comment.