Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check if a signed URL is specified and use it download tools #953

Merged
merged 5 commits into from
Jul 2, 2024
Merged

Check if a signed URL is specified and use it download tools #953

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0a87ca2
Wrap v2 tools install function inside tools.Download
MatteoPologruto May 15, 2024
ed8eac6
Download tools defaulting to the replace behaviour
MatteoPologruto May 22, 2024
887b22b
Improve archive renamer and fix failing tests
MatteoPologruto May 29, 2024
a882ed7
Find the correct tool and system when `version=latest` is specified
MatteoPologruto May 29, 2024
18e9250
Reintroduce caching option when downloading tools
MatteoPologruto Jun 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 13 additions & 157 deletions tools/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,43 +16,18 @@
package tools

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"

"github.com/arduino/arduino-create-agent/gen/tools"
"github.com/arduino/arduino-create-agent/utilities"
"github.com/arduino/arduino-create-agent/v2/pkgs"
"github.com/arduino/go-paths-helper"
"github.com/blang/semver"
"github.com/codeclysm/extract/v3"
)

// public vars to allow override in the tests
var (
OS = runtime.GOOS
Arch = runtime.GOARCH
)

func pathExists(path string) bool {
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
return true
}

// Download will parse the index at the indexURL for the tool to download.
// It will extract it in a folder in .arduino-create, and it will update the
// Installed map.
Expand All @@ -70,97 +45,21 @@ func pathExists(path string) bool {
// if it already exists.
func (t *Tools) Download(pack, name, version, behaviour string) error {

body, err := t.index.Read()
if err != nil {
return err
}

var data pkgs.Index
json.Unmarshal(body, &data)

// Find the tool by name
correctTool, correctSystem := findTool(pack, name, version, data)

if correctTool.Name == "" || correctSystem.URL == "" {
t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack)
return nil
}

key := correctTool.Name + "-" + correctTool.Version

// Check if it already exists
if behaviour == "keep" {
location, ok := t.getMapValue(key)
if ok && pathExists(location) {
// overwrite the default tool with this one
t.setMapValue(correctTool.Name, location)
t.logger("The tool is already present on the system")
return t.writeMap()
}
}

// Download the tool
t.logger("Downloading tool " + name + " from " + correctSystem.URL)
resp, err := http.Get(correctSystem.URL)
tool := pkgs.New(t.index, t.directory.String(), behaviour)
_, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack})
if err != nil {
return err
}
defer resp.Body.Close()

// Read the body
body, err = io.ReadAll(resp.Body)
if err != nil {
return err
}

// Checksum
checksum := sha256.Sum256(body)
checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])

if checkSumString != correctSystem.Checksum {
return errors.New("checksum doesn't match")
}

tempPath := paths.TempDir()
// Create a temporary dir to extract package
if err := tempPath.MkdirAll(); err != nil {
return fmt.Errorf("creating temp dir for extraction: %s", err)
}
tempDir, err := tempPath.MkTempDir("package-")
if err != nil {
return fmt.Errorf("creating temp dir for extraction: %s", err)
}
defer tempDir.RemoveAll()

t.logger("Unpacking tool " + name)
ctx := context.Background()
reader := bytes.NewReader(body)
// Extract into temp directory
if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
return fmt.Errorf("extracting archive: %s", err)
}

location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
err = location.RemoveAll()
path := filepath.Join(pack, name, version)
safePath, err := utilities.SafeJoin(t.directory.String(), path)
if err != nil {
return err
}

// Check package content and find package root dir
root, err := findPackageRoot(tempDir)
if err != nil {
return fmt.Errorf("searching package root dir: %s", err)
}

if err := root.Rename(location); err != nil {
if err := root.CopyDirTo(location); err != nil {
return fmt.Errorf("moving extracted archive to destination dir: %s", err)
}
}

// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
// AFAIK this is only the case for the windows-driver tool
err = t.installDrivers(location.String())
err = t.installDrivers(safePath)
if err != nil {
return err
}
Expand All @@ -169,63 +68,20 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
t.logger("Ensure that the files are executable")

// Update the tool map
t.logger("Updating map with location " + location.String())

t.setMapValue(name, location.String())
t.setMapValue(name+"-"+correctTool.Version, location.String())
return t.writeMap()
}
t.logger("Updating map with location " + safePath)

func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
files, err := parent.ReadDir()
if err != nil {
return nil, fmt.Errorf("reading package root dir: %s", err)
}
files.FilterOutPrefix("__MACOSX")
t.setMapValue(name, safePath)
t.setMapValue(name+"-"+version, safePath)

// if there is only one dir, it is the root dir
if len(files) == 1 && files[0].IsDir() {
return files[0], nil
}
return parent, nil
}

func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
var correctTool pkgs.Tool
correctTool.Version = "0.0"

for _, p := range data.Packages {
if p.Name != pack {
continue
}
for _, t := range p.Tools {
if version != "latest" {
if t.Name == name && t.Version == version {
correctTool = t
}
} else {
// Find latest
v1, _ := semver.Make(t.Version)
v2, _ := semver.Make(correctTool.Version)
if t.Name == name && v1.Compare(v2) > 0 {
correctTool = t
}
}
}
}

// Find the url based on system
correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)

return correctTool, correctSystem
return nil
}

func (t *Tools) installDrivers(location string) error {
OkPressed := 6
extension := ".bat"
// add .\ to force locality
preamble := ".\\"
if OS != "windows" {
if runtime.GOOS != "windows" {
extension = ".sh"
// add ./ to force locality
preamble = "./"
Expand All @@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error {
os.Chdir(location)
t.logger(preamble + "post_install" + extension)
oscmd := exec.Command(preamble + "post_install" + extension)
if OS != "linux" {
if runtime.GOOS != "linux" {
// spawning a shell could be the only way to let the user type his password
TellCommandNotToSpawnShell(oscmd)
}
Expand Down
24 changes: 13 additions & 11 deletions tools/download_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) {
{"linux", "arm", "arm-linux-gnueabihf"},
}
defer func() {
OS = runtime.GOOS // restore `runtime.OS`
Arch = runtime.GOARCH // restore `runtime.ARCH`
pkgs.OS = runtime.GOOS // restore `runtime.OS`
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
}()
testIndex := paths.New("testdata", "test_tool_index.json")
buf, err := testIndex.ReadFile()
Expand All @@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) {
require.NoError(t, err)
for _, tc := range testCases {
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
OS = tc.hostOS // override `runtime.OS` for testing purposes
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
// Find the tool by name
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
require.True(t, found)
require.NotNil(t, correctTool)
require.NotNil(t, correctSystem)
require.Equal(t, correctTool.Name, "arduino-fwuploader")
Expand All @@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) {
{"windows", "amd64", "i686-mingw32"},
}
defer func() {
OS = runtime.GOOS // restore `runtime.OS`
Arch = runtime.GOARCH // restore `runtime.ARCH`
pkgs.OS = runtime.GOOS // restore `runtime.OS`
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
}()
testIndex := paths.New("testdata", "test_tool_index.json")
buf, err := testIndex.ReadFile()
Expand All @@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) {
require.NoError(t, err)
for _, tc := range testCases {
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
OS = tc.hostOS // override `runtime.OS` for testing purposes
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
// Find the tool by name
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
require.True(t, found)
require.NotNil(t, correctTool)
require.NotNil(t, correctSystem)
require.Equal(t, correctTool.Name, "arduino-fwuploader")
Expand Down Expand Up @@ -145,7 +147,7 @@ func TestDownload(t *testing.T) {
if filePath.IsDir() {
require.DirExists(t, filePath.String())
} else {
if OS == "windows" {
if runtime.GOOS == "windows" {
require.FileExists(t, filePath.String()+".exe")
} else {
require.FileExists(t, filePath.String())
Expand Down
12 changes: 0 additions & 12 deletions tools/tools.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) {
return value, ok
}

// writeMap() writes installed map to the json file "installed.json"
func (t *Tools) writeMap() error {
t.mutex.RLock()
defer t.mutex.RUnlock()
b, err := json.Marshal(t.installed)
if err != nil {
return err
}
filePath := t.directory.Join("installed.json")
return filePath.WriteFile(b)
}

// readMap() reads the installed map from json file "installed.json"
func (t *Tools) readMap() error {
t.mutex.Lock()
Expand Down
2 changes: 1 addition & 1 deletion v2/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler {
logAdapter := LogAdapter{Logger: logger}

// Mount tools
toolsSvc := pkgs.New(index, directory)
toolsSvc := pkgs.New(index, directory, "replace")
toolsEndpoints := toolssvc.NewEndpoints(toolsSvc)
toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil)
toolssvr.Mount(mux, toolsServer)
Expand Down
Loading