Added tests for lib install with flags

Author: vinay-lanka

arduino/libraries/librariesmanager/install.go

@@ -16,18 +16,17 @@
 package librariesmanager
 
 import (
-	"archive/zip"
+	"context"
 	"errors"
 	"fmt"
-	"io"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesindex"
 	"github.com/arduino/arduino-cli/arduino/utils"
 	paths "github.com/arduino/go-paths-helper"
+	"github.com/codeclysm/extract/v3"
 	"gopkg.in/src-d/go-git.v4"
 )
 
@@ -93,94 +92,37 @@ func (lm *LibrariesManager) Uninstall(lib *libraries.Library) error {
 }
 
 //InstallZipLib  installs a Zip library on the specified path.
-func (lm *LibrariesManager) InstallZipLib(libPath string) error {
+func (lm *LibrariesManager) InstallZipLib(ctx context.Context, archivePath string) error {
 	libsDir := lm.getUserLibrariesDir()
 	if libsDir == nil {
 		return fmt.Errorf("User directory not set")
 	}
-	err := Unzip(libPath, libsDir.String())
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-//Unzip takes the ZipLibPath and Extracts it to LibsDir
-func Unzip(src string, dest string) error {
-
-	var filenames []string
 
-	r, err := zip.OpenReader(src)
+	file, err := os.Open(archivePath)
 	if err != nil {
 		return err
 	}
-	defer r.Close()
-
-	for _, f := range r.File {
-
-		// Store filename/path for returning and using later on
-		fpath := filepath.Join(dest, f.Name)
-
-		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
-		if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("%s: illegal file path", fpath)
-		}
-
-		filenames = append(filenames, fpath)
-
-		if f.FileInfo().IsDir() {
-			// Make Folder
-			os.MkdirAll(fpath, os.ModePerm)
-			continue
-		}
+	defer file.Close()
 
-		// Make File
-		if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
-			return err
-		}
-
-		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-		if err != nil {
-			return err
-		}
-
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-
-		_, err = io.Copy(outFile, rc)
-
-		// Close the file without defer to close before next iteration of loop
-		outFile.Close()
-		rc.Close()
-
-		if err != nil {
-			return err
-		}
+	if err := extract.Archive(ctx, file, libsDir.String(), nil); err != nil {
+		return fmt.Errorf("extracting archive: %s", err)
 	}
 	return nil
 }
 
 //InstallGitLib  installs a library hosted on a git repository on the specified path.
-func (lm *LibrariesManager) InstallGitLib(Name string, gitURL string) error {
+func (lm *LibrariesManager) InstallGitLib(url string) error {
 	libsDir := lm.getUserLibrariesDir()
 	if libsDir == nil {
 		return fmt.Errorf("User directory not set")
 	}
-	err := Fetch(Name, gitURL, libsDir.String())
-	if err != nil {
-		return err
-	}
-	return nil
-}
+	i := strings.LastIndex(url, "/")
+	folder := strings.TrimRight(url[i+1:], ".git")
+	path := libsDir.Join(folder)
 
-//Fetch Clones the repository to LibsDir
-func Fetch(name string, url string, dest string) error {
-	directory := dest + "/" + name
-	_, err := git.PlainClone(directory, false, &git.CloneOptions{
-		URL:               url,
-		RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
+	_, err := git.PlainClone(path.String(), false, &git.CloneOptions{
+		URL:      url,
+		Progress: os.Stdout,
 	})
 	if err != nil {
 		return err

cli/lib/install.go

@@ -40,38 +40,37 @@ func initInstallCommand() *cobra.Command {
 		Run:  runInstallCommand,
 	}
 	installCommand.Flags().BoolVar(&installFlags.noDeps, "no-deps", false, "Do not install dependencies.")
-	installCommand.Flags().StringVarP(&installFlags.gitURL, "git-url", "g", "", "Enter git url for libraries hosted on repositories")
-	installCommand.Flags().StringVarP(&installFlags.zipPath, "zip-path", "z", "", "Enter a path to zip file")
+	installCommand.Flags().BoolVar(&installFlags.gitURL, "git-url", false, "Enter git url for libraries hosted on repositories")
+	installCommand.Flags().BoolVar(&installFlags.zipPath, "zip-path", false, "Enter a path to zip file")
 	return installCommand
 }
 
 var installFlags struct {
 	noDeps  bool
-	gitURL  string
-	zipPath string
+	gitURL  bool
+	zipPath bool
 }
 
 func runInstallCommand(cmd *cobra.Command, args []string) {
 	instance := instance.CreateInstanceIgnorePlatformIndexErrors()
-	if installFlags.zipPath != "" {
-		ZiplibraryInstallReq := &rpc.ZipLibraryInstallReq{
+	if installFlags.zipPath {
+		ziplibraryInstallReq := &rpc.ZipLibraryInstallReq{
 			Instance: instance,
-			Path:     installFlags.zipPath,
+			Path:     args[0],
 		}
-		_, err := lib.ZipLibraryInstall(context.Background(), ZiplibraryInstallReq)
+		err := lib.ZipLibraryInstall(context.Background(), ziplibraryInstallReq, output.TaskProgress())
 		if err != nil {
-			feedback.Errorf("Error installing Zip Library %v", err)
+			feedback.Errorf("Error installing Zip Library: %v", err)
 			os.Exit(errorcodes.ErrGeneric)
 		}
-	} else if installFlags.gitURL != "" {
-		GitlibraryInstallReq := &rpc.GitLibraryInstallReq{
+	} else if installFlags.gitURL {
+		gitlibraryInstallReq := &rpc.GitLibraryInstallReq{
 			Instance: instance,
-			Name:     args[0],
-			Url:      installFlags.gitURL,
+			Url:      args[0],
 		}
-		_, err := lib.GitLibraryInstall(context.Background(), GitlibraryInstallReq)
+		err := lib.GitLibraryInstall(context.Background(), gitlibraryInstallReq, output.TaskProgress())
 		if err != nil {
-			feedback.Errorf("Error installing Git Library %v", err)
+			feedback.Errorf("Error installing Git Library: %v", err)
 			os.Exit(errorcodes.ErrGeneric)
 		}
 	} else {

commands/daemon/daemon.go

@@ -358,11 +358,25 @@ func (s *ArduinoCoreServerImpl) ArchiveSketch(ctx context.Context, req *rpc.Arch
 }
 
 //ZipLibraryInstall FIXMEDOC
-func (s *ArduinoCoreServerImpl) ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallReq) (*rpc.ZipLibraryInstallResp, error) {
-	return lib.ZipLibraryInstall(ctx, req)
+func (s *ArduinoCoreServerImpl) ZipLibraryInstall(req *rpc.ZipLibraryInstallReq, stream rpc.ArduinoCore_ZipLibraryInstallServer) error {
+	err := lib.ZipLibraryInstall(
+		stream.Context(), req,
+		func(p *rpc.TaskProgress) { stream.Send(&rpc.ZipLibraryInstallResp{TaskProgress: p}) },
+	)
+	if err != nil {
+		return err
+	}
+	return stream.Send(&rpc.ZipLibraryInstallResp{})
 }
 
 //GitLibraryInstall FIXMEDOC
-func (s *ArduinoCoreServerImpl) GitLibraryInstall(ctx context.Context, req *rpc.GitLibraryInstallReq) (*rpc.GitLibraryInstallResp, error) {
-	return lib.GitLibraryInstall(ctx, req)
+func (s *ArduinoCoreServerImpl) GitLibraryInstall(req *rpc.GitLibraryInstallReq, stream rpc.ArduinoCore_GitLibraryInstallServer) error {
+	err := lib.GitLibraryInstall(
+		stream.Context(), req,
+		func(p *rpc.TaskProgress) { stream.Send(&rpc.GitLibraryInstallResp{TaskProgress: p}) },
+	)
+	if err != nil {
+		return err
+	}
+	return stream.Send(&rpc.GitLibraryInstallResp{})
 }

commands/lib/install.go

@@ -77,42 +77,23 @@ func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *libraries
 }
 
 //ZipLibraryInstall FIXMEDOC
-func ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallReq) (*rpc.ZipLibraryInstallResp, error) {
-	res := &rpc.ZipLibraryInstallResp{}
+func ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallReq, taskCB commands.TaskProgressCB) error {
 	lm := commands.GetLibraryManager(req.GetInstance().GetId())
-	Path := req.GetPath()
-	if err := installZipLibrary(lm, Path); err != nil {
-		res.Status = "Error installing Zip Library"
-		return res, err
-	}
-	res.Status = "Success! Installed Zip Library"
-	return res, nil
-}
-
-func installZipLibrary(lm *librariesmanager.LibrariesManager, libPath string) error {
-	if err := lm.InstallZipLib(libPath); err != nil {
+	archivePath := req.GetPath()
+	if err := lm.InstallZipLib(ctx, archivePath); err != nil {
 		return err
 	}
+	taskCB(&rpc.TaskProgress{Message: "Installed Archived Library", Completed: true})
 	return nil
 }
 
 //GitLibraryInstall FIXMEDOC
-func GitLibraryInstall(ctx context.Context, req *rpc.GitLibraryInstallReq) (*rpc.GitLibraryInstallResp, error) {
-	res := &rpc.GitLibraryInstallResp{}
+func GitLibraryInstall(ctx context.Context, req *rpc.GitLibraryInstallReq, taskCB commands.TaskProgressCB) error {
 	lm := commands.GetLibraryManager(req.GetInstance().GetId())
-	URL := req.GetUrl()
-	Name := req.GetName()
-	if err := installGitLibrary(lm, Name, URL); err != nil {
-		res.Status = "Error installing Git Library"
-		return res, err
-	}
-	res.Status = "Success! Installed Git Library"
-	return res, nil
-}
-
-func installGitLibrary(lm *librariesmanager.LibrariesManager, Name string, gitURL string) error {
-	if err := lm.InstallGitLib(Name, gitURL); err != nil {
+	url := req.GetUrl()
+	if err := lm.InstallGitLib(url); err != nil {
 		return err
 	}
+	taskCB(&rpc.TaskProgress{Message: "Installed Library from Git URL", Completed: true})
 	return nil
 }

go.mod

@@ -21,11 +21,10 @@ require (
 	github.com/h2non/filetype v1.0.8 // indirect
 	github.com/imjasonmiller/godice v0.1.2
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
-	github.com/leonelquinteros/gotext v1.4.0
-	github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2
-	github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc // indirect
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/leonelquinteros/gotext v1.4.0
+	github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2
 	github.com/mattn/go-colorable v0.1.2
 	github.com/mattn/go-isatty v0.0.8
 	github.com/mattn/go-runewidth v0.0.9 // indirect
@@ -48,17 +47,13 @@ require (
 	go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
 	go.bug.st/serial v1.1.1
 	go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect
-	golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 // indirect
+	golang.org/x/crypto v0.0.0-20200406173513-056763e48d71
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
-	golang.org/x/sys v0.0.0-20200408040146-ea54a3c99b9b // indirect
 	golang.org/x/text v0.3.2
 	google.golang.org/grpc v1.27.0
 	google.golang.org/protobuf v1.25.0
-	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
-	gopkg.in/src-d/go-git.v4 v4.13.1
-	gopkg.in/yaml.v2 v2.3.0
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1
-	gopkg.in/yaml.v2 v2.2.4
+	gopkg.in/yaml.v2 v2.3.0
 )

go.sum

@@ -123,10 +123,10 @@ service ArduinoCore {
   rpc LibraryInstall(LibraryInstallReq) returns (stream LibraryInstallResp);
 
   // Install a library from a Zip File 
-  rpc ZipLibraryInstall(ZipLibraryInstallReq) returns (ZipLibraryInstallResp);
+  rpc ZipLibraryInstall(ZipLibraryInstallReq) returns (stream ZipLibraryInstallResp);
 
   // Download and install a library from a git url
-  rpc GitLibraryInstall(GitLibraryInstallReq) returns (GitLibraryInstallResp);
+  rpc GitLibraryInstall(GitLibraryInstallReq) returns (stream GitLibraryInstallResp);
 
   // Uninstall an Arduino library.
   rpc LibraryUninstall(LibraryUninstallReq) returns (stream LibraryUninstallResp);

rpc/commands/commands.pb.go

@@ -296,20 +296,25 @@ enum LibraryLocation {
 }
 
 message ZipLibraryInstallReq {
+    // Arduino Core Service instance from the `Init` response.
     Instance instance = 1;
+    //Path to the archived library
     string Path = 2;
 }
 
 message ZipLibraryInstallResp {
-    string status = 1;
+    // Description of the current stage of the installation.
+    TaskProgress task_progress = 1;
 }
 
 message GitLibraryInstallReq {
+    // Arduino Core Service instance from the `Init` response.
     Instance instance = 1;
-    string name = 2;
-    string url = 3;
+    // URL to the repository containing the library
+    string url = 2;
 }
 
 message GitLibraryInstallResp {
-    string status = 1;
+    // Description of the current stage of the installation.
+    TaskProgress task_progress = 1;
 } 

rpc/commands/commands.proto

@@ -12,6 +12,7 @@
 # otherwise use the software for commercial activities involving the Arduino
 # software without disclosing the source code of your own applications. To purchase
 # a commercial license, send an email to license@arduino.cc.
+import os
 import simplejson as json
 
 
@@ -140,6 +141,21 @@ def test_list_with_fqbn(run_command):
     assert data[0]["library"]["compatible_with"]["arduino:avr:uno"]
 
 
+def test_lib_download(run_command, downloads_dir):
+
+    # Download a specific lib version
+    assert run_command("lib download AudioZero@1.0.0")
+    assert os.path.exists(os.path.join(downloads_dir, "libraries", "AudioZero-1.0.0.zip"))
+
+    # Wrong lib version
+    result = run_command("lib download AudioZero@69.42.0")
+    assert result.failed
+
+    # Wrong lib
+    result = run_command("lib download AudioZ")
+    assert result.failed
+
+
 def test_install(run_command):
     libs = ['"AzureIoTProtocol_MQTT"', '"CMMC MQTT Connector"', '"WiFiNINA"']
     # Should be safe to run install multiple times
@@ -152,6 +168,25 @@ def test_install(run_command):
     assert "Error resolving dependencies for MD_Parola@3.2.0: dependency 'MD_MAX72xx' is not available" in result.stderr
 
 
+def test_install_with_git_url(run_command):
+    # Test git-url library install
+    assert run_command("lib install --git-url https://github.com/arduino-libraries/WiFi101.git")
+
+    # Test failing-install as repository already exists
+    result = run_command("lib install --git-url https://github.com/arduino-libraries/WiFi101.git")
+    assert "Error installing Git Library: repository already exists" in result.stderr
+
+
+def test_install_with_zip_path(run_command, downloads_dir):
+    # Download a specific lib version
+    assert run_command("lib download AudioZero@1.0.0")
+    assert os.path.exists(os.path.join(downloads_dir, "libraries", "AudioZero-1.0.0.zip"))
+
+    zip_path = os.path.join(downloads_dir, "libraries", "AudioZero-1.0.0.zip")
+    # Test zip-path install
+    assert run_command("lib install --zip-path {zip_path}".format(zip_path=zip_path))
+
+
 def test_update_index(run_command):
     result = run_command("lib update-index")
     assert result.ok